@dxos/echo 0.8.4-main.e098934 → 0.8.4-main.ead640a

package/src/internal/object/json-serializer.ts

@@ -0,0 +1,225 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Schema from 'effect/Schema';
+
+import { raise } from '@dxos/debug';
+import { type EncodedReference, type ObjectMeta, isEncodedReference } from '@dxos/echo-protocol';
+import { assertArgument, invariant } from '@dxos/invariant';
+import { DXN, ObjectId } from '@dxos/keys';
+import { defineHiddenProperty } from '@dxos/live-object';
+import { assumeType, deepMapValues, visitValues } from '@dxos/util';
+
+import { EntityKind } from '../ast';
+import { Ref, type RefResolver, refFromEncodedReference, setRefResolver } from '../ref';
+import { type AnyEchoObject } from '../types';
+
+import { setSchema } from './accessors';
+import { ObjectMetaSchema } from './meta';
+import {
+  ATTR_DELETED,
+  ATTR_META,
+  ATTR_RELATION_SOURCE,
+  ATTR_RELATION_TARGET,
+  ATTR_SELF_DXN,
+  ATTR_TYPE,
+  EntityKindId,
+  MetaId,
+  type ObjectJSON,
+  RelationSourceDXNId,
+  RelationSourceId,
+  RelationTargetDXNId,
+  RelationTargetId,
+  SelfDXNId,
+  TypeId,
+  assertObjectModelShape,
+} from './model';
+import { getType, setTypename } from './typename';
+
+type DeepReplaceRef<T> =
+  T extends Ref<any> ? EncodedReference : T extends object ? { [K in keyof T]: DeepReplaceRef<T[K]> } : T;
+
+type SerializedObject<T extends { id: string }> = { [K in keyof T]: DeepReplaceRef<T[K]> } & ObjectJSON;
+
+/**
+ * Converts object to it's JSON representation.
+ */
+export const objectToJSON = <T extends AnyEchoObject>(obj: T): SerializedObject<T> => {
+  const typename = getType(obj)?.toString();
+  invariant(typename && typeof typename === 'string');
+  return typedJsonSerializer.call(obj);
+};
+
+/**
+ * Creates an object from it's json representation.
+ * Performs schema validation.
+ * References and schema will be resolvable if the `refResolver` is provided.
+ *
+ * The function need to be async to support resolving the schema as well as the relation endpoints.
+ */
+export const objectFromJSON = async (
+  jsonData: unknown,
+  { refResolver, dxn }: { refResolver?: RefResolver; dxn?: DXN } = {},
+): Promise<AnyEchoObject> => {
+  assumeType<ObjectJSON>(jsonData);
+  assertArgument(typeof jsonData === 'object' && jsonData !== null, 'jsonData', 'expect object');
+  assertArgument(typeof jsonData[ATTR_TYPE] === 'string', 'jsonData[ATTR_TYPE]', 'expected object to have a type');
+  assertArgument(typeof jsonData.id === 'string', 'jsonData.id', 'expected object to have an id');
+
+  const type = DXN.parse(jsonData[ATTR_TYPE]);
+  const schema = await refResolver?.resolveSchema(type);
+  invariant(schema === undefined || Schema.isSchema(schema));
+
+  let obj: any;
+  if (schema != null) {
+    obj = await schema.pipe(Schema.decodeUnknownPromise)(jsonData);
+    if (refResolver) {
+      setRefResolverOnData(obj, refResolver);
+    }
+  } else {
+    obj = decodeGeneric(jsonData, { refResolver });
+  }
+
+  invariant(ObjectId.isValid(obj.id), 'Invalid object id');
+
+  setTypename(obj, type);
+  if (schema) {
+    setSchema(obj, schema);
+  }
+
+  const isRelation =
+    typeof jsonData[ATTR_RELATION_SOURCE] === 'string' || typeof jsonData[ATTR_RELATION_TARGET] === 'string';
+  if (isRelation) {
+    const sourceDxn: DXN = DXN.parse(jsonData[ATTR_RELATION_SOURCE] ?? raise(new TypeError('Missing relation source')));
+    const targetDxn: DXN = DXN.parse(jsonData[ATTR_RELATION_TARGET] ?? raise(new TypeError('Missing relation target')));
+
+    // TODO(dmaretskyi): Async!
+    const source = (await refResolver?.resolve(sourceDxn)) as AnyEchoObject | undefined;
+    const target = (await refResolver?.resolve(targetDxn)) as AnyEchoObject | undefined;
+
+    defineHiddenProperty(obj, EntityKindId, EntityKind.Relation);
+    defineHiddenProperty(obj, RelationSourceDXNId, sourceDxn);
+    defineHiddenProperty(obj, RelationTargetDXNId, targetDxn);
+    defineHiddenProperty(obj, RelationSourceId, source);
+    defineHiddenProperty(obj, RelationTargetId, target);
+  } else {
+    defineHiddenProperty(obj, EntityKindId, EntityKind.Object);
+  }
+
+  if (typeof jsonData[ATTR_META] === 'object') {
+    const meta = await ObjectMetaSchema.pipe(Schema.decodeUnknownPromise)(jsonData[ATTR_META]);
+
+    // Defensive programming.
+    invariant(Array.isArray(meta.keys));
+
+    defineHiddenProperty(obj, MetaId, meta);
+  }
+
+  if (dxn) {
+    defineHiddenProperty(obj, SelfDXNId, dxn);
+  }
+
+  assertObjectModelShape(obj);
+  invariant((obj as any)[ATTR_TYPE] === undefined, 'Invalid object model');
+  invariant((obj as any)[ATTR_SELF_DXN] === undefined, 'Invalid object model');
+  invariant((obj as any)[ATTR_DELETED] === undefined, 'Invalid object model');
+  invariant((obj as any)[ATTR_RELATION_SOURCE] === undefined, 'Invalid object model');
+  invariant((obj as any)[ATTR_RELATION_TARGET] === undefined, 'Invalid object model');
+  invariant((obj as any)[ATTR_META] === undefined, 'Invalid object model');
+  return obj;
+};
+
+const decodeGeneric = (jsonData: unknown, options: { refResolver?: RefResolver }) => {
+  const {
+    [ATTR_TYPE]: _type,
+    [ATTR_META]: _meta,
+    [ATTR_DELETED]: _deleted,
+    [ATTR_RELATION_SOURCE]: _relationSource,
+    [ATTR_RELATION_TARGET]: _relationTarget,
+    [ATTR_SELF_DXN]: _selfDxn,
+    ...props
+  } = jsonData as any;
+
+  return deepMapValues(props, (value, recurse) => {
+    if (isEncodedReference(value)) {
+      return refFromEncodedReference(value, options.refResolver);
+    }
+    return recurse(value);
+  });
+};
+
+export const setRefResolverOnData = (obj: AnyEchoObject, refResolver: RefResolver) => {
+  const go = (value: unknown) => {
+    if (Ref.isRef(value)) {
+      setRefResolver(value, refResolver);
+    } else {
+      visitValues(value, go);
+    }
+  };
+
+  go(obj);
+};
+
+export const attachTypedJsonSerializer = (obj: any) => {
+  const descriptor = Object.getOwnPropertyDescriptor(obj, 'toJSON');
+  if (descriptor) {
+    return;
+  }
+
+  Object.defineProperty(obj, 'toJSON', {
+    value: typedJsonSerializer,
+    writable: false,
+    enumerable: false,
+    // Setting `configurable` to false breaks proxy invariants, should be fixable.
+    configurable: true,
+  });
+};
+
+// NOTE: KEEP as function.
+const typedJsonSerializer = function (this: any) {
+  const { id, ...rest } = this;
+  const result: any = {
+    id,
+  };
+
+  if (this[TypeId]) {
+    result[ATTR_TYPE] = this[TypeId].toString();
+  }
+
+  if (this[SelfDXNId]) {
+    result[ATTR_SELF_DXN] = this[SelfDXNId].toString();
+  }
+
+  if (this[RelationSourceDXNId]) {
+    const sourceDXN = this[RelationSourceDXNId];
+    invariant(sourceDXN instanceof DXN);
+    result[ATTR_RELATION_SOURCE] = sourceDXN.toString();
+  }
+  if (this[RelationTargetDXNId]) {
+    const targetDXN = this[RelationTargetDXNId];
+    invariant(targetDXN instanceof DXN);
+    result[ATTR_RELATION_TARGET] = targetDXN.toString();
+  }
+
+  if (this[MetaId]) {
+    result[ATTR_META] = serializeMeta(this[MetaId]);
+  }
+
+  Object.assign(result, serializeData(rest));
+  return result;
+};
+
+const serializeData = (data: unknown) => {
+  return deepMapValues(data, (value, recurse) => {
+    if (Ref.isRef(value)) {
+      // TODO(dmaretskyi): Should this be configurable?
+      return value.noInline().encode();
+    }
+    return recurse(value);
+  });
+};
+
+const serializeMeta = (meta: ObjectMeta) => {
+  return deepMapValues(meta, (value, recurse) => recurse(value));
+};

package/src/internal/object/meta.ts

@@ -0,0 +1,61 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import * as Schema from 'effect/Schema';
+
+import { ForeignKey } from '@dxos/echo-protocol';
+import { invariant } from '@dxos/invariant';
+import { type Comparator, intersection } from '@dxos/util';
+
+import { type BaseObject } from '../types';
+
+import { MetaId } from './model';
+
+//
+// ObjectMeta
+//
+
+// TODO(dmaretskyi): Rename to ObjectMeta
+export const ObjectMetaSchema = Schema.mutable(
+  Schema.Struct({
+    keys: Schema.mutable(Schema.Array(ForeignKey)),
+
+    /**
+     * A set of tags.
+     * Tags are arbitrary application-defined strings.
+     * ECHO makes no assumptions about the tag structure.
+     */
+    // TODO(dmaretskyi): Has to be optional for compatibility with old data.
+    // Defaulting to an empty array is possible but requires a bit more work.
+    tags: Schema.optional(Schema.mutable(Schema.Array(Schema.String))),
+  }),
+);
+export type ObjectMeta = Schema.Schema.Type<typeof ObjectMetaSchema>;
+
+export const foreignKey = (source: string, id: string): ForeignKey => ({ source, id });
+export const foreignKeyEquals = (a: ForeignKey, b: ForeignKey) => a.source === b.source && a.id === b.id;
+
+/**
+ * Get metadata from object.
+ * Only callable on the object root.
+ * @deprecated Use {@link getMeta}.
+ */
+// TODO(dmaretskyi): Remove.
+export const getObjectMeta = (obj: any): ObjectMeta => {
+  return getMeta(obj);
+};
+
+/*
+ * Get metadata from object.
+ * Only callable on the object root.
+ */
+export const getMeta = (obj: BaseObject): ObjectMeta => {
+  const metadata = (obj as any)[MetaId];
+  invariant(metadata, 'ObjectMeta not found.');
+  return metadata;
+};
+
+// TODO(dmaretskyi): Move to echo-schema.
+export const compareForeignKeys: Comparator<BaseObject> = (a: BaseObject, b: BaseObject) =>
+  intersection(getMeta(a).keys, getMeta(b).keys, foreignKeyEquals).length > 0;

package/src/internal/object/model.ts

@@ -0,0 +1,170 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import type * as Schema from 'effect/Schema';
+
+import { type ForeignKey } from '@dxos/echo-protocol';
+import { invariant } from '@dxos/invariant';
+import { DXN, ObjectId } from '@dxos/keys';
+import { assumeType } from '@dxos/util';
+
+import { EntityKind } from '../ast';
+
+import { type ObjectMeta } from './meta';
+import type { Version } from './version';
+
+//
+// Defines the internal model of the echo object.
+//
+
+/**
+ * Entity kind.
+ */
+export const EntityKindId = Symbol('@dxos/echo/EntityKind');
+
+/**
+ * DXN to the object itself.
+ */
+export const SelfDXNId = Symbol('@dxos/echo/DXN');
+
+/**
+ * Property name for self DXN when object is serialized to JSON.
+ */
+export const ATTR_SELF_DXN = '@dxn';
+
+/**
+ * DXN to the object type.
+ */
+export const TypeId = Symbol('@dxos/echo/Type');
+
+/**
+ * Property name for typename when object is serialized to JSON.
+ */
+export const ATTR_TYPE = '@type';
+
+/**
+ * Reference to the object schema.
+ */
+export const SchemaId = Symbol('@dxos/echo/Schema');
+
+/**
+ * Deletion marker.
+ */
+export const DeletedId = Symbol('@dxos/echo/Deleted');
+
+/**
+ * Property name for deleted when object is serialized to JSON.
+ */
+export const ATTR_DELETED = '@deleted';
+
+/**
+ * Metadata section.
+ */
+export const MetaId = Symbol('@dxos/echo/Meta');
+
+/**
+ * Property name for meta when object is serialized to JSON.
+ */
+export const ATTR_META = '@meta';
+
+/**
+ * Used to access relation source ref on live ECHO objects.
+ * Reading this symbol must return `Live<EchoObject<any>>` or a DXN.
+ */
+export const RelationSourceDXNId: unique symbol = Symbol('@dxos/echo/RelationSourceDXN');
+
+/**
+ * Property name for relation source when object is serialized to JSON.
+ */
+export const ATTR_RELATION_SOURCE = '@relationSource';
+
+/**
+ * Used to access relation target ref on live ECHO objects.
+ * Reading this symbol must return `Live<EchoObject<any>>` or a DXN.
+ */
+export const RelationTargetDXNId: unique symbol = Symbol('@dxos/echo/RelationTargetDXN');
+
+/**
+ * Property name for relation target when object is serialized to JSON.
+ */
+export const ATTR_RELATION_TARGET = '@relationTarget';
+
+/**
+ * Used to access relation source ref on live ECHO objects.
+ * Reading this symbol must return `Live<EchoObject<any>>` or a DXN.
+ */
+export const RelationSourceId: unique symbol = Symbol('@dxos/echo/RelationSource');
+
+/**
+ * Used to access relation target ref on live ECHO objects.
+ * Reading this symbol must return `Live<EchoObject<any>>` or a DXN.
+ */
+export const RelationTargetId: unique symbol = Symbol('@dxos/echo/RelationTarget');
+
+/**
+ * Getter to get object version.
+ */
+export const ObjectVersionId: unique symbol = Symbol('@dxos/echo/ObjectVersion');
+
+/**
+ * Internal runtime representation of an object.
+ * The fields are accessed through getter functions.
+ */
+export interface InternalObjectProps {
+  id: ObjectId;
+  readonly [SelfDXNId]?: DXN;
+
+  // Echo supports untyped objects O_O.
+  readonly [TypeId]?: DXN;
+
+  /**
+   * Returns the schema for the object.
+   */
+  readonly [SchemaId]?: Schema.Schema.AnyNoContext;
+  readonly [EntityKindId]: EntityKind;
+  readonly [DeletedId]?: boolean;
+  readonly [MetaId]?: ObjectMeta;
+  readonly [RelationSourceDXNId]?: DXN;
+  readonly [RelationTargetDXNId]?: DXN;
+  readonly [RelationSourceId]?: InternalObjectProps;
+  readonly [RelationTargetId]?: InternalObjectProps;
+  readonly [ObjectVersionId]?: Version;
+}
+
+/**
+ * JSON representation of an object or relation.
+ */
+export interface ObjectJSON {
+  id: string;
+  [ATTR_SELF_DXN]?: DXN.String;
+  [ATTR_TYPE]: DXN.String;
+  [ATTR_DELETED]?: boolean;
+  [ATTR_META]?: ObjectMetaJSON;
+  [ATTR_RELATION_SOURCE]?: DXN.String;
+  [ATTR_RELATION_TARGET]?: DXN.String;
+}
+
+export interface ObjectMetaJSON {
+  keys: ForeignKey[];
+  tags?: string[];
+}
+
+// NOTE: Keep as `function` to avoid type inference issues.
+// eslint-disable-next-line prefer-arrow-functions/prefer-arrow-functions
+export function assertObjectModelShape(obj: unknown): asserts obj is InternalObjectProps {
+  invariant(typeof obj === 'object' && obj !== null, 'Invalid object model: not an object');
+  assumeType<InternalObjectProps>(obj);
+  invariant(ObjectId.isValid(obj.id), 'Invalid object model: invalid id');
+  invariant(obj[TypeId] === undefined || obj[TypeId] instanceof DXN, 'Invalid object model: invalid type');
+  invariant(
+    obj[EntityKindId] === EntityKind.Object || obj[EntityKindId] === EntityKind.Relation,
+    'Invalid object model: invalid entity kind',
+  );
+  if (obj[EntityKindId] === EntityKind.Relation) {
+    invariant(obj[RelationSourceDXNId] instanceof DXN, 'Invalid object model: invalid relation source');
+    invariant(obj[RelationTargetDXNId] instanceof DXN, 'Invalid object model: invalid relation target');
+    invariant(!(obj[RelationSourceId] instanceof DXN), 'Invalid object model: source pointer is a DXN');
+    invariant(!(obj[RelationTargetId] instanceof DXN), 'Invalid object model: target pointer is a DXN');
+  }
+}

package/src/internal/object/relation.ts

@@ -0,0 +1,24 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import type { RelationSourceId, RelationTargetId } from './model';
+
+/**
+ * Source and target props on relations.
+ */
+// TODO(burdon): any?
+export type RelationSourceTargetRefs<Source = any, Target = any> = {
+  /**
+   * Source ECHO live object.
+   */
+  [RelationSourceId]: Source;
+
+  /**
+   * Target ECHO live object.
+   */
+  [RelationTargetId]: Target;
+};
+
+export type RelationSource<R> = R extends RelationSourceTargetRefs<infer Source, infer _Target> ? Source : never;
+export type RelationTarget<R> = R extends RelationSourceTargetRefs<infer _Source, infer Target> ? Target : never;

package/src/internal/object/schema-validator.test.ts

@@ -0,0 +1,186 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import * as Schema from 'effect/Schema';
+import { describe, expect, test } from 'vitest';
+
+import { getDeep } from '@dxos/util';
+
+import { SchemaValidator } from './schema-validator';
+
+describe('schema-validator', () => {
+  describe('validateSchema', () => {
+    test('throws on ambiguous discriminated type union', () => {
+      const TestSchema = Schema.Struct({
+        union: Schema.Union(Schema.Struct({ a: Schema.Number }), Schema.Struct({ b: Schema.String })),
+      });
+
+      expect(() => SchemaValidator.validateSchema(TestSchema)).to.throw();
+    });
+  });
+
+  describe('hasPropertyAnnotation', () => {
+    test('has annotation', () => {
+      const annotationId = Symbol('foo');
+      const annotationValue = 'bar';
+      const TestSchema: Schema.Schema.AnyNoContext = Schema.Struct({
+        name: Schema.String.annotations({ [annotationId]: annotationValue }),
+        parent: Schema.optional(Schema.suspend(() => TestSchema.annotations({ [annotationId]: annotationValue }))),
+        friends: Schema.suspend(() =>
+          Schema.mutable(Schema.Array(TestSchema.annotations({ [annotationId]: annotationValue }))),
+        ),
+      });
+      expect(SchemaValidator.hasTypeAnnotation(TestSchema, 'name', annotationId)).to.be.true;
+      expect(SchemaValidator.hasTypeAnnotation(TestSchema, 'parent', annotationId)).to.be.true;
+      expect(SchemaValidator.hasTypeAnnotation(TestSchema, 'friends', annotationId)).to.be.true;
+    });
+
+    test('no annotation', () => {
+      const annotationId = Symbol('foo');
+      const Person: Schema.Schema.AnyNoContext = Schema.Struct({
+        name: Schema.String,
+        parent: Schema.optional(Schema.suspend(() => Person)),
+        friends: Schema.suspend(() => Schema.mutable(Schema.Array(Person))),
+      });
+      expect(SchemaValidator.hasTypeAnnotation(Person, 'name', annotationId)).to.be.false;
+      expect(SchemaValidator.hasTypeAnnotation(Person, 'parent', annotationId)).to.be.false;
+      expect(SchemaValidator.hasTypeAnnotation(Person, 'friends', annotationId)).to.be.false;
+    });
+  });
+
+  describe('getPropertySchema', () => {
+    const validateValueToAssign = (args: {
+      schema: Schema.Schema.AnyNoContext;
+      target: any;
+      path: string[];
+      valueToAssign: any;
+      expectToThrow?: boolean;
+    }) => {
+      const expectation = expect(() => {
+        const nestedSchema = SchemaValidator.getPropertySchema(args.schema, args.path, (path) => {
+          return getDeep(args.target, path);
+        });
+        Schema.validateSync(nestedSchema)(args.valueToAssign);
+      });
+      if (args.expectToThrow) {
+        expectation.to.throw();
+      } else {
+        expectation.not.to.throw();
+      }
+    };
+
+    test('basic', () => {
+      for (const value of [42, '42']) {
+        validateValueToAssign({
+          schema: Schema.Struct({ object: Schema.Struct({ field: Schema.Number }) }),
+          target: {},
+          path: ['object', 'field'],
+          valueToAssign: value,
+          expectToThrow: typeof value !== 'number',
+        });
+      }
+    });
+
+    test('preserves annotations', () => {
+      const annotationId = Symbol('foo');
+      const annotationValue = 'bar';
+      const Person: Schema.Schema.AnyNoContext = Schema.Struct({
+        parent: Schema.optional(Schema.suspend(() => Person.annotations({ [annotationId]: annotationValue }))),
+        friends: Schema.suspend(() =>
+          Schema.mutable(Schema.Array(Person.annotations({ [annotationId]: annotationValue }))),
+        ),
+      });
+      expect(SchemaValidator.getPropertySchema(Person, ['parent']).ast.annotations[annotationId]).to.eq(
+        annotationValue,
+      );
+      expect(SchemaValidator.getPropertySchema(Person, ['friends', '0']).ast.annotations[annotationId]).to.eq(
+        annotationValue,
+      );
+    });
+
+    test('discriminated union', () => {
+      const square = Schema.Struct({ type: Schema.Literal('square'), side: Schema.Number });
+      const circle = Schema.Struct({ type: Schema.Literal('circle'), radius: Schema.Number });
+      const shape = Schema.Union(square, circle);
+      validateValueToAssign({
+        schema: shape,
+        target: { type: 'square' },
+        path: ['side'],
+        valueToAssign: 1,
+      });
+      validateValueToAssign({
+        schema: shape,
+        target: { type: 'circle' },
+        path: ['side'],
+        valueToAssign: 1,
+        expectToThrow: true,
+      });
+      validateValueToAssign({
+        schema: shape,
+        target: { type: 'square' },
+        path: ['radius'],
+        valueToAssign: 1,
+        expectToThrow: true,
+      });
+    });
+
+    test('any', () => {
+      validateValueToAssign({
+        schema: Schema.Any,
+        target: { field: { nested: { value: Schema.Number } } },
+        path: ['field', 'nested'],
+        valueToAssign: { any: 'value' },
+      });
+    });
+    test('index signatures', () => {
+      for (const value of [42, '42']) {
+        validateValueToAssign({
+          schema: Schema.Struct({ field: Schema.String }, { key: Schema.String, value: Schema.Number }),
+          target: {},
+          path: ['unknownField'],
+          valueToAssign: value,
+          expectToThrow: typeof value !== 'number',
+        });
+      }
+    });
+
+    test('index signature from optional record', () => {
+      for (const value of [42, '42']) {
+        validateValueToAssign({
+          schema: Schema.Struct({
+            field: Schema.optional(Schema.Record({ key: Schema.String, value: Schema.Number })),
+          }),
+          target: {},
+          path: ['field', 'unknownField'],
+          valueToAssign: value,
+          expectToThrow: typeof value !== 'number',
+        });
+      }
+    });
+
+    test('suspend', () => {
+      const schemaWithSuspend = Schema.Struct({
+        array: Schema.optional(Schema.suspend(() => Schema.Array(Schema.Union(Schema.Null, Schema.Number)))),
+        object: Schema.optional(
+          Schema.suspend(() => Schema.Union(Schema.Null, Schema.Struct({ field: Schema.Number }))),
+        ),
+      });
+      const target: any = { array: [1, 2, null], object: { field: 3 } };
+      for (const value of [42, '42']) {
+        for (const path of [
+          ['array', '0'],
+          ['object', 'field'],
+        ]) {
+          validateValueToAssign({
+            schema: schemaWithSuspend,
+            target,
+            path,
+            valueToAssign: value,
+            expectToThrow: typeof value !== 'number',
+          });
+        }
+      }
+    });
+  });
+});