@dxos/echo 0.8.4-main.c4373fc → 0.8.4-main.cb12b3f963

package/src/internal/Obj/json-serializer.test.ts

@@ -0,0 +1,120 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { describe, expect, test } from 'vitest';
+
+import { DXN } from '@dxos/keys';
+
+import * as Obj from '../../Obj';
+import { TestSchema } from '../../testing';
+import { getSchemaDXN, getSchemaTypename, getTypeDXN, getTypename } from '../Annotation';
+import { getMetaChecked } from '../common/api';
+import { makeObject } from '../common/proxy';
+import { ATTR_TYPE, EntityKind, KindId, MetaId, TypeId, getSchema } from '../common/types';
+import { RelationSourceId, RelationTargetId, getObjectDXN } from '../Entity';
+import { Ref, StaticRefResolver } from '../Ref';
+import { createObject } from './create-object';
+import { objectFromJSON, objectToJSON } from './json-serializer';
+
+describe('Object JSON serializer', () => {
+  test('should serialize and deserialize object', async () => {
+    const contact = makeObject(TestSchema.Person, { name: 'Alice' });
+    Obj.update(contact, (contact) => {
+      getMetaChecked(contact).keys.push({ id: '12345', source: 'example.com' });
+    });
+
+    const task = createObject(TestSchema.Task, {
+      title: 'Fix the tests',
+      assignee: Ref.make(contact),
+    });
+
+    const contactJson = objectToJSON(contact);
+    const taskJson = objectToJSON(task);
+
+    expect(contactJson.id).toBe(contact.id);
+    expect(contactJson[ATTR_TYPE]).toEqual(getSchemaDXN(TestSchema.Person)!.toString());
+    expect(contactJson.name).toEqual('Alice');
+
+    expect(taskJson.id).toBe(task.id);
+    expect(taskJson[ATTR_TYPE]).toEqual(getSchemaDXN(TestSchema.Task)!.toString());
+    expect(taskJson.title).toEqual('Fix the tests');
+    expect(taskJson.assignee).toEqual({ '/': DXN.fromLocalObjectId(contact.id).toString() });
+
+    const refResolver = new StaticRefResolver()
+      .addSchema(TestSchema.Person)
+      .addSchema(TestSchema.Task)
+      .addObject(contact)
+      .addObject(task);
+
+    const contactFromJson = (await objectFromJSON(contactJson, { refResolver })) as TestSchema.Person;
+    const taskFromJson = (await objectFromJSON(taskJson, { refResolver })) as TestSchema.Task;
+
+    expect(contactFromJson.id).toBe(contact.id);
+    expect(contactFromJson.name).toBe('Alice');
+    expect((contactFromJson as any)[TypeId]).toEqual(getSchemaDXN(TestSchema.Person));
+    expect((contactFromJson as any)[KindId]).toBe(EntityKind.Object);
+    expect((contactFromJson as any)[RelationSourceId]).toBeUndefined();
+    expect((contactFromJson as any)[RelationTargetId]).toBeUndefined();
+    expect((contactFromJson as any)[MetaId]).toEqual({
+      keys: [
+        {
+          id: '12345',
+          source: 'example.com',
+        },
+      ],
+    });
+    expect(getTypeDXN(contactFromJson)?.toString()).toBe(getSchemaDXN(TestSchema.Person)!.toString());
+    expect(getTypename(contactFromJson)).toBe(getSchemaTypename(TestSchema.Person));
+    expect(getObjectDXN(contactFromJson)?.toString()).toEqual(getObjectDXN(contact)?.toString());
+    expect(getSchema(contactFromJson)).toEqual(TestSchema.Person);
+
+    expect(taskFromJson.id).toBe(task.id);
+    expect(taskFromJson.title).toBe('Fix the tests');
+    expect(taskFromJson.assignee!.dxn).toEqual(DXN.fromLocalObjectId(contact.id));
+    expect(taskFromJson.assignee!.target).toEqual(contact);
+    expect(await taskFromJson.assignee!.load()).toEqual(contact);
+    expect((taskFromJson as any)[TypeId]).toEqual(getSchemaDXN(TestSchema.Task));
+    expect((taskFromJson as any)[KindId]).toBe(EntityKind.Object);
+    expect((taskFromJson as any)[RelationSourceId]).toBeUndefined();
+    expect((taskFromJson as any)[RelationTargetId]).toBeUndefined();
+    expect((taskFromJson as any)[MetaId]).toEqual({ keys: [] });
+    expect(getSchema(taskFromJson)).toEqual(TestSchema.Task);
+  });
+
+  test('serialize with unresolved schema', async () => {
+    const contact = createObject(TestSchema.Person, { name: 'Alice' });
+    const contactJson = objectToJSON(contact);
+    const contactFromJson: any = await objectFromJSON(contactJson);
+
+    expect(contactFromJson.id).toBe(contact.id);
+    expect(contactFromJson.name).toBe('Alice');
+    expect(getSchema(contactFromJson)).toBeUndefined();
+    expect(getTypename(contactFromJson)).toEqual(getSchemaTypename(TestSchema.Person));
+    expect(getObjectDXN(contactFromJson)).toEqual(getObjectDXN(contact));
+    expect(getTypeDXN(contactFromJson)).toEqual(getSchemaDXN(TestSchema.Person));
+  });
+
+  test('deserializes expando without leaking internal json keys', async () => {
+    const expando = Obj.make(TestSchema.Expando, { message: 'local-only' });
+    const expandoJson = objectToJSON(expando);
+
+    const refResolver = new StaticRefResolver().addSchema(TestSchema.Expando);
+    const expandoFromJson = (await objectFromJSON(expandoJson, { refResolver })) as TestSchema.Expando;
+
+    expect(expandoFromJson.id).toBe(expando.id);
+    expect(expandoFromJson.message).toBe('local-only');
+    expect((expandoFromJson as any)[ATTR_TYPE]).toBeUndefined();
+  });
+
+  test('deserializes expando without schema resolver and without leaking internal json keys', async () => {
+    const expando = Obj.make(TestSchema.Expando, { message: 'local-only' });
+    const expandoJson = objectToJSON(expando);
+
+    const expandoFromJson = (await objectFromJSON(expandoJson)) as TestSchema.Expando;
+
+    expect(expandoFromJson.id).toBe(expando.id);
+    expect(expandoFromJson.message).toBe('local-only');
+    expect((expandoFromJson as any)[ATTR_TYPE]).toBeUndefined();
+  });
+});

package/src/internal/{object → Obj}/json-serializer.ts

@@ -5,48 +5,64 @@
 import * as Schema from 'effect/Schema';
 
 import { raise } from '@dxos/debug';
-import { type EncodedReference, type ObjectMeta, isEncodedReference } from '@dxos/echo-protocol';
+import { type EncodedReference, ObjectStructure, isEncodedReference } from '@dxos/echo-protocol';
 import { assertArgument, invariant } from '@dxos/invariant';
 import { DXN, ObjectId } from '@dxos/keys';
-import { defineHiddenProperty } from '@dxos/live-object';
 import { assumeType, deepMapValues, visitValues } from '@dxos/util';
 
-import { EntityKind } from '../ast';
-import { Ref, type RefResolver, refFromEncodedReference, setRefResolver } from '../ref';
-import { type AnyEchoObject } from '../types';
-
-import { setSchema } from './accessors';
-import { ObjectMetaSchema } from './meta';
+import type * as Database from '../../Database';
+import type * as Obj from '../../Obj';
+import { getTypeDXN, setTypename } from '../Annotation';
+import { attachTypedJsonSerializer, defineHiddenProperty, typedJsonSerializer } from '../common/proxy';
 import {
-  ATTR_DELETED,
   ATTR_META,
+  ATTR_PARENT,
+  ATTR_TYPE,
+  type AnyEntity,
+  EntityKind,
+  KindId,
+  MetaId,
+  ObjectMetaSchema,
+  ParentId,
+  setSchema,
+} from '../common/types';
+import {
+  ATTR_DELETED,
   ATTR_RELATION_SOURCE,
   ATTR_RELATION_TARGET,
   ATTR_SELF_DXN,
-  ATTR_TYPE,
-  EntityKindId,
-  MetaId,
+  ObjectDatabaseId,
   type ObjectJSON,
   RelationSourceDXNId,
   RelationSourceId,
   RelationTargetDXNId,
   RelationTargetId,
   SelfDXNId,
-  TypeId,
-  assertObjectModelShape,
-} from './model';
-import { getType, setTypename } from './typename';
+  assertObjectModel,
+} from '../Entity';
+import { Ref, type RefResolver, refFromEncodedReference, setRefResolver } from '../Ref';
 
-type DeepReplaceRef<T> =
-  T extends Ref<any> ? EncodedReference : T extends object ? { [K in keyof T]: DeepReplaceRef<T[K]> } : T;
+// Re-export for backward compatibility.
+export { attachTypedJsonSerializer };
 
-type SerializedObject<T extends { id: string }> = { [K in keyof T]: DeepReplaceRef<T[K]> } & ObjectJSON;
+type DeepReplaceRef<T> =
+  T extends Ref<any>
+    ? EncodedReference
+    : T extends object
+      ? {
+          [K in keyof T]: DeepReplaceRef<T[K]>;
+        }
+      : T;
+
+type SerializedObject<T extends { id: string }> = {
+  [K in keyof T]: DeepReplaceRef<T[K]>;
+} & ObjectJSON;
 
 /**
  * Converts object to it's JSON representation.
  */
-export const objectToJSON = <T extends AnyEchoObject>(obj: T): SerializedObject<T> => {
-  const typename = getType(obj)?.toString();
+export const objectToJSON = <T extends AnyEntity>(obj: T): SerializedObject<T> => {
+  const typename = getTypeDXN(obj)?.toString();
   invariant(typename && typeof typename === 'string');
   return typedJsonSerializer.call(obj);
 };
@@ -55,13 +71,22 @@ export const objectToJSON = <T extends AnyEchoObject>(obj: T): SerializedObject<
  * Creates an object from it's json representation.
  * Performs schema validation.
  * References and schema will be resolvable if the `refResolver` is provided.
- *
  * The function need to be async to support resolving the schema as well as the relation endpoints.
+ *
+ * @param jsonData - JSON representation of the object.
+ * @param options.refResolver - Resolver for references.
+ * @param options.dxn - Override object DXN.
+ * @param options.database - Database to associate with the object.
  */
 export const objectFromJSON = async (
   jsonData: unknown,
-  { refResolver, dxn }: { refResolver?: RefResolver; dxn?: DXN } = {},
-): Promise<AnyEchoObject> => {
+  {
+    refResolver,
+    dxn,
+    database,
+    parent,
+  }: { refResolver?: RefResolver; dxn?: DXN; database?: Database.Database; parent?: Obj.Unknown } = {},
+): Promise<AnyEntity> => {
   assumeType<ObjectJSON>(jsonData);
   assertArgument(typeof jsonData === 'object' && jsonData !== null, 'jsonData', 'expect object');
   assertArgument(typeof jsonData[ATTR_TYPE] === 'string', 'jsonData[ATTR_TYPE]', 'expected object to have a type');
@@ -70,19 +95,19 @@ export const objectFromJSON = async (
   const type = DXN.parse(jsonData[ATTR_TYPE]);
   const schema = await refResolver?.resolveSchema(type);
   invariant(schema === undefined || Schema.isSchema(schema));
+  const decodedInput = stripInternalJsonKeys(jsonData);
 
   let obj: any;
   if (schema != null) {
-    obj = await schema.pipe(Schema.decodeUnknownPromise)(jsonData);
+    obj = await schema.pipe(Schema.decodeUnknownPromise)(decodedInput);
     if (refResolver) {
       setRefResolverOnData(obj, refResolver);
     }
   } else {
-    obj = decodeGeneric(jsonData, { refResolver });
+    obj = decodeGeneric(decodedInput, { refResolver });
   }
 
   invariant(ObjectId.isValid(obj.id), 'Invalid object id');
-
   setTypename(obj, type);
   if (schema) {
     setSchema(obj, schema);
@@ -94,132 +119,104 @@ export const objectFromJSON = async (
     const sourceDxn: DXN = DXN.parse(jsonData[ATTR_RELATION_SOURCE] ?? raise(new TypeError('Missing relation source')));
     const targetDxn: DXN = DXN.parse(jsonData[ATTR_RELATION_TARGET] ?? raise(new TypeError('Missing relation target')));
 
-    // TODO(dmaretskyi): Async!
-    const source = (await refResolver?.resolve(sourceDxn)) as AnyEchoObject | undefined;
-    const target = (await refResolver?.resolve(targetDxn)) as AnyEchoObject | undefined;
+    const source = (await refResolver?.resolve(sourceDxn)) as AnyEntity | undefined;
+    const target = (await refResolver?.resolve(targetDxn)) as AnyEntity | undefined;
 
-    defineHiddenProperty(obj, EntityKindId, EntityKind.Relation);
+    defineHiddenProperty(obj, KindId, EntityKind.Relation);
     defineHiddenProperty(obj, RelationSourceDXNId, sourceDxn);
     defineHiddenProperty(obj, RelationTargetDXNId, targetDxn);
     defineHiddenProperty(obj, RelationSourceId, source);
     defineHiddenProperty(obj, RelationTargetId, target);
   } else {
-    defineHiddenProperty(obj, EntityKindId, EntityKind.Object);
+    defineHiddenProperty(obj, KindId, EntityKind.Object);
   }
 
   if (typeof jsonData[ATTR_META] === 'object') {
     const meta = await ObjectMetaSchema.pipe(Schema.decodeUnknownPromise)(jsonData[ATTR_META]);
-
-    // Defensive programming.
     invariant(Array.isArray(meta.keys));
-
     defineHiddenProperty(obj, MetaId, meta);
+  } else {
+    defineHiddenProperty(obj, MetaId, {
+      keys: [],
+    });
+  }
+
+  if (jsonData[ATTR_PARENT]) {
+    const parentDxn = DXN.parse(jsonData[ATTR_PARENT]);
+    const resolvedParent = (await refResolver?.resolve(parentDxn)) as Obj.Unknown | undefined;
+    defineHiddenProperty(obj, ParentId, resolvedParent);
+  } else if (parent) {
+    defineHiddenProperty(obj, ParentId, parent);
   }
 
   if (dxn) {
     defineHiddenProperty(obj, SelfDXNId, dxn);
   }
 
-  assertObjectModelShape(obj);
+  if (database) {
+    defineHiddenProperty(obj, ObjectDatabaseId, database);
+  }
+
+  assertObjectModel(obj);
   invariant((obj as any)[ATTR_TYPE] === undefined, 'Invalid object model');
-  invariant((obj as any)[ATTR_SELF_DXN] === undefined, 'Invalid object model');
+  invariant((obj as any)[ATTR_META] === undefined, 'Invalid object model');
   invariant((obj as any)[ATTR_DELETED] === undefined, 'Invalid object model');
+  invariant((obj as any)[ATTR_SELF_DXN] === undefined, 'Invalid object model');
   invariant((obj as any)[ATTR_RELATION_SOURCE] === undefined, 'Invalid object model');
   invariant((obj as any)[ATTR_RELATION_TARGET] === undefined, 'Invalid object model');
-  invariant((obj as any)[ATTR_META] === undefined, 'Invalid object model');
   return obj;
 };
 
 const decodeGeneric = (jsonData: unknown, options: { refResolver?: RefResolver }) => {
+  const props = stripInternalJsonKeys(jsonData);
+
+  return deepMapValues(props, (value, visitor) => {
+    if (isEncodedReference(value)) {
+      return refFromEncodedReference(value, options.refResolver);
+    }
+
+    return visitor(value);
+  });
+};
+
+const stripInternalJsonKeys = (jsonData: unknown) => {
   const {
     [ATTR_TYPE]: _type,
     [ATTR_META]: _meta,
     [ATTR_DELETED]: _deleted,
+    [ATTR_SELF_DXN]: _selfDxn,
     [ATTR_RELATION_SOURCE]: _relationSource,
     [ATTR_RELATION_TARGET]: _relationTarget,
-    [ATTR_SELF_DXN]: _selfDxn,
     ...props
   } = jsonData as any;
 
-  return deepMapValues(props, (value, recurse) => {
-    if (isEncodedReference(value)) {
-      return refFromEncodedReference(value, options.refResolver);
-    }
-    return recurse(value);
-  });
+  return props;
 };
 
-export const setRefResolverOnData = (obj: AnyEchoObject, refResolver: RefResolver) => {
-  const go = (value: unknown) => {
+export const setRefResolverOnData = (obj: AnyEntity, refResolver: RefResolver) => {
+  const visitor = (value: unknown) => {
     if (Ref.isRef(value)) {
       setRefResolver(value, refResolver);
     } else {
-      visitValues(value, go);
+      visitValues(value, visitor);
     }
   };
 
-  go(obj);
-};
-
-export const attachTypedJsonSerializer = (obj: any) => {
-  const descriptor = Object.getOwnPropertyDescriptor(obj, 'toJSON');
-  if (descriptor) {
-    return;
-  }
-
-  Object.defineProperty(obj, 'toJSON', {
-    value: typedJsonSerializer,
-    writable: false,
-    enumerable: false,
-    // Setting `configurable` to false breaks proxy invariants, should be fixable.
-    configurable: true,
-  });
+  visitor(obj);
 };
 
-// NOTE: KEEP as function.
-const typedJsonSerializer = function (this: any) {
-  const { id, ...rest } = this;
-  const result: any = {
-    id,
+/**
+ * Convert ObjectStructure to JSON data for indexing.
+ * Different from {@link objectToJSON} as it takes the internal {@link ObjectStructure} representation directly
+ */
+export const objectStructureToJson = (objectId: string, structure: ObjectStructure): Obj.JSON => {
+  return {
+    ...structure.data,
+    id: objectId,
+    [ATTR_TYPE]: (ObjectStructure.getTypeReference(structure)?.['/'] ?? '') as DXN.String,
+    [ATTR_DELETED]: ObjectStructure.isDeleted(structure),
+    [ATTR_PARENT]: ObjectStructure.getParent(structure)?.['/'] as DXN.String | undefined,
+    [ATTR_RELATION_SOURCE]: ObjectStructure.getRelationSource(structure)?.['/'] as DXN.String | undefined,
+    [ATTR_RELATION_TARGET]: ObjectStructure.getRelationTarget(structure)?.['/'] as DXN.String | undefined,
   };
-
-  if (this[TypeId]) {
-    result[ATTR_TYPE] = this[TypeId].toString();
-  }
-
-  if (this[SelfDXNId]) {
-    result[ATTR_SELF_DXN] = this[SelfDXNId].toString();
-  }
-
-  if (this[RelationSourceDXNId]) {
-    const sourceDXN = this[RelationSourceDXNId];
-    invariant(sourceDXN instanceof DXN);
-    result[ATTR_RELATION_SOURCE] = sourceDXN.toString();
-  }
-  if (this[RelationTargetDXNId]) {
-    const targetDXN = this[RelationTargetDXNId];
-    invariant(targetDXN instanceof DXN);
-    result[ATTR_RELATION_TARGET] = targetDXN.toString();
-  }
-
-  if (this[MetaId]) {
-    result[ATTR_META] = serializeMeta(this[MetaId]);
-  }
-
-  Object.assign(result, serializeData(rest));
-  return result;
-};
-
-const serializeData = (data: unknown) => {
-  return deepMapValues(data, (value, recurse) => {
-    if (Ref.isRef(value)) {
-      // TODO(dmaretskyi): Should this be configurable?
-      return value.noInline().encode();
-    }
-    return recurse(value);
-  });
-};
-
-const serializeMeta = (meta: ObjectMeta) => {
-  return deepMapValues(meta, (value, recurse) => recurse(value));
 };

package/src/internal/{object → Obj}/schema-validator.test.ts

@@ -27,9 +27,7 @@ describe('schema-validator', () => {
       const TestSchema: Schema.Schema.AnyNoContext = Schema.Struct({
         name: Schema.String.annotations({ [annotationId]: annotationValue }),
         parent: Schema.optional(Schema.suspend(() => TestSchema.annotations({ [annotationId]: annotationValue }))),
-        friends: Schema.suspend(() =>
-          Schema.mutable(Schema.Array(TestSchema.annotations({ [annotationId]: annotationValue }))),
-        ),
+        friends: Schema.suspend(() => Schema.Array(TestSchema.annotations({ [annotationId]: annotationValue }))),
       });
       expect(SchemaValidator.hasTypeAnnotation(TestSchema, 'name', annotationId)).to.be.true;
       expect(SchemaValidator.hasTypeAnnotation(TestSchema, 'parent', annotationId)).to.be.true;
@@ -41,7 +39,7 @@ describe('schema-validator', () => {
       const Person: Schema.Schema.AnyNoContext = Schema.Struct({
         name: Schema.String,
         parent: Schema.optional(Schema.suspend(() => Person)),
-        friends: Schema.suspend(() => Schema.mutable(Schema.Array(Person))),
+        friends: Schema.suspend(() => Schema.Array(Person)),
       });
       expect(SchemaValidator.hasTypeAnnotation(Person, 'name', annotationId)).to.be.false;
       expect(SchemaValidator.hasTypeAnnotation(Person, 'parent', annotationId)).to.be.false;
@@ -87,9 +85,7 @@ describe('schema-validator', () => {
       const annotationValue = 'bar';
       const Person: Schema.Schema.AnyNoContext = Schema.Struct({
         parent: Schema.optional(Schema.suspend(() => Person.annotations({ [annotationId]: annotationValue }))),
-        friends: Schema.suspend(() =>
-          Schema.mutable(Schema.Array(Person.annotations({ [annotationId]: annotationValue }))),
-        ),
+        friends: Schema.suspend(() => Schema.Array(Person.annotations({ [annotationId]: annotationValue }))),
       });
       expect(SchemaValidator.getPropertySchema(Person, ['parent']).ast.annotations[annotationId]).to.eq(
         annotationValue,

package/src/internal/Obj/schema-validator.ts

@@ -0,0 +1,6 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+// Re-export from proxy for backward compatibility.
+export { SchemaValidator, checkIdNotPresentOnSchema } from '../common/proxy';