@dxos/echo 0.8.4-main.c4373fc → 0.8.4-main.c85a9c8dae

package/src/internal/object/json-serializer.test.ts

@@ -6,26 +6,27 @@ import { describe, expect, test } from 'vitest';
 
 import { DXN } from '@dxos/keys';
 
-import { EntityKind, getSchemaDXN, getSchemaTypename } from '../ast';
+import * as Obj from '../../Obj';
+import { TestSchema } from '../../testing';
+import { getSchemaDXN, getSchemaTypename, getTypeDXN, getTypename } from '../annotations';
+import { getMetaChecked } from '../api';
+import { RelationSourceId, RelationTargetId, getObjectDXN } from '../entities';
+import { makeObject } from '../proxy';
 import { Ref, StaticRefResolver } from '../ref';
-import { Testing } from '../testing';
+import { ATTR_TYPE, EntityKind, KindId, MetaId, TypeId, getSchema } from '../types';
 
-import { getObjectDXN, getSchema } from './accessors';
-import { create } from './create';
+import { createObject } from './create-object';
 import { objectFromJSON, objectToJSON } from './json-serializer';
-import { getMeta } from './meta';
-import { ATTR_TYPE, EntityKindId, MetaId, RelationSourceId, RelationTargetId, TypeId } from './model';
-import { getType, getTypename } from './typename';
 
 describe('Object JSON serializer', () => {
   test('should serialize and deserialize object', async () => {
-    const contact = create(Testing.Contact, {
-      name: 'John Doe',
+    const contact = makeObject(TestSchema.Person, { name: 'Alice' });
+    Obj.change(contact, (c) => {
+      getMetaChecked(c).keys.push({ id: '12345', source: 'example.com' });
     });
-    getMeta(contact).keys.push({ id: '12345', source: 'crm.example.com' });
 
-    const task = create(Testing.Task, {
-      title: 'Polish my shoes',
+    const task = createObject(TestSchema.Task, {
+      title: 'Fix the tests',
       assignee: Ref.make(contact),
     });
 
@@ -33,67 +34,88 @@ describe('Object JSON serializer', () => {
     const taskJson = objectToJSON(task);
 
     expect(contactJson.id).toBe(contact.id);
-    expect(contactJson[ATTR_TYPE]).toEqual(getSchemaDXN(Testing.Contact)!.toString());
-    expect(contactJson.name).toEqual('John Doe');
+    expect(contactJson[ATTR_TYPE]).toEqual(getSchemaDXN(TestSchema.Person)!.toString());
+    expect(contactJson.name).toEqual('Alice');
 
     expect(taskJson.id).toBe(task.id);
-    expect(taskJson[ATTR_TYPE]).toEqual(getSchemaDXN(Testing.Task)!.toString());
-    expect(taskJson.title).toEqual('Polish my shoes');
+    expect(taskJson[ATTR_TYPE]).toEqual(getSchemaDXN(TestSchema.Task)!.toString());
+    expect(taskJson.title).toEqual('Fix the tests');
     expect(taskJson.assignee).toEqual({ '/': DXN.fromLocalObjectId(contact.id).toString() });
 
     const refResolver = new StaticRefResolver()
-      .addSchema(Testing.Contact)
-      .addSchema(Testing.Task)
+      .addSchema(TestSchema.Person)
+      .addSchema(TestSchema.Task)
       .addObject(contact)
       .addObject(task);
 
-    const contactFromJson = (await objectFromJSON(contactJson, { refResolver })) as Testing.Contact;
-    const taskFromJson = (await objectFromJSON(taskJson, { refResolver })) as Testing.Task;
+    const contactFromJson = (await objectFromJSON(contactJson, { refResolver })) as TestSchema.Person;
+    const taskFromJson = (await objectFromJSON(taskJson, { refResolver })) as TestSchema.Task;
 
     expect(contactFromJson.id).toBe(contact.id);
-    expect(contactFromJson.name).toBe('John Doe');
-    expect((contactFromJson as any)[TypeId]).toEqual(getSchemaDXN(Testing.Contact));
-    expect((contactFromJson as any)[EntityKindId]).toBe(EntityKind.Object);
+    expect(contactFromJson.name).toBe('Alice');
+    expect((contactFromJson as any)[TypeId]).toEqual(getSchemaDXN(TestSchema.Person));
+    expect((contactFromJson as any)[KindId]).toBe(EntityKind.Object);
     expect((contactFromJson as any)[RelationSourceId]).toBeUndefined();
     expect((contactFromJson as any)[RelationTargetId]).toBeUndefined();
     expect((contactFromJson as any)[MetaId]).toEqual({
       keys: [
         {
           id: '12345',
-          source: 'crm.example.com',
+          source: 'example.com',
         },
       ],
     });
-    expect(getType(contactFromJson)?.toString()).toBe(getSchemaDXN(Testing.Contact)!.toString());
-    expect(getTypename(contactFromJson)).toBe(getSchemaTypename(Testing.Contact));
+    expect(getTypeDXN(contactFromJson)?.toString()).toBe(getSchemaDXN(TestSchema.Person)!.toString());
+    expect(getTypename(contactFromJson)).toBe(getSchemaTypename(TestSchema.Person));
     expect(getObjectDXN(contactFromJson)?.toString()).toEqual(getObjectDXN(contact)?.toString());
-    expect(getSchema(contactFromJson)).toEqual(Testing.Contact);
+    expect(getSchema(contactFromJson)).toEqual(TestSchema.Person);
 
     expect(taskFromJson.id).toBe(task.id);
-    expect(taskFromJson.title).toBe('Polish my shoes');
+    expect(taskFromJson.title).toBe('Fix the tests');
     expect(taskFromJson.assignee!.dxn).toEqual(DXN.fromLocalObjectId(contact.id));
     expect(taskFromJson.assignee!.target).toEqual(contact);
     expect(await taskFromJson.assignee!.load()).toEqual(contact);
-    expect((taskFromJson as any)[TypeId]).toEqual(getSchemaDXN(Testing.Task));
-    expect((taskFromJson as any)[EntityKindId]).toBe(EntityKind.Object);
+    expect((taskFromJson as any)[TypeId]).toEqual(getSchemaDXN(TestSchema.Task));
+    expect((taskFromJson as any)[KindId]).toBe(EntityKind.Object);
     expect((taskFromJson as any)[RelationSourceId]).toBeUndefined();
     expect((taskFromJson as any)[RelationTargetId]).toBeUndefined();
     expect((taskFromJson as any)[MetaId]).toEqual({ keys: [] });
-    expect(getSchema(taskFromJson)).toEqual(Testing.Task);
+    expect(getSchema(taskFromJson)).toEqual(TestSchema.Task);
   });
 
   test('serialize with unresolved schema', async () => {
-    const contact = create(Testing.Contact, {
-      name: 'John Doe',
-    });
+    const contact = createObject(TestSchema.Person, { name: 'Alice' });
     const contactJson = objectToJSON(contact);
     const contactFromJson: any = await objectFromJSON(contactJson);
 
     expect(contactFromJson.id).toBe(contact.id);
-    expect(contactFromJson.name).toBe('John Doe');
+    expect(contactFromJson.name).toBe('Alice');
     expect(getSchema(contactFromJson)).toBeUndefined();
-    expect(getTypename(contactFromJson)).toEqual(getSchemaTypename(Testing.Contact));
+    expect(getTypename(contactFromJson)).toEqual(getSchemaTypename(TestSchema.Person));
     expect(getObjectDXN(contactFromJson)).toEqual(getObjectDXN(contact));
-    expect(getType(contactFromJson)).toEqual(getSchemaDXN(Testing.Contact));
+    expect(getTypeDXN(contactFromJson)).toEqual(getSchemaDXN(TestSchema.Person));
+  });
+
+  test('deserializes expando without leaking internal json keys', async () => {
+    const expando = Obj.make(TestSchema.Expando, { message: 'local-only' });
+    const expandoJson = objectToJSON(expando);
+
+    const refResolver = new StaticRefResolver().addSchema(TestSchema.Expando);
+    const expandoFromJson = (await objectFromJSON(expandoJson, { refResolver })) as TestSchema.Expando;
+
+    expect(expandoFromJson.id).toBe(expando.id);
+    expect(expandoFromJson.message).toBe('local-only');
+    expect((expandoFromJson as any)[ATTR_TYPE]).toBeUndefined();
+  });
+
+  test('deserializes expando without schema resolver and without leaking internal json keys', async () => {
+    const expando = Obj.make(TestSchema.Expando, { message: 'local-only' });
+    const expandoJson = objectToJSON(expando);
+
+    const expandoFromJson = (await objectFromJSON(expandoJson)) as TestSchema.Expando;
+
+    expect(expandoFromJson.id).toBe(expando.id);
+    expect(expandoFromJson.message).toBe('local-only');
+    expect((expandoFromJson as any)[ATTR_TYPE]).toBeUndefined();
   });
 });

package/src/internal/object/json-serializer.ts

@@ -5,48 +5,62 @@
 import * as Schema from 'effect/Schema';
 
 import { raise } from '@dxos/debug';
-import { type EncodedReference, type ObjectMeta, isEncodedReference } from '@dxos/echo-protocol';
+import { type EncodedReference, ObjectStructure, isEncodedReference } from '@dxos/echo-protocol';
 import { assertArgument, invariant } from '@dxos/invariant';
 import { DXN, ObjectId } from '@dxos/keys';
-import { defineHiddenProperty } from '@dxos/live-object';
 import { assumeType, deepMapValues, visitValues } from '@dxos/util';
 
-import { EntityKind } from '../ast';
-import { Ref, type RefResolver, refFromEncodedReference, setRefResolver } from '../ref';
-import { type AnyEchoObject } from '../types';
-
-import { setSchema } from './accessors';
-import { ObjectMetaSchema } from './meta';
+import type * as Obj from '../../Obj';
+import { getTypeDXN, setTypename } from '../annotations';
 import {
   ATTR_DELETED,
-  ATTR_META,
   ATTR_RELATION_SOURCE,
   ATTR_RELATION_TARGET,
   ATTR_SELF_DXN,
-  ATTR_TYPE,
-  EntityKindId,
-  MetaId,
   type ObjectJSON,
   RelationSourceDXNId,
   RelationSourceId,
   RelationTargetDXNId,
   RelationTargetId,
   SelfDXNId,
-  TypeId,
-  assertObjectModelShape,
-} from './model';
-import { getType, setTypename } from './typename';
+  assertObjectModel,
+} from '../entities';
+import { attachTypedJsonSerializer, defineHiddenProperty, typedJsonSerializer } from '../proxy';
+import { Ref, type RefResolver, refFromEncodedReference, setRefResolver } from '../ref';
+import {
+  ATTR_META,
+  ATTR_PARENT,
+  ATTR_TYPE,
+  type AnyEntity,
+  EntityKind,
+  KindId,
+  MetaId,
+  ObjectMetaSchema,
+  ParentId,
+  setSchema,
+} from '../types';
 
-type DeepReplaceRef<T> =
-  T extends Ref<any> ? EncodedReference : T extends object ? { [K in keyof T]: DeepReplaceRef<T[K]> } : T;
+// Re-export for backward compatibility.
+export { attachTypedJsonSerializer };
 
-type SerializedObject<T extends { id: string }> = { [K in keyof T]: DeepReplaceRef<T[K]> } & ObjectJSON;
+type DeepReplaceRef<T> =
+  T extends Ref<any>
+    ? EncodedReference
+    : T extends object
+      ? {
+          [K in keyof T]: DeepReplaceRef<T[K]>;
+        }
+      : T;
+
+type SerializedObject<T extends { id: string }> = {
+  [K in keyof T]: DeepReplaceRef<T[K]>;
+} & ObjectJSON;
 
 /**
  * Converts object to it's JSON representation.
  */
-export const objectToJSON = <T extends AnyEchoObject>(obj: T): SerializedObject<T> => {
-  const typename = getType(obj)?.toString();
+export const objectToJSON = <T extends AnyEntity>(obj: T): SerializedObject<T> => {
+  const typename = getTypeDXN(obj)?.toString();
   invariant(typename && typeof typename === 'string');
   return typedJsonSerializer.call(obj);
 };
@@ -55,13 +69,12 @@ export const objectToJSON = <T extends AnyEchoObject>(obj: T): SerializedObject<
  * Creates an object from it's json representation.
  * Performs schema validation.
  * References and schema will be resolvable if the `refResolver` is provided.
- *
  * The function need to be async to support resolving the schema as well as the relation endpoints.
  */
 export const objectFromJSON = async (
   jsonData: unknown,
   { refResolver, dxn }: { refResolver?: RefResolver; dxn?: DXN } = {},
-): Promise<AnyEchoObject> => {
+): Promise<AnyEntity> => {
   assumeType<ObjectJSON>(jsonData);
   assertArgument(typeof jsonData === 'object' && jsonData !== null, 'jsonData', 'expect object');
   assertArgument(typeof jsonData[ATTR_TYPE] === 'string', 'jsonData[ATTR_TYPE]', 'expected object to have a type');
@@ -70,19 +83,19 @@ export const objectFromJSON = async (
   const type = DXN.parse(jsonData[ATTR_TYPE]);
   const schema = await refResolver?.resolveSchema(type);
   invariant(schema === undefined || Schema.isSchema(schema));
+  const decodedInput = stripInternalJsonKeys(jsonData);
 
   let obj: any;
   if (schema != null) {
-    obj = await schema.pipe(Schema.decodeUnknownPromise)(jsonData);
+    obj = await schema.pipe(Schema.decodeUnknownPromise)(decodedInput);
     if (refResolver) {
       setRefResolverOnData(obj, refResolver);
     }
   } else {
-    obj = decodeGeneric(jsonData, { refResolver });
+    obj = decodeGeneric(decodedInput, { refResolver });
   }
 
   invariant(ObjectId.isValid(obj.id), 'Invalid object id');
-
   setTypename(obj, type);
   if (schema) {
     setSchema(obj, schema);
@@ -94,132 +107,98 @@ export const objectFromJSON = async (
     const sourceDxn: DXN = DXN.parse(jsonData[ATTR_RELATION_SOURCE] ?? raise(new TypeError('Missing relation source')));
     const targetDxn: DXN = DXN.parse(jsonData[ATTR_RELATION_TARGET] ?? raise(new TypeError('Missing relation target')));
 
-    // TODO(dmaretskyi): Async!
-    const source = (await refResolver?.resolve(sourceDxn)) as AnyEchoObject | undefined;
-    const target = (await refResolver?.resolve(targetDxn)) as AnyEchoObject | undefined;
+    const source = (await refResolver?.resolve(sourceDxn)) as AnyEntity | undefined;
+    const target = (await refResolver?.resolve(targetDxn)) as AnyEntity | undefined;
 
-    defineHiddenProperty(obj, EntityKindId, EntityKind.Relation);
+    defineHiddenProperty(obj, KindId, EntityKind.Relation);
     defineHiddenProperty(obj, RelationSourceDXNId, sourceDxn);
     defineHiddenProperty(obj, RelationTargetDXNId, targetDxn);
     defineHiddenProperty(obj, RelationSourceId, source);
     defineHiddenProperty(obj, RelationTargetId, target);
   } else {
-    defineHiddenProperty(obj, EntityKindId, EntityKind.Object);
+    defineHiddenProperty(obj, KindId, EntityKind.Object);
   }
 
   if (typeof jsonData[ATTR_META] === 'object') {
     const meta = await ObjectMetaSchema.pipe(Schema.decodeUnknownPromise)(jsonData[ATTR_META]);
-
-    // Defensive programming.
     invariant(Array.isArray(meta.keys));
-
     defineHiddenProperty(obj, MetaId, meta);
+  } else {
+    defineHiddenProperty(obj, MetaId, {
+      keys: [],
+    });
+  }
+
+  if (jsonData[ATTR_PARENT]) {
+    const parentDxn = DXN.parse(jsonData[ATTR_PARENT]);
+    const parent = (await refResolver?.resolve(parentDxn)) as Obj.Unknown | undefined;
+    defineHiddenProperty(obj, ParentId, parent);
   }
 
   if (dxn) {
     defineHiddenProperty(obj, SelfDXNId, dxn);
   }
 
-  assertObjectModelShape(obj);
+  assertObjectModel(obj);
   invariant((obj as any)[ATTR_TYPE] === undefined, 'Invalid object model');
-  invariant((obj as any)[ATTR_SELF_DXN] === undefined, 'Invalid object model');
+  invariant((obj as any)[ATTR_META] === undefined, 'Invalid object model');
   invariant((obj as any)[ATTR_DELETED] === undefined, 'Invalid object model');
+  invariant((obj as any)[ATTR_SELF_DXN] === undefined, 'Invalid object model');
   invariant((obj as any)[ATTR_RELATION_SOURCE] === undefined, 'Invalid object model');
   invariant((obj as any)[ATTR_RELATION_TARGET] === undefined, 'Invalid object model');
-  invariant((obj as any)[ATTR_META] === undefined, 'Invalid object model');
   return obj;
 };
 
 const decodeGeneric = (jsonData: unknown, options: { refResolver?: RefResolver }) => {
+  const props = stripInternalJsonKeys(jsonData);
+
+  return deepMapValues(props, (value, visitor) => {
+    if (isEncodedReference(value)) {
+      return refFromEncodedReference(value, options.refResolver);
+    }
+
+    return visitor(value);
+  });
+};
+
+const stripInternalJsonKeys = (jsonData: unknown) => {
   const {
     [ATTR_TYPE]: _type,
     [ATTR_META]: _meta,
     [ATTR_DELETED]: _deleted,
+    [ATTR_SELF_DXN]: _selfDxn,
     [ATTR_RELATION_SOURCE]: _relationSource,
     [ATTR_RELATION_TARGET]: _relationTarget,
-    [ATTR_SELF_DXN]: _selfDxn,
     ...props
   } = jsonData as any;
 
-  return deepMapValues(props, (value, recurse) => {
-    if (isEncodedReference(value)) {
-      return refFromEncodedReference(value, options.refResolver);
-    }
-    return recurse(value);
-  });
+  return props;
 };
 
-export const setRefResolverOnData = (obj: AnyEchoObject, refResolver: RefResolver) => {
-  const go = (value: unknown) => {
+export const setRefResolverOnData = (obj: AnyEntity, refResolver: RefResolver) => {
+  const visitor = (value: unknown) => {
     if (Ref.isRef(value)) {
       setRefResolver(value, refResolver);
     } else {
-      visitValues(value, go);
+      visitValues(value, visitor);
     }
   };
 
-  go(obj);
+  visitor(obj);
 };
 
-export const attachTypedJsonSerializer = (obj: any) => {
-  const descriptor = Object.getOwnPropertyDescriptor(obj, 'toJSON');
-  if (descriptor) {
-    return;
-  }
-
-  Object.defineProperty(obj, 'toJSON', {
-    value: typedJsonSerializer,
-    writable: false,
-    enumerable: false,
-    // Setting `configurable` to false breaks proxy invariants, should be fixable.
-    configurable: true,
-  });
-};
-
-// NOTE: KEEP as function.
-const typedJsonSerializer = function (this: any) {
-  const { id, ...rest } = this;
-  const result: any = {
-    id,
+/**
+ * Convert ObjectStructure to JSON data for indexing.
+ * Different from {@link objectToJSON} as it takes the internal {@link ObjectStructure} representation directly
+ */
+export const objectStructureToJson = (objectId: string, structure: ObjectStructure): Obj.JSON => {
+  return {
+    ...structure.data,
+    id: objectId,
+    [ATTR_TYPE]: (ObjectStructure.getTypeReference(structure)?.['/'] ?? '') as DXN.String,
+    [ATTR_DELETED]: ObjectStructure.isDeleted(structure),
+    [ATTR_PARENT]: ObjectStructure.getParent(structure)?.['/'] as DXN.String | undefined,
+    [ATTR_RELATION_SOURCE]: ObjectStructure.getRelationSource(structure)?.['/'] as DXN.String | undefined,
+    [ATTR_RELATION_TARGET]: ObjectStructure.getRelationTarget(structure)?.['/'] as DXN.String | undefined,
   };
-
-  if (this[TypeId]) {
-    result[ATTR_TYPE] = this[TypeId].toString();
-  }
-
-  if (this[SelfDXNId]) {
-    result[ATTR_SELF_DXN] = this[SelfDXNId].toString();
-  }
-
-  if (this[RelationSourceDXNId]) {
-    const sourceDXN = this[RelationSourceDXNId];
-    invariant(sourceDXN instanceof DXN);
-    result[ATTR_RELATION_SOURCE] = sourceDXN.toString();
-  }
-  if (this[RelationTargetDXNId]) {
-    const targetDXN = this[RelationTargetDXNId];
-    invariant(targetDXN instanceof DXN);
-    result[ATTR_RELATION_TARGET] = targetDXN.toString();
-  }
-
-  if (this[MetaId]) {
-    result[ATTR_META] = serializeMeta(this[MetaId]);
-  }
-
-  Object.assign(result, serializeData(rest));
-  return result;
-};
-
-const serializeData = (data: unknown) => {
-  return deepMapValues(data, (value, recurse) => {
-    if (Ref.isRef(value)) {
-      // TODO(dmaretskyi): Should this be configurable?
-      return value.noInline().encode();
-    }
-    return recurse(value);
-  });
-};
-
-const serializeMeta = (meta: ObjectMeta) => {
-  return deepMapValues(meta, (value, recurse) => recurse(value));
 };

package/src/internal/object/schema-validator.test.ts

@@ -27,9 +27,7 @@ describe('schema-validator', () => {
       const TestSchema: Schema.Schema.AnyNoContext = Schema.Struct({
         name: Schema.String.annotations({ [annotationId]: annotationValue }),
         parent: Schema.optional(Schema.suspend(() => TestSchema.annotations({ [annotationId]: annotationValue }))),
-        friends: Schema.suspend(() =>
-          Schema.mutable(Schema.Array(TestSchema.annotations({ [annotationId]: annotationValue }))),
-        ),
+        friends: Schema.suspend(() => Schema.Array(TestSchema.annotations({ [annotationId]: annotationValue }))),
       });
       expect(SchemaValidator.hasTypeAnnotation(TestSchema, 'name', annotationId)).to.be.true;
       expect(SchemaValidator.hasTypeAnnotation(TestSchema, 'parent', annotationId)).to.be.true;
@@ -41,7 +39,7 @@ describe('schema-validator', () => {
       const Person: Schema.Schema.AnyNoContext = Schema.Struct({
         name: Schema.String,
         parent: Schema.optional(Schema.suspend(() => Person)),
-        friends: Schema.suspend(() => Schema.mutable(Schema.Array(Person))),
+        friends: Schema.suspend(() => Schema.Array(Person)),
       });
       expect(SchemaValidator.hasTypeAnnotation(Person, 'name', annotationId)).to.be.false;
       expect(SchemaValidator.hasTypeAnnotation(Person, 'parent', annotationId)).to.be.false;
@@ -87,9 +85,7 @@ describe('schema-validator', () => {
       const annotationValue = 'bar';
       const Person: Schema.Schema.AnyNoContext = Schema.Struct({
         parent: Schema.optional(Schema.suspend(() => Person.annotations({ [annotationId]: annotationValue }))),
-        friends: Schema.suspend(() =>
-          Schema.mutable(Schema.Array(Person.annotations({ [annotationId]: annotationValue }))),
-        ),
+        friends: Schema.suspend(() => Schema.Array(Person.annotations({ [annotationId]: annotationValue }))),
       });
       expect(SchemaValidator.getPropertySchema(Person, ['parent']).ast.annotations[annotationId]).to.eq(
         annotationValue,