@dxos/echo 0.8.4-main.fd6878d → 0.8.4-staging.60fe92afc8

package/src/internal/Obj/inspect.ts

@@ -0,0 +1,48 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import type { InspectOptionsStylized, inspect as inspectFn } from 'node:util';
+
+import { type CustomInspectFunction, inspectCustom } from '@dxos/debug';
+
+import { getTypeURI } from '../Annotation';
+import { ATTR_TYPE, type AnyEntity } from '../common/types';
+import { ATTR_META } from '../common/types/meta';
+import { MetaId } from '../common/types/model-symbols';
+
+/*
+ * @internal
+ */
+export const attachedTypedObjectInspector = (obj: any) => {
+  const descriptor = Object.getOwnPropertyDescriptor(obj, inspectCustom);
+  if (descriptor) {
+    return;
+  }
+
+  Object.defineProperty(obj, inspectCustom, {
+    value: typedObjectInspectFunction,
+    writable: false,
+    enumerable: false,
+    configurable: true,
+  });
+};
+
+// NOTE: KEEP as function.
+const typedObjectInspectFunction: CustomInspectFunction<AnyEntity> = function (
+  this: AnyEntity,
+  depth: number,
+  options: InspectOptionsStylized,
+  inspect: typeof inspectFn,
+) {
+  const { id, ...props } = this;
+  return inspect(
+    {
+      id,
+      [ATTR_TYPE]: getTypeURI(this),
+      ...props,
+      [ATTR_META]: (this as any)[MetaId], // TODO(dmaretskyi): Couldn't use getMeta since that throw's if the object has no meta.
+    },
+    options,
+  );
+};

package/src/internal/Obj/json-serializer.test.ts

@@ -0,0 +1,199 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Schema from 'effect/Schema';
+import { describe, expect, test } from 'vitest';
+
+import { DXN, EID } from '@dxos/keys';
+
+import * as Obj from '../../Obj';
+import { TestSchema } from '../../testing';
+import * as Type from '../../Type';
+import { getTypeURI, getTypename } from '../Annotation';
+import { getMetaChecked } from '../common/api';
+import { ATTR_TYPE, EntityKind, KindId, TypeId, getSchema } from '../common/types';
+import { MetaId } from '../common/types/model-symbols';
+import { RelationSourceId, RelationTargetId, getObjectEchoUri } from '../Entity';
+import * as JsonSchema from '../JsonSchema';
+import { Ref, StaticRefResolver } from '../Ref';
+import { createObject } from './create-object';
+import { objectFromJSON, objectToJSON } from './json-serializer';
+
+describe('Object JSON serializer', () => {
+  test('should serialize and deserialize object', async () => {
+    const contact = Obj.make(TestSchema.Person, { name: 'Alice' });
+    Obj.update(contact, (contact) => {
+      getMetaChecked(contact).keys.push({ id: '12345', source: 'example.com' });
+    });
+
+    const task = createObject(TestSchema.Task, {
+      title: 'Fix the tests',
+      assignee: Ref.make(contact),
+    });
+
+    const contactJson = objectToJSON(contact);
+    const taskJson = objectToJSON(task);
+
+    expect(contactJson.id).toBe(contact.id);
+    expect(contactJson[ATTR_TYPE]).toEqual(Type.getURI(TestSchema.Person).toString());
+    expect(contactJson.name).toEqual('Alice');
+
+    expect(taskJson.id).toBe(task.id);
+    expect(taskJson[ATTR_TYPE]).toEqual(Type.getURI(TestSchema.Task).toString());
+    expect(taskJson.title).toEqual('Fix the tests');
+    expect(taskJson.assignee).toEqual({ '/': EID.make({ entityId: contact.id }) });
+
+    const refResolver = new StaticRefResolver()
+      .addSchema(TestSchema.Person)
+      .addSchema(TestSchema.Task)
+      .addObject(contact)
+      .addObject(task);
+
+    const contactFromJson = (await objectFromJSON(contactJson, { refResolver })) as TestSchema.Person;
+    const taskFromJson = (await objectFromJSON(taskJson, { refResolver })) as TestSchema.Task;
+
+    expect(contactFromJson.id).toBe(contact.id);
+    expect(contactFromJson.name).toBe('Alice');
+    expect((contactFromJson as any)[TypeId]).toEqual(Type.getURI(TestSchema.Person));
+    expect((contactFromJson as any)[KindId]).toBe(EntityKind.Object);
+    expect((contactFromJson as any)[RelationSourceId]).toBeUndefined();
+    expect((contactFromJson as any)[RelationTargetId]).toBeUndefined();
+    expect((contactFromJson as any)[MetaId]).toEqual({
+      keys: [
+        {
+          id: '12345',
+          source: 'example.com',
+        },
+      ],
+      tags: [],
+      annotations: {},
+    });
+    expect(getTypeURI(contactFromJson)?.toString()).toBe(Type.getURI(TestSchema.Person).toString());
+    expect(getTypename(contactFromJson)).toBe(Type.getTypename(TestSchema.Person));
+    expect(getObjectEchoUri(contactFromJson)?.toString()).toEqual(getObjectEchoUri(contact)?.toString());
+    expect(getSchema(contactFromJson)).toEqual(Type.getSchema(TestSchema.Person));
+
+    expect(taskFromJson.id).toBe(task.id);
+    expect(taskFromJson.title).toBe('Fix the tests');
+    expect(taskFromJson.assignee!.uri).toEqual(EID.make({ entityId: contact.id }));
+    expect(taskFromJson.assignee!.target).toEqual(contact);
+    expect(await taskFromJson.assignee!.load()).toEqual(contact);
+    expect((taskFromJson as any)[TypeId]).toEqual(Type.getURI(TestSchema.Task));
+    expect((taskFromJson as any)[KindId]).toBe(EntityKind.Object);
+    expect((taskFromJson as any)[RelationSourceId]).toBeUndefined();
+    expect((taskFromJson as any)[RelationTargetId]).toBeUndefined();
+    expect((taskFromJson as any)[MetaId]).toEqual({ keys: [], tags: [], annotations: {} });
+    expect(getSchema(taskFromJson)).toEqual(Type.getSchema(TestSchema.Task));
+  });
+
+  test('serialize with unresolved schema', async () => {
+    const contact = createObject(TestSchema.Person, { name: 'Alice' });
+    const contactJson = objectToJSON(contact);
+    const contactFromJson: any = await objectFromJSON(contactJson);
+
+    expect(contactFromJson.id).toBe(contact.id);
+    expect(contactFromJson.name).toBe('Alice');
+    expect(getSchema(contactFromJson)).toBeUndefined();
+    expect(getTypename(contactFromJson)).toEqual(Type.getTypename(TestSchema.Person));
+    expect(getObjectEchoUri(contactFromJson)).toEqual(getObjectEchoUri(contact));
+    expect(getTypeURI(contactFromJson)).toEqual(Type.getURI(TestSchema.Person));
+  });
+
+  test('upgrades bare string tags to encoded references on deserialize', async () => {
+    const expando = Obj.make(TestSchema.Expando, { message: 'hi' });
+    const json = objectToJSON(expando) as any;
+    // Simulate data serialized before the tags-as-refs migration: bare id strings.
+    json['@meta'] = { keys: [], tags: ['echo:/TAGLEGACY'] };
+
+    const fromJson = (await objectFromJSON(json)) as any;
+    // Decodes to a materialized `Ref` (the shared ref codec), not a raw encoded reference.
+    expect(fromJson[MetaId].tags.map((ref: any) => ref.uri)).toEqual(['echo:/TAGLEGACY']);
+  });
+
+  test('deserializes expando without leaking internal json keys', async () => {
+    const expando = Obj.make(TestSchema.Expando, { message: 'local-only' });
+    const expandoJson = objectToJSON(expando);
+
+    const refResolver = new StaticRefResolver().addSchema(TestSchema.Expando);
+    const expandoFromJson = (await objectFromJSON(expandoJson, { refResolver })) as TestSchema.Expando;
+
+    expect(expandoFromJson.id).toBe(expando.id);
+    expect(expandoFromJson.message).toBe('local-only');
+    expect((expandoFromJson as any)[ATTR_TYPE]).toBeUndefined();
+  });
+
+  test('deserializes expando without schema resolver and without leaking internal json keys', async () => {
+    const expando = Obj.make(TestSchema.Expando, { message: 'local-only' });
+    const expandoJson = objectToJSON(expando);
+
+    const expandoFromJson = (await objectFromJSON(expandoJson)) as TestSchema.Expando;
+
+    expect(expandoFromJson.id).toBe(expando.id);
+    expect(expandoFromJson.message).toBe('local-only');
+    expect((expandoFromJson as any)[ATTR_TYPE]).toBeUndefined();
+  });
+
+  // `objectFromJSON` is the deserialization path for queue messages and devtools
+  // round-trips. For persisted `Type.Type` entities it must stamp `KindId = Type`
+  // (not Object), mirroring the kind resolution that `createObject` does for the
+  // in-memory path. Otherwise `Filter.type(Type.Type)` / `Type.isType` skip them.
+  describe('Type.Type round-trip', () => {
+    test('preserves KindId=Type for a persisted Type.Type entity', async ({ expect }) => {
+      const typeEntity = Type.makeObjectFromJsonSchema({
+        typename: 'com.example.type.regression',
+        version: '0.1.0',
+        jsonSchema: JsonSchema.toJsonSchema(Schema.Struct({ field: Schema.Number })),
+        name: 'Regression Type',
+      });
+
+      const typeJson = objectToJSON(typeEntity as any);
+      const refResolver = new StaticRefResolver().addSchema(Type.Type);
+      const reconstructed = (await objectFromJSON(typeJson, { refResolver })) as any;
+
+      expect(reconstructed[KindId]).toBe(EntityKind.Type);
+      expect(Type.isType(reconstructed)).toBe(true);
+      // `typename` lives in `EntityMeta.key` on persisted Type.Type entities
+      // — surfaced via `Type.getTypename`.
+      expect(Type.getTypename(reconstructed)).toBe('com.example.type.regression');
+    });
+  });
+
+  describe('Uint8Array', () => {
+    const Blob = Schema.Struct({
+      name: Schema.String,
+      bytes: Schema.Uint8ArrayFromSelf,
+    }).pipe(Type.makeObject(DXN.make('com.example.type.blob', '0.1.0')));
+    type Blob = Type.InstanceType<typeof Blob>;
+
+    test('round-trips Uint8Array field through JSON with schema', async ({ expect }) => {
+      const bytes = new Uint8Array([0, 1, 2, 3, 250, 251, 252, 253, 254, 255]);
+      const blob = Obj.make(Blob, { name: 'blob', bytes });
+
+      const blobJson = objectToJSON(blob);
+      // JSON must round-trip through stringify/parse without loss.
+      const roundTripped = JSON.parse(JSON.stringify(blobJson));
+
+      const refResolver = new StaticRefResolver().addSchema(Blob);
+      const blobFromJson = (await objectFromJSON(roundTripped, { refResolver })) as Blob;
+
+      expect(blobFromJson.name).toBe('blob');
+      expect(blobFromJson.bytes).toBeInstanceOf(Uint8Array);
+      expect(Array.from(blobFromJson.bytes)).toEqual(Array.from(bytes));
+    });
+
+    test('round-trips Uint8Array field through JSON without schema resolver', async ({ expect }) => {
+      const bytes = new Uint8Array([10, 20, 30, 40, 50]);
+      const blob = Obj.make(Blob, { name: 'blob', bytes });
+
+      const blobJson = objectToJSON(blob);
+      const roundTripped = JSON.parse(JSON.stringify(blobJson));
+
+      const blobFromJson = (await objectFromJSON(roundTripped)) as Blob;
+
+      expect(blobFromJson.name).toBe('blob');
+      expect(blobFromJson.bytes).toBeInstanceOf(Uint8Array);
+      expect(Array.from(blobFromJson.bytes)).toEqual(Array.from(bytes));
+    });
+  });
+});

package/src/internal/Obj/json-serializer.ts

@@ -0,0 +1,278 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Schema from 'effect/Schema';
+
+import { raise } from '@dxos/debug';
+import { type EncodedReference, EntityStructure, isEncodedReference } from '@dxos/echo-protocol';
+import { assertArgument, invariant } from '@dxos/invariant';
+import { EID, EntityId, URI } from '@dxos/keys';
+import { assumeType, decodeUint8ArrayFromJson, deepMapValues, isEncodedUint8Array, visitValues } from '@dxos/util';
+
+import type * as Database from '../../Database';
+import type * as Obj from '../../Obj';
+import { getTypeAnnotation, getTypeURI, setTypename } from '../Annotation';
+import { attachTypedJsonSerializer, defineHiddenProperty, typedJsonSerializer } from '../common/proxy';
+import {
+  ATTR_PARENT,
+  ATTR_TYPE,
+  type AnyEntity,
+  EntityKind,
+  KindId,
+  ParentId,
+  setSchema,
+  setType,
+} from '../common/types';
+import { ATTR_META, EntityMetaSchema } from '../common/types/meta';
+import { MetaId } from '../common/types/model-symbols';
+import {
+  ATTR_DELETED,
+  ATTR_RELATION_SOURCE,
+  ATTR_RELATION_TARGET,
+  ATTR_SELF_URI,
+  ATTR_SELF_URI_LEGACY,
+  ObjectDatabaseId,
+  type ObjectJSON,
+  RelationSourceDXNId,
+  RelationSourceId,
+  RelationTargetDXNId,
+  RelationTargetId,
+  SelfURIId,
+  assertObjectModel,
+} from '../Entity';
+import { Ref, type RefResolver, refFromEncodedReference, setRefResolver } from '../Ref';
+
+// Re-export for backward compatibility.
+export { attachTypedJsonSerializer };
+
+type DeepReplaceRef<T> =
+  T extends Ref<any>
+    ? EncodedReference
+    : T extends object
+      ? {
+          [K in keyof T]: DeepReplaceRef<T[K]>;
+        }
+      : T;
+
+type SerializedObject<T extends { id: string }> = {
+  [K in keyof T]: DeepReplaceRef<T[K]>;
+} & ObjectJSON;
+
+/**
+ * Converts object to it's JSON representation.
+ */
+export const objectToJSON = <T extends AnyEntity>(obj: T): SerializedObject<T> => {
+  const typename = getTypeURI(obj);
+  invariant(typename && typeof typename === 'string');
+  return typedJsonSerializer.call(obj);
+};
+
+/**
+ * Creates an object from it's json representation.
+ * Performs schema validation.
+ * References and schema will be resolvable if the `refResolver` is provided.
+ * The function need to be async to support resolving the schema as well as the relation endpoints.
+ *
+ * @param jsonData - JSON representation of the object.
+ * @param options.refResolver - Resolver for references.
+ * @param options.uri - Override object URI.
+ * @param options.database - Database to associate with the object.
+ */
+export const objectFromJSON = async (
+  jsonData: unknown,
+  {
+    refResolver,
+    uri,
+    database,
+    parent,
+  }: { refResolver?: RefResolver; uri?: URI.URI; database?: Database.Database; parent?: Obj.Unknown } = {},
+): Promise<AnyEntity> => {
+  assumeType<ObjectJSON>(jsonData);
+  assertArgument(typeof jsonData === 'object' && jsonData !== null, 'jsonData', 'expect object');
+  assertArgument(typeof jsonData[ATTR_TYPE] === 'string', 'jsonData[ATTR_TYPE]', 'expected object to have a type');
+  assertArgument(typeof jsonData.id === 'string', 'jsonData.id', 'expected object to have an id');
+
+  const type = URI.make(jsonData[ATTR_TYPE]);
+  const schema = await refResolver?.resolveSchema(type);
+  invariant(schema === undefined || Schema.isSchema(schema));
+  const decodedInput = restoreUint8Arrays(stripInternalJsonKeys(jsonData));
+
+  let obj: any;
+  if (schema != null) {
+    obj = await schema.pipe(Schema.decodeUnknownPromise)(decodedInput);
+    if (refResolver) {
+      setRefResolverOnData(obj, refResolver);
+    }
+  } else {
+    obj = decodeGeneric(decodedInput, { refResolver });
+  }
+
+  invariant(EntityId.isValid(obj.id), 'Invalid object id');
+  setTypename(obj, type);
+  if (schema) {
+    setSchema(obj, schema);
+  }
+  // Resolve and stamp the source type entity, if the resolver provides one.
+  // Lets `Obj.getType` / `Entity.getType` return a stable entity for objects
+  // loaded via `Obj.fromJSON` (serializer / queue paths).
+  if (refResolver?.resolveType) {
+    const typeEntity = await refResolver.resolveType(type);
+    if (typeEntity != null) {
+      setType(obj, typeEntity);
+    }
+  }
+
+  const isRelation =
+    typeof jsonData[ATTR_RELATION_SOURCE] === 'string' || typeof jsonData[ATTR_RELATION_TARGET] === 'string';
+  if (isRelation) {
+    const sourceDxn = jsonData[ATTR_RELATION_SOURCE] ?? raise(new TypeError('Missing relation source'));
+    const targetDxn = jsonData[ATTR_RELATION_TARGET] ?? raise(new TypeError('Missing relation target'));
+
+    const source = (await refResolver?.resolve(sourceDxn)) as AnyEntity | undefined;
+    const target = (await refResolver?.resolve(targetDxn)) as AnyEntity | undefined;
+
+    defineHiddenProperty(obj, KindId, EntityKind.Relation);
+    defineHiddenProperty(obj, RelationSourceDXNId, sourceDxn);
+    defineHiddenProperty(obj, RelationTargetDXNId, targetDxn);
+    defineHiddenProperty(obj, RelationSourceId, source);
+    defineHiddenProperty(obj, RelationTargetId, target);
+  } else {
+    // Honour the schema's TypeAnnotation kind — persisted `Type.Type` entities
+    // (e.g. dynamic schemas loaded from a snapshot import) must brand as
+    // `KindId = Type`, not Object, otherwise `Filter.type(Type.Type)` /
+    // `Type.isType` skip them and the schema registry never picks them up.
+    // Mirrors the kind resolution in `createObject` (the in-memory path).
+    const annotationKind = schema != null ? getTypeAnnotation(schema)?.kind : undefined;
+    defineHiddenProperty(obj, KindId, annotationKind === EntityKind.Type ? EntityKind.Type : EntityKind.Object);
+  }
+
+  if (typeof jsonData[ATTR_META] === 'object') {
+    const meta = await EntityMetaSchema.pipe(Schema.decodeUnknownPromise)(normalizeMeta(jsonData[ATTR_META]));
+    invariant(Array.isArray(meta.keys));
+    defineHiddenProperty(obj, MetaId, meta);
+  } else {
+    defineHiddenProperty(obj, MetaId, {
+      keys: [],
+      tags: [],
+      annotations: {},
+    });
+  }
+
+  if (jsonData[ATTR_PARENT]) {
+    const parentDxn = jsonData[ATTR_PARENT];
+    const resolvedParent = (await refResolver?.resolve(parentDxn)) as Obj.Unknown | undefined;
+    defineHiddenProperty(obj, ParentId, resolvedParent);
+  } else if (parent) {
+    defineHiddenProperty(obj, ParentId, parent);
+  }
+
+  if (uri) {
+    defineHiddenProperty(obj, SelfURIId, uri);
+  }
+
+  if (database) {
+    defineHiddenProperty(obj, ObjectDatabaseId, database);
+  }
+
+  assertObjectModel(obj);
+  invariant((obj as any)[ATTR_TYPE] === undefined, 'Invalid object model');
+  invariant((obj as any)[ATTR_META] === undefined, 'Invalid object model');
+  invariant((obj as any)[ATTR_DELETED] === undefined, 'Invalid object model');
+  invariant((obj as any)[ATTR_SELF_URI] === undefined, 'Invalid object model');
+  invariant((obj as any)[ATTR_SELF_URI_LEGACY] === undefined, 'Invalid object model');
+  invariant((obj as any)[ATTR_RELATION_SOURCE] === undefined, 'Invalid object model');
+  invariant((obj as any)[ATTR_RELATION_TARGET] === undefined, 'Invalid object model');
+  return obj;
+};
+
+/**
+ * Backfills required meta fields and upgrades legacy `tags` (bare URI strings) to encoded references
+ * so serialized data produced before the tags-as-refs migration still decodes.
+ */
+const normalizeMeta = (meta: any): any => {
+  const tags = Array.isArray(meta?.tags)
+    ? meta.tags.map((tag: unknown) => (typeof tag === 'string' ? { '/': URI.make(tag) } : tag))
+    : [];
+  // Coalesce required fields so explicit `undefined` in legacy input doesn't override the defaults.
+  return {
+    ...meta,
+    keys: Array.isArray(meta?.keys) ? meta.keys : [],
+    tags,
+    annotations: meta?.annotations ?? {},
+  };
+};
+
+const decodeGeneric = (jsonData: unknown, options: { refResolver?: RefResolver }) => {
+  const props = stripInternalJsonKeys(jsonData);
+
+  return deepMapValues(props, (value, visitor) => {
+    if (isEncodedReference(value)) {
+      return refFromEncodedReference(value, options.refResolver);
+    }
+    if (isEncodedUint8Array(value)) {
+      return decodeUint8ArrayFromJson(value);
+    }
+
+    return visitor(value);
+  });
+};
+
+/**
+ * Recursively replaces encoded `Uint8Array` JSON markers with actual `Uint8Array` instances.
+ * Runs before schema decoding so `Schema.Uint8ArrayFromSelf` sees real bytes.
+ */
+const restoreUint8Arrays = (data: unknown): any =>
+  deepMapValues(data, (value, recurse) => {
+    if (isEncodedUint8Array(value)) {
+      return decodeUint8ArrayFromJson(value);
+    }
+    return recurse(value);
+  });
+
+const stripInternalJsonKeys = (jsonData: unknown) => {
+  const {
+    [ATTR_TYPE]: _type,
+    [ATTR_META]: _meta,
+    [ATTR_DELETED]: _deleted,
+    [ATTR_SELF_URI]: _selfUri,
+    [ATTR_SELF_URI_LEGACY]: _legacySelfUri,
+    [ATTR_RELATION_SOURCE]: _relationSource,
+    [ATTR_RELATION_TARGET]: _relationTarget,
+    ...props
+  } = jsonData as any;
+
+  return props;
+};
+
+export const setRefResolverOnData = (obj: AnyEntity, refResolver: RefResolver) => {
+  const visitor = (value: unknown) => {
+    if (Ref.isRef(value)) {
+      setRefResolver(value, refResolver);
+    } else {
+      visitValues(value, visitor);
+    }
+  };
+
+  visitor(obj);
+};
+
+/**
+ * Convert EntityStructure to JSON data for indexing.
+ * Different from {@link objectToJSON} as it takes the internal {@link EntityStructure} representation directly
+ */
+export const objectStructureToJson = (objectId: EntityId, structure: EntityStructure): Obj.JSON => {
+  const typeRef = EntityStructure.getTypeReference(structure)?.['/'];
+  const parent = EntityStructure.getParent(structure)?.['/'];
+  const source = EntityStructure.getRelationSource(structure)?.['/'];
+  const target = EntityStructure.getRelationTarget(structure)?.['/'];
+  return {
+    ...structure.data,
+    id: objectId,
+    [ATTR_TYPE]: typeRef ? URI.make(typeRef) : undefined,
+    [ATTR_DELETED]: EntityStructure.isDeleted(structure),
+    [ATTR_PARENT]: parent !== undefined ? EID.tryParse(parent) : undefined,
+    [ATTR_RELATION_SOURCE]: source !== undefined ? EID.tryParse(source) : undefined,
+    [ATTR_RELATION_TARGET]: target !== undefined ? EID.tryParse(target) : undefined,
+  };
+};