@dxos/echo 0.8.4-main.70d3990 → 0.8.4-main.74a063c4e0

package/src/internal/Obj/clone.ts

@@ -0,0 +1,48 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import { invariant } from '@dxos/invariant';
+import { deepMapValues } from '@dxos/util';
+
+import type * as Obj from '../../Obj';
+import { makeObject } from '../common/proxy';
+import { getMeta, getSchema } from '../common/types';
+import { Ref } from '../Ref';
+
+/**
+ * Clones an object or relation.
+ * This does not clone referenced objects, only the properties in the object.
+ * @returns A new object with the same schema and properties.
+ */
+export const clone = <T extends Obj.Any>(obj: T, opts?: Obj.CloneOptions): T => {
+  const { id, ...data } = obj;
+  const schema = getSchema(obj);
+  invariant(schema != null, 'Object should have a schema');
+  const props: any = deepMapValues(data, (value, recurse) => {
+    if (Ref.isRef(value)) {
+      if (opts?.deep) {
+        // TODO(dmaretskyi): Will break on circular references.
+        return Ref.make(clone(value.target!, opts));
+      }
+      return value;
+    }
+    return recurse(value);
+  });
+
+  if (opts?.retainId) {
+    props.id = id;
+  }
+  const meta = deepMapValues(getMeta(obj), (value, recurse) => {
+    if (Ref.isRef(value)) {
+      if (opts?.deep) {
+        // TODO(dmaretskyi): Will break on circular references.
+        return Ref.make(clone(value.target!, opts));
+      }
+      return value;
+    }
+    return recurse(value);
+  });
+
+  return makeObject(schema, props, meta);
+};

package/src/internal/{object → Obj}/common.ts

@@ -4,8 +4,8 @@
 
 import * as Schema from 'effect/Schema';
 
-import { getTypename } from '../annotations';
-import { type AnyProperties } from '../types';
+import { getTypename } from '../Annotation';
+import { type AnyProperties } from '../common/types';
 
 // TODO(dmaretskyi): Rename to represent commonality between objects and relations (e.g. `entity`).
 export type TypedObjectOptions = {

package/src/internal/{object → Obj}/create-object.test.ts

@@ -2,19 +2,17 @@
 // Copyright 2025 DXOS.org
 //
 
-import { inspect } from 'util';
-
 import * as Schema from 'effect/Schema';
+import { inspect } from 'util';
 import { describe, expect, test } from 'vitest';
 
 import { DXN } from '@dxos/keys';
 
 import { Relation } from '../../index';
 import { TestSchema } from '../../testing';
-import { getSchemaDXN, getTypeDXN, isInstanceOf } from '../annotations';
-import { ATTR_RELATION_SOURCE, ATTR_RELATION_TARGET } from '../entities';
-import { ATTR_META, ATTR_TYPE, getSchema } from '../types';
-
+import { getSchemaDXN, getTypeDXN, isInstanceOf } from '../Annotation';
+import { ATTR_META, ATTR_TYPE, getSchema } from '../common/types';
+import { ATTR_RELATION_SOURCE, ATTR_RELATION_TARGET } from '../Entity';
 import { createObject } from './create-object';
 import { objectToJSON } from './json-serializer';
 
@@ -110,7 +108,7 @@ describe('create (static version)', () => {
     const text = inspect(contact);
     expect(text).toContain('Bot');
     expect(text).toContain('bot@example.com');
-    expect(text).toContain('example.com/type/Person');
+    expect(text).toContain('com.example.type.person');
     expect(text).toContain('0.1.0');
   });
 });

package/src/internal/{object → Obj}/create-object.ts

@@ -7,9 +7,10 @@ import type * as Schema from 'effect/Schema';
 import { raise } from '@dxos/debug';
 import { assertArgument, failedInvariant } from '@dxos/invariant';
 import { ObjectId } from '@dxos/keys';
-import { defineHiddenProperty } from '@dxos/live-object';
 
-import { getSchemaDXN, getTypeAnnotation, setTypename } from '../annotations';
+import { getSchemaDXN, getTypeAnnotation, setTypename } from '../Annotation';
+import { defineHiddenProperty } from '../common/proxy';
+import { EntityKind, KindId, MetaId, setSchema } from '../common/types';
 import {
   RelationSourceDXNId,
   RelationSourceId,
@@ -17,9 +18,7 @@ import {
   RelationTargetId,
   assertObjectModel,
   getObjectDXN,
-} from '../entities';
-import { EntityKind, KindId, MetaId, setSchema } from '../types';
-
+} from '../Entity';
 import { attachedTypedObjectInspector } from './inspect';
 import { attachTypedJsonSerializer } from './json-serializer';
 
@@ -41,8 +40,8 @@ export type CreateObjectProps<T> = T extends { id: string } ? Omit<T, 'id' | Kin
  * const Contact = Schema.Struct({
  *   name: Schema.String,
  *   email: Schema.String,
- * }).pipe(Type.Obj({
- *   typename: 'example.com/type/Person',
+ * }).pipe(Type.object({
+ *   typename: 'com.example.type.person',
  *   version: '0.1.0',
  * }))
  *

package/src/internal/{object → Obj}/deleted.ts

@@ -2,8 +2,8 @@
 // Copyright 2025 DXOS.org
 //
 
-import { ObjectDeletedId } from '../entities';
-import { type AnyProperties } from '../types';
+import { type AnyProperties } from '../common/types';
+import { ObjectDeletedId } from '../Entity';
 
 /**
  * @returns `true` if the object has been marked as deleted.

package/src/internal/{object → Obj}/index.ts

@@ -8,4 +8,7 @@ export * from './deleted';
 export * from './ids';
 export * from './json-serializer';
 export * from './schema-validator';
+export * from './set-value';
+export * from './snapshot';
 export * from './typed-object';
+export * from './clone';

package/src/internal/{object → Obj}/inspect.ts

@@ -6,8 +6,8 @@ import type { InspectOptionsStylized, inspect as inspectFn } from 'node:util';
 
 import { type CustomInspectFunction, inspectCustom } from '@dxos/debug';
 
-import { getTypeDXN } from '../annotations';
-import { ATTR_META, ATTR_TYPE, type AnyEchoObject, MetaId } from '../types';
+import { getTypeDXN } from '../Annotation';
+import { ATTR_META, ATTR_TYPE, type AnyEntity, MetaId } from '../common/types';
 
 /*
  * @internal
@@ -27,8 +27,8 @@ export const attachedTypedObjectInspector = (obj: any) => {
 };
 
 // NOTE: KEEP as function.
-const typedObjectInspectFunction: CustomInspectFunction<AnyEchoObject> = function (
-  this: AnyEchoObject,
+const typedObjectInspectFunction: CustomInspectFunction<AnyEntity> = function (
+  this: AnyEntity,
   depth: number,
   options: InspectOptionsStylized,
   inspect: typeof inspectFn,

package/src/internal/{object → Obj}/json-serializer.test.ts

@@ -6,20 +6,23 @@ import { describe, expect, test } from 'vitest';
 
 import { DXN } from '@dxos/keys';
 
+import * as Obj from '../../Obj';
 import { TestSchema } from '../../testing';
-import { getSchemaDXN, getSchemaTypename, getTypeDXN, getTypename } from '../annotations';
-import { RelationSourceId, RelationTargetId, getObjectDXN } from '../entities';
-import { makeObject } from '../proxy';
-import { Ref, StaticRefResolver } from '../ref';
-import { ATTR_TYPE, EntityKind, KindId, MetaId, TypeId, getMeta, getSchema } from '../types';
-
+import { getSchemaDXN, getSchemaTypename, getTypeDXN, getTypename } from '../Annotation';
+import { getMetaChecked } from '../common/api';
+import { makeObject } from '../common/proxy';
+import { ATTR_TYPE, EntityKind, KindId, MetaId, TypeId, getSchema } from '../common/types';
+import { RelationSourceId, RelationTargetId, getObjectDXN } from '../Entity';
+import { Ref, StaticRefResolver } from '../Ref';
 import { createObject } from './create-object';
 import { objectFromJSON, objectToJSON } from './json-serializer';
 
 describe('Object JSON serializer', () => {
   test('should serialize and deserialize object', async () => {
     const contact = makeObject(TestSchema.Person, { name: 'Alice' });
-    getMeta(contact).keys.push({ id: '12345', source: 'example.com' });
+    Obj.change(contact, (obj) => {
+      getMetaChecked(obj).keys.push({ id: '12345', source: 'example.com' });
+    });
 
     const task = createObject(TestSchema.Task, {
       title: 'Fix the tests',
@@ -91,4 +94,27 @@ describe('Object JSON serializer', () => {
     expect(getObjectDXN(contactFromJson)).toEqual(getObjectDXN(contact));
     expect(getTypeDXN(contactFromJson)).toEqual(getSchemaDXN(TestSchema.Person));
   });
+
+  test('deserializes expando without leaking internal json keys', async () => {
+    const expando = Obj.make(TestSchema.Expando, { message: 'local-only' });
+    const expandoJson = objectToJSON(expando);
+
+    const refResolver = new StaticRefResolver().addSchema(TestSchema.Expando);
+    const expandoFromJson = (await objectFromJSON(expandoJson, { refResolver })) as TestSchema.Expando;
+
+    expect(expandoFromJson.id).toBe(expando.id);
+    expect(expandoFromJson.message).toBe('local-only');
+    expect((expandoFromJson as any)[ATTR_TYPE]).toBeUndefined();
+  });
+
+  test('deserializes expando without schema resolver and without leaking internal json keys', async () => {
+    const expando = Obj.make(TestSchema.Expando, { message: 'local-only' });
+    const expandoJson = objectToJSON(expando);
+
+    const expandoFromJson = (await objectFromJSON(expandoJson)) as TestSchema.Expando;
+
+    expect(expandoFromJson.id).toBe(expando.id);
+    expect(expandoFromJson.message).toBe('local-only');
+    expect((expandoFromJson as any)[ATTR_TYPE]).toBeUndefined();
+  });
 });

package/src/internal/{object → Obj}/json-serializer.ts

@@ -5,18 +5,33 @@
 import * as Schema from 'effect/Schema';
 
 import { raise } from '@dxos/debug';
-import { type EncodedReference, type ObjectMeta, isEncodedReference } from '@dxos/echo-protocol';
+import { type EncodedReference, ObjectStructure, isEncodedReference } from '@dxos/echo-protocol';
 import { assertArgument, invariant } from '@dxos/invariant';
 import { DXN, ObjectId } from '@dxos/keys';
-import { defineHiddenProperty } from '@dxos/live-object';
 import { assumeType, deepMapValues, visitValues } from '@dxos/util';
 
-import { getTypeDXN, setTypename } from '../annotations';
+import type * as Database from '../../Database';
+import type * as Obj from '../../Obj';
+import { getTypeDXN, setTypename } from '../Annotation';
+import { attachTypedJsonSerializer, defineHiddenProperty, typedJsonSerializer } from '../common/proxy';
+import {
+  ATTR_META,
+  ATTR_PARENT,
+  ATTR_TYPE,
+  type AnyEntity,
+  EntityKind,
+  KindId,
+  MetaId,
+  ObjectMetaSchema,
+  ParentId,
+  setSchema,
+} from '../common/types';
 import {
   ATTR_DELETED,
   ATTR_RELATION_SOURCE,
   ATTR_RELATION_TARGET,
   ATTR_SELF_DXN,
+  ObjectDatabaseId,
   type ObjectJSON,
   RelationSourceDXNId,
   RelationSourceId,
@@ -24,19 +39,11 @@ import {
   RelationTargetId,
   SelfDXNId,
   assertObjectModel,
-} from '../entities';
-import { Ref, type RefResolver, refFromEncodedReference, setRefResolver } from '../ref';
-import {
-  ATTR_META,
-  ATTR_TYPE,
-  type AnyEchoObject,
-  EntityKind,
-  KindId,
-  MetaId,
-  ObjectMetaSchema,
-  TypeId,
-  setSchema,
-} from '../types';
+} from '../Entity';
+import { Ref, type RefResolver, refFromEncodedReference, setRefResolver } from '../Ref';
+
+// Re-export for backward compatibility.
+export { attachTypedJsonSerializer };
 
 type DeepReplaceRef<T> =
   T extends Ref<any>
@@ -54,7 +61,7 @@ type SerializedObject<T extends { id: string }> = {
 /**
  * Converts object to it's JSON representation.
  */
-export const objectToJSON = <T extends AnyEchoObject>(obj: T): SerializedObject<T> => {
+export const objectToJSON = <T extends AnyEntity>(obj: T): SerializedObject<T> => {
   const typename = getTypeDXN(obj)?.toString();
   invariant(typename && typeof typename === 'string');
   return typedJsonSerializer.call(obj);
@@ -65,11 +72,16 @@ export const objectToJSON = <T extends AnyEchoObject>(obj: T): SerializedObject<
  * Performs schema validation.
  * References and schema will be resolvable if the `refResolver` is provided.
  * The function need to be async to support resolving the schema as well as the relation endpoints.
+ *
+ * @param jsonData - JSON representation of the object.
+ * @param options.refResolver - Resolver for references.
+ * @param options.dxn - Override object DXN.
+ * @param options.database - Database to associate with the object.
  */
 export const objectFromJSON = async (
   jsonData: unknown,
-  { refResolver, dxn }: { refResolver?: RefResolver; dxn?: DXN } = {},
-): Promise<AnyEchoObject> => {
+  { refResolver, dxn, database }: { refResolver?: RefResolver; dxn?: DXN; database?: Database.Database } = {},
+): Promise<AnyEntity> => {
   assumeType<ObjectJSON>(jsonData);
   assertArgument(typeof jsonData === 'object' && jsonData !== null, 'jsonData', 'expect object');
   assertArgument(typeof jsonData[ATTR_TYPE] === 'string', 'jsonData[ATTR_TYPE]', 'expected object to have a type');
@@ -78,15 +90,16 @@ export const objectFromJSON = async (
   const type = DXN.parse(jsonData[ATTR_TYPE]);
   const schema = await refResolver?.resolveSchema(type);
   invariant(schema === undefined || Schema.isSchema(schema));
+  const decodedInput = stripInternalJsonKeys(jsonData);
 
   let obj: any;
   if (schema != null) {
-    obj = await schema.pipe(Schema.decodeUnknownPromise)(jsonData);
+    obj = await schema.pipe(Schema.decodeUnknownPromise)(decodedInput);
     if (refResolver) {
       setRefResolverOnData(obj, refResolver);
     }
   } else {
-    obj = decodeGeneric(jsonData, { refResolver });
+    obj = decodeGeneric(decodedInput, { refResolver });
   }
 
   invariant(ObjectId.isValid(obj.id), 'Invalid object id');
@@ -101,8 +114,8 @@ export const objectFromJSON = async (
     const sourceDxn: DXN = DXN.parse(jsonData[ATTR_RELATION_SOURCE] ?? raise(new TypeError('Missing relation source')));
     const targetDxn: DXN = DXN.parse(jsonData[ATTR_RELATION_TARGET] ?? raise(new TypeError('Missing relation target')));
 
-    const source = (await refResolver?.resolve(sourceDxn)) as AnyEchoObject | undefined;
-    const target = (await refResolver?.resolve(targetDxn)) as AnyEchoObject | undefined;
+    const source = (await refResolver?.resolve(sourceDxn)) as AnyEntity | undefined;
+    const target = (await refResolver?.resolve(targetDxn)) as AnyEntity | undefined;
 
     defineHiddenProperty(obj, KindId, EntityKind.Relation);
     defineHiddenProperty(obj, RelationSourceDXNId, sourceDxn);
@@ -117,12 +130,26 @@ export const objectFromJSON = async (
     const meta = await ObjectMetaSchema.pipe(Schema.decodeUnknownPromise)(jsonData[ATTR_META]);
     invariant(Array.isArray(meta.keys));
     defineHiddenProperty(obj, MetaId, meta);
+  } else {
+    defineHiddenProperty(obj, MetaId, {
+      keys: [],
+    });
+  }
+
+  if (jsonData[ATTR_PARENT]) {
+    const parentDxn = DXN.parse(jsonData[ATTR_PARENT]);
+    const parent = (await refResolver?.resolve(parentDxn)) as Obj.Unknown | undefined;
+    defineHiddenProperty(obj, ParentId, parent);
   }
 
   if (dxn) {
     defineHiddenProperty(obj, SelfDXNId, dxn);
   }
 
+  if (database) {
+    defineHiddenProperty(obj, ObjectDatabaseId, database);
+  }
+
   assertObjectModel(obj);
   invariant((obj as any)[ATTR_TYPE] === undefined, 'Invalid object model');
   invariant((obj as any)[ATTR_META] === undefined, 'Invalid object model');
@@ -134,6 +161,18 @@ export const objectFromJSON = async (
 };
 
 const decodeGeneric = (jsonData: unknown, options: { refResolver?: RefResolver }) => {
+  const props = stripInternalJsonKeys(jsonData);
+
+  return deepMapValues(props, (value, visitor) => {
+    if (isEncodedReference(value)) {
+      return refFromEncodedReference(value, options.refResolver);
+    }
+
+    return visitor(value);
+  });
+};
+
+const stripInternalJsonKeys = (jsonData: unknown) => {
   const {
     [ATTR_TYPE]: _type,
     [ATTR_META]: _meta,
@@ -144,16 +183,10 @@ const decodeGeneric = (jsonData: unknown, options: { refResolver?: RefResolver }
     ...props
   } = jsonData as any;
 
-  return deepMapValues(props, (value, visitor) => {
-    if (isEncodedReference(value)) {
-      return refFromEncodedReference(value, options.refResolver);
-    }
-
-    return visitor(value);
-  });
+  return props;
 };
 
-export const setRefResolverOnData = (obj: AnyEchoObject, refResolver: RefResolver) => {
+export const setRefResolverOnData = (obj: AnyEntity, refResolver: RefResolver) => {
   const visitor = (value: unknown) => {
     if (Ref.isRef(value)) {
       setRefResolver(value, refResolver);
@@ -165,66 +198,18 @@ export const setRefResolverOnData = (obj: AnyEchoObject, refResolver: RefResolve
   visitor(obj);
 };
 
-export const attachTypedJsonSerializer = (obj: any) => {
-  const descriptor = Object.getOwnPropertyDescriptor(obj, 'toJSON');
-  if (descriptor) {
-    return;
-  }
-
-  Object.defineProperty(obj, 'toJSON', {
-    value: typedJsonSerializer,
-    writable: false,
-    enumerable: false,
-    // Setting `configurable` to false breaks proxy invariants, should be fixable.
-    configurable: true,
-  });
-};
-
-// NOTE: KEEP as function.
-const typedJsonSerializer = function (this: any) {
-  const { id, ...rest } = this;
-  const result: any = {
-    id,
+/**
+ * Convert ObjectStructure to JSON data for indexing.
+ * Different from {@link objectToJSON} as it takes the internal {@link ObjectStructure} representation directly
+ */
+export const objectStructureToJson = (objectId: string, structure: ObjectStructure): Obj.JSON => {
+  return {
+    ...structure.data,
+    id: objectId,
+    [ATTR_TYPE]: (ObjectStructure.getTypeReference(structure)?.['/'] ?? '') as DXN.String,
+    [ATTR_DELETED]: ObjectStructure.isDeleted(structure),
+    [ATTR_PARENT]: ObjectStructure.getParent(structure)?.['/'] as DXN.String | undefined,
+    [ATTR_RELATION_SOURCE]: ObjectStructure.getRelationSource(structure)?.['/'] as DXN.String | undefined,
+    [ATTR_RELATION_TARGET]: ObjectStructure.getRelationTarget(structure)?.['/'] as DXN.String | undefined,
   };
-
-  if (this[TypeId]) {
-    result[ATTR_TYPE] = this[TypeId].toString();
-  }
-
-  if (this[MetaId]) {
-    result[ATTR_META] = serializeMeta(this[MetaId]);
-  }
-
-  if (this[SelfDXNId]) {
-    result[ATTR_SELF_DXN] = this[SelfDXNId].toString();
-  }
-
-  if (this[RelationSourceDXNId]) {
-    const sourceDXN = this[RelationSourceDXNId];
-    invariant(sourceDXN instanceof DXN);
-    result[ATTR_RELATION_SOURCE] = sourceDXN.toString();
-  }
-  if (this[RelationTargetDXNId]) {
-    const targetDXN = this[RelationTargetDXNId];
-    invariant(targetDXN instanceof DXN);
-    result[ATTR_RELATION_TARGET] = targetDXN.toString();
-  }
-
-  Object.assign(result, serializeData(rest));
-  return result;
-};
-
-const serializeData = (data: unknown) => {
-  return deepMapValues(data, (value, recurse) => {
-    if (Ref.isRef(value)) {
-      // TODO(dmaretskyi): Should this be configurable?
-      return value.noInline().encode();
-    }
-
-    return recurse(value);
-  });
-};
-
-const serializeMeta = (meta: ObjectMeta) => {
-  return deepMapValues(meta, (value, recurse) => recurse(value));
 };

package/src/internal/{object → Obj}/schema-validator.test.ts

@@ -27,9 +27,7 @@ describe('schema-validator', () => {
       const TestSchema: Schema.Schema.AnyNoContext = Schema.Struct({
         name: Schema.String.annotations({ [annotationId]: annotationValue }),
         parent: Schema.optional(Schema.suspend(() => TestSchema.annotations({ [annotationId]: annotationValue }))),
-        friends: Schema.suspend(() =>
-          Schema.mutable(Schema.Array(TestSchema.annotations({ [annotationId]: annotationValue }))),
-        ),
+        friends: Schema.suspend(() => Schema.Array(TestSchema.annotations({ [annotationId]: annotationValue }))),
       });
       expect(SchemaValidator.hasTypeAnnotation(TestSchema, 'name', annotationId)).to.be.true;
       expect(SchemaValidator.hasTypeAnnotation(TestSchema, 'parent', annotationId)).to.be.true;
@@ -41,7 +39,7 @@ describe('schema-validator', () => {
       const Person: Schema.Schema.AnyNoContext = Schema.Struct({
         name: Schema.String,
         parent: Schema.optional(Schema.suspend(() => Person)),
-        friends: Schema.suspend(() => Schema.mutable(Schema.Array(Person))),
+        friends: Schema.suspend(() => Schema.Array(Person)),
       });
       expect(SchemaValidator.hasTypeAnnotation(Person, 'name', annotationId)).to.be.false;
       expect(SchemaValidator.hasTypeAnnotation(Person, 'parent', annotationId)).to.be.false;
@@ -87,9 +85,7 @@ describe('schema-validator', () => {
       const annotationValue = 'bar';
       const Person: Schema.Schema.AnyNoContext = Schema.Struct({
         parent: Schema.optional(Schema.suspend(() => Person.annotations({ [annotationId]: annotationValue }))),
-        friends: Schema.suspend(() =>
-          Schema.mutable(Schema.Array(Person.annotations({ [annotationId]: annotationValue }))),
-        ),
+        friends: Schema.suspend(() => Schema.Array(Person.annotations({ [annotationId]: annotationValue }))),
       });
       expect(SchemaValidator.getPropertySchema(Person, ['parent']).ast.annotations[annotationId]).to.eq(
         annotationValue,

package/src/internal/Obj/schema-validator.ts

@@ -0,0 +1,6 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+// Re-export from proxy for backward compatibility.
+export { SchemaValidator, checkIdNotPresentOnSchema } from '../common/proxy';