@dxos/echo 0.8.4-main.ae835ea → 0.8.4-main.bc674ce

package/src/internal/schema/echo-schema.ts

@@ -8,10 +8,11 @@ import * as SchemaAST from 'effect/SchemaAST';
 import { invariant } from '@dxos/invariant';
 import { type ObjectId } from '@dxos/keys';
 
-import { type SchemaMeta, SchemaMetaSymbol, type TypeAnnotation, getTypeAnnotation } from '../ast';
-import { toEffectSchema, toJsonSchema } from '../json';
-import { type JsonSchemaType } from '../json-schema';
-import { type TypedObject, type TypedObjectPrototype } from '../object';
+import { type SchemaMeta, SchemaMetaSymbol, type TypeAnnotation, getTypeAnnotation } from '../annotations';
+import { type JsonSchemaType, toEffectSchema, toJsonSchema } from '../json-schema';
+import { type TypedObject, type TypedObjectPrototype, getSnapshot } from '../object';
+import { ChangeId } from '../proxy';
+import { EntityKind, SchemaKindId } from '../types';
 
 import {
   addFieldsToSchema,
@@ -20,14 +21,14 @@ import {
   updateFieldNameInSchema,
   updateFieldsInSchema,
 } from './manipulation';
-import { getSnapshot } from './snapshot';
-import { StoredSchema } from './stored-schema';
+import { PersistentSchema } from './persistent-schema';
 
 /**
  * Base schema type.
  */
 // TODO(burdon): Merge with ImmutableSchema.
 export interface BaseSchema<A = any, I = any> extends TypedObject<A, I> {
+  // TODO(burdon): Different from mutable?
   get readonly(): boolean;
   // TODO(burdon): Change to external function.
   get mutable(): EchoSchema<A, I>;
@@ -83,6 +84,10 @@ export class ImmutableSchema<A = any, I = any> implements BaseSchema<A, I> {
   // TypedObject
   //
 
+  get kind(): EntityKind {
+    return this._objectAnnotation.kind;
+  }
+
   get typename(): string {
     return this._objectAnnotation.typename;
   }
@@ -124,8 +129,10 @@ const EchoSchemaConstructor = (): TypedObjectPrototype => {
    */
   return class {
     private static get _schema() {
-      // The field is DynamicEchoSchema in runtime, but is serialized as StoredEchoSchema in automerge.
-      return Schema.Union(StoredSchema, Schema.instanceOf(EchoSchema)).annotations(StoredSchema.ast.annotations);
+      // The field is DynamicEchoSchema in runtime, but is serialized as PersisentSchema in automerge.
+      return Schema.Union(PersistentSchema, Schema.instanceOf(EchoSchema)).annotations(
+        PersistentSchema.ast.annotations,
+      );
     }
 
     static readonly [Schema.TypeId] = schemaVariance;
@@ -159,7 +166,7 @@ const schemaVariance = {
 };
 
 /**
- * Represents a schema that is stored in the ECHO database.
+ * Represents a schema that is persisted in the ECHO database.
  * Schema can me mutable or readonly (specified by the {@link EchoSchema.readonly} field).
  *
  * Schema that can be modified at runtime via the API.
@@ -170,20 +177,26 @@ const schemaVariance = {
  *
  * @example
  * ```ts
- * export class TableType extends Schema.Struct({...}).pipe(Type.Obj({ typename: 'example.org/type/Table', version: '0.1.0' })){
+ * export class TableType extends Schema.Struct({...}).pipe(Type.object({ typename: 'example.org/type/Table', version: '0.1.0' })){
  *   title: Schema.String,
  *   schema: Schema.optional(ref(EchoSchema)),
  *   props: Schema.mutable(S.Array(TablePropSchema)),
  * }) {}
  * ```
  *
- * The ECHO API will translate any references to StoredSchema objects to be resolved as EchoSchema objects.
+ * The ECHO API will translate any references to PersistentSchema objects to be resolved as EchoSchema objects.
  */
 export class EchoSchema<A = any, I = any> extends EchoSchemaConstructor() implements BaseSchema<A, I> {
   private _schema: Schema.Schema.AnyNoContext | undefined;
   private _isDirty = true;
 
-  constructor(private readonly _storedSchema: StoredSchema) {
+  /**
+   * Schema kind key that marks this as an ECHO schema.
+   * Makes EchoSchema satisfy the Type.Obj.Any type.
+   */
+  readonly [SchemaKindId]: EntityKind.Object = EntityKind.Object;
+
+  constructor(private readonly _persistentSchema: PersistentSchema) {
     super();
   }
 
@@ -196,11 +209,11 @@ export class EchoSchema<A = any, I = any> extends EchoSchemaConstructor() implem
   }
 
   public get Type() {
-    return this._storedSchema as A;
+    return this._persistentSchema as A;
   }
 
   public get Encoded() {
-    return this._storedSchema as I;
+    return this._persistentSchema as I;
   }
 
   public get Context() {
@@ -227,12 +240,17 @@ export class EchoSchema<A = any, I = any> extends EchoSchemaConstructor() implem
   // BaseSchema
   //
 
+  public get kind(): EntityKind.Object {
+    // Dynamic schemas are always object schemas.
+    return EntityKind.Object;
+  }
+
   public get typename(): string {
-    return this._storedSchema.typename;
+    return this._persistentSchema.typename;
   }
 
   public get version(): string {
-    return this._storedSchema.version;
+    return this._persistentSchema.version;
   }
 
   public get readonly(): boolean {
@@ -247,10 +265,10 @@ export class EchoSchema<A = any, I = any> extends EchoSchemaConstructor() implem
   }
 
   /**
-   * @reactive
+   * Returns the JSON schema for the schema.
    */
   public get jsonSchema(): JsonSchemaType {
-    return this._storedSchema.jsonSchema;
+    return this._persistentSchema.jsonSchema;
   }
 
   /**
@@ -269,25 +287,25 @@ export class EchoSchema<A = any, I = any> extends EchoSchemaConstructor() implem
    * Id of the ECHO object containing the schema.
    */
   public get id(): ObjectId {
-    return this._storedSchema.id;
+    return this._persistentSchema.id;
   }
 
   /**
    * Short name of the schema.
    */
   public get name(): string | undefined {
-    return this._storedSchema.name;
+    return this._persistentSchema.name;
   }
 
   public get [SchemaMetaSymbol](): SchemaMeta {
-    return { id: this.id, typename: this.typename, version: this._storedSchema.version };
+    return { id: this.id, typename: this.typename, version: this._persistentSchema.version };
   }
 
   /**
-   * Reference to the underlying stored schema object.
+   * Reference to the underlying persistent schema object.
    */
-  public get storedSchema(): StoredSchema {
-    return this._storedSchema;
+  public get persistentSchema(): PersistentSchema {
+    return this._persistentSchema;
   }
 
   public getProperties(): SchemaAST.PropertySignature[] {
@@ -307,8 +325,10 @@ export class EchoSchema<A = any, I = any> extends EchoSchemaConstructor() implem
    */
   public updateTypename(typename: string): void {
     const updated = setTypenameInSchema(this._getSchema(), typename);
-    this._storedSchema.typename = typename;
-    this._storedSchema.jsonSchema = toJsonSchema(updated);
+    this._change((schema) => {
+      schema.typename = typename;
+      schema.jsonSchema = toJsonSchema(updated);
+    });
   }
 
   /**
@@ -316,7 +336,9 @@ export class EchoSchema<A = any, I = any> extends EchoSchemaConstructor() implem
    */
   public addFields(fields: Schema.Struct.Fields): void {
     const extended = addFieldsToSchema(this._getSchema(), fields);
-    this._storedSchema.jsonSchema = toJsonSchema(extended);
+    this._change((schema) => {
+      schema.jsonSchema = toJsonSchema(extended);
+    });
   }
 
   /**
@@ -324,7 +346,9 @@ export class EchoSchema<A = any, I = any> extends EchoSchemaConstructor() implem
    */
   public updateFields(fields: Schema.Struct.Fields): void {
     const updated = updateFieldsInSchema(this._getSchema(), fields);
-    this._storedSchema.jsonSchema = toJsonSchema(updated);
+    this._change((schema) => {
+      schema.jsonSchema = toJsonSchema(updated);
+    });
   }
 
   /**
@@ -332,7 +356,9 @@ export class EchoSchema<A = any, I = any> extends EchoSchemaConstructor() implem
    */
   public updateFieldPropertyName({ before, after }: { before: PropertyKey; after: PropertyKey }): void {
     const renamed = updateFieldNameInSchema(this._getSchema(), { before, after });
-    this._storedSchema.jsonSchema = toJsonSchema(renamed);
+    this._change((schema) => {
+      schema.jsonSchema = toJsonSchema(renamed);
+    });
   }
 
   /**
@@ -340,7 +366,21 @@ export class EchoSchema<A = any, I = any> extends EchoSchemaConstructor() implem
    */
   public removeFields(fieldNames: string[]): void {
     const removed = removeFieldsFromSchema(this._getSchema(), fieldNames);
-    this._storedSchema.jsonSchema = toJsonSchema(removed);
+    this._change((schema) => {
+      schema.jsonSchema = toJsonSchema(removed);
+    });
+  }
+
+  /**
+   * Wrapper for Obj.change that handles the change context for the persistent schema.
+   */
+  private _change(callback: (schema: PersistentSchema) => void): void {
+    const changeFn = (this._persistentSchema as any)[ChangeId];
+    if (changeFn) {
+      changeFn(callback);
+    } else {
+      callback(this._persistentSchema);
+    }
   }
 
   //
@@ -348,7 +388,7 @@ export class EchoSchema<A = any, I = any> extends EchoSchemaConstructor() implem
   //
 
   /**
-   * Called by EchoSchemaRegistry on update.
+   * Called by DatabaseSchemaRegistry on update.
    */
   _invalidate(): void {
     this._isDirty = true;
@@ -359,12 +399,12 @@ export class EchoSchema<A = any, I = any> extends EchoSchemaConstructor() implem
    */
   _rebuild(): void {
     if (this._isDirty || this._schema == null) {
-      this._schema = toEffectSchema(getSnapshot(this._storedSchema.jsonSchema));
+      this._schema = toEffectSchema(getSnapshot(this._persistentSchema.jsonSchema));
       this._isDirty = false;
     }
   }
 
-  private _getSchema(): Schema.Schema.AnyNoContext {
+  _getSchema(): Schema.Schema.AnyNoContext {
     this._rebuild();
     return this._schema!;
   }

package/src/internal/schema/index.ts

@@ -3,8 +3,5 @@
 //
 
 export * from './echo-schema';
-export * from './snapshot';
-export * from './stored-schema';
-export * from './runtime-schema-registry';
 export * from './manipulation';
-export * from './echo-schema';
+export * from './persistent-schema';

package/src/internal/schema/manipulation.ts

@@ -8,7 +8,7 @@ import * as SchemaAST from 'effect/SchemaAST';
 import { invariant } from '@dxos/invariant';
 import { DXN } from '@dxos/keys';
 
-import { type TypeAnnotation, TypeAnnotationId, TypeIdentifierAnnotationId } from '../ast';
+import { type TypeAnnotation, TypeAnnotationId, TypeIdentifierAnnotationId } from '../annotations';
 
 // TODO(ZaymonFC): Do this one at a time. This might be dangerous.
 export const addFieldsToSchema = (

package/src/internal/schema/persistent-schema.ts

@@ -0,0 +1,27 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import * as Schema from 'effect/Schema';
+
+import { TypenameSchema, VersionSchema } from '../annotations';
+import { EchoObjectSchema } from '../entities';
+import { JsonSchemaType } from '../json-schema';
+
+/**
+ * Persistent representation of a schema.
+ */
+// TODO(burdon): Move.
+export const PersistentSchema = Schema.Struct({
+  name: Schema.optional(Schema.String),
+  typename: TypenameSchema,
+  version: VersionSchema,
+  jsonSchema: JsonSchemaType,
+}).pipe(
+  EchoObjectSchema({
+    typename: 'dxos.org/type/Schema',
+    version: '0.1.0',
+  }),
+);
+
+export interface PersistentSchema extends Schema.Schema.Type<typeof PersistentSchema> {}

package/src/internal/types/base.ts

@@ -0,0 +1,43 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import * as Schema from 'effect/Schema';
+import * as SchemaAST from 'effect/SchemaAST';
+
+import { type ObjectId } from '@dxos/keys';
+
+import { type ATTR_META, type ObjectMeta } from './meta';
+
+/**
+ * Base type for all data objects (reactive, ECHO, and other raw objects).
+ * NOTE: This describes the base type for all database objects.
+ * It is stricter than `T extends {}` or `T extends object`.
+ */
+// TODO(burdon): Make internal.
+// TODO(burdon): Prefer Record<string, unknown>.
+export type AnyProperties = Record<string, any>;
+
+/**
+ * Canonical type for all ECHO entities (objects and relations).
+ */
+// TODO(wittjosiah): Remove. Prefer higher level types (e.g. Entity.Unknown).
+export interface AnyEntity {
+  readonly id: ObjectId;
+}
+
+export type ExcludeId<T extends AnyProperties> = Omit<T, 'id'>;
+
+export type PropertyKey<T extends AnyProperties> = Extract<keyof ExcludeId<T>, string>;
+
+// TODO(dmaretskyi): Remove. This should be using the symbol type.
+type WithMeta = { [ATTR_META]?: ObjectMeta };
+
+/**
+ * The raw object should not include the ECHO id, but may include metadata.
+ */
+export const RawObject = <S extends Schema.Schema.AnyNoContext>(
+  schema: S,
+): Schema.Schema<ExcludeId<Schema.Schema.Type<S>> & WithMeta, Schema.Schema.Encoded<S>> => {
+  return Schema.make(SchemaAST.omit(schema.ast, ['id']));
+};

package/src/internal/types/entity.ts

@@ -0,0 +1,54 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Schema from 'effect/Schema';
+
+// NOTE: String literals are used instead of unique symbols for both KindId and SchemaKindId.
+//   Unique symbols cause TS4023 "cannot be named" errors when external packages
+//   try to export types that reference this key (e.g., `export const Graph = ...`).
+//   TypeScript cannot emit declaration files that reference unique symbols from
+//   external modules. Using a string literal allows the type to be inlined in
+//   declaration files, making the API portable across package boundaries.
+
+/**
+ * String key used to identify the kind of an entity instance (object or relation).
+ */
+export const KindId = '~@dxos/echo/Kind' as const;
+export type KindId = typeof KindId;
+
+/**
+ * String key used to identify the kind of a schema (object schema or relation schema).
+ * Parallels KindId which identifies instance kinds.
+ */
+export const SchemaKindId = '~@dxos/echo/SchemaKind' as const;
+export type SchemaKindId = typeof SchemaKindId;
+
+/**
+ * String key used to brand snapshot types.
+ * Snapshots have SnapshotKindId instead of KindId, making them
+ * distinguishable from reactive objects at the type level.
+ */
+export const SnapshotKindId = '~@dxos/echo/SnapshotKind' as const;
+export type SnapshotKindId = typeof SnapshotKindId;
+
+/**
+ * Kinds of entities stored in ECHO: objects and relations.
+ */
+export enum EntityKind {
+  Object = 'object',
+  Relation = 'relation',
+}
+
+export const EntityKindSchema = Schema.Enums(EntityKind);
+
+/**
+ * Typename for generic object references (Type.Obj / Type.Ref(Type.Obj)).
+ * Used when referencing any object without a specific schema.
+ */
+export const ANY_OBJECT_TYPENAME = 'dxos.org/schema/AnyObject';
+
+/**
+ * Version for generic object references.
+ */
+export const ANY_OBJECT_VERSION = '0.0.0';

package/src/internal/types/index.ts

@@ -2,5 +2,8 @@
 // Copyright 2024 DXOS.org
 //
 
-export * from './types';
-export type * from './util';
+export * from './base';
+export * from './entity';
+export * from './meta';
+export * from './typename';
+export * from './version';

package/src/internal/types/meta.ts

@@ -0,0 +1,65 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import * as Schema from 'effect/Schema';
+
+import { ForeignKey } from '@dxos/echo-protocol';
+import { invariant } from '@dxos/invariant';
+import { type Comparator, intersection } from '@dxos/util';
+
+import { type AnyProperties } from './base';
+
+/**
+ * Property name for meta when object is serialized to JSON.
+ */
+export const ATTR_META = '@meta';
+
+/**
+ * Metadata section.
+ */
+export const MetaId = Symbol.for('@dxos/echo/Meta');
+
+//
+// ObjectMeta
+//
+
+// TODO(dmaretskyi): Rename to ObjectMeta
+export const ObjectMetaSchema = Schema.Struct({
+  keys: Schema.Array(ForeignKey),
+
+  /**
+   * A set of tags.
+   * Tags are arbitrary application-defined strings.
+   * ECHO makes no assumptions about the tag structure.
+   */
+  // TODO(dmaretskyi): Has to be optional for compatibility with old data.
+  // Defaulting to an empty array is possible but requires a bit more work.
+  tags: Schema.optional(Schema.Array(Schema.String)),
+});
+
+export type ObjectMeta = Schema.Schema.Type<typeof ObjectMetaSchema>;
+
+/*
+ * Get metadata from object.
+ * Only callable on the object root.
+ *
+ * @internal (use Obj.getMeta or Relation.getMeta)
+ */
+// TODO(burdon): Refine type to BaseObj.
+export const getMeta = (obj: AnyProperties): ObjectMeta => {
+  const metadata = (obj as any)[MetaId];
+  invariant(metadata, 'ObjectMeta not found.');
+  return metadata;
+};
+
+//
+// Foreign keys
+//
+
+export const foreignKey = (source: string, id: string): ForeignKey => ({ source, id });
+export const foreignKeyEquals = (a: ForeignKey, b: ForeignKey) => a.source === b.source && a.id === b.id;
+
+// TODO(dmaretskyi): Move to echo-schema.
+export const compareForeignKeys: Comparator<AnyProperties> = (a: AnyProperties, b: AnyProperties) =>
+  intersection(getMeta(a).keys, getMeta(b).keys, foreignKeyEquals).length > 0;

package/src/internal/types/typename.ts

@@ -0,0 +1,45 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import type * as Schema from 'effect/Schema';
+
+/**
+ * Property name for typename when object is serialized to JSON.
+ */
+export const ATTR_TYPE = '@type';
+
+/**
+ * DXN to the object type.
+ */
+export const TypeId = Symbol.for('@dxos/echo/Type');
+
+/**
+ * Reference to the object schema.
+ */
+export const SchemaId = Symbol.for('@dxos/echo/Schema');
+
+/**
+ * Returns the schema for the given object if one is defined.
+ *
+ * @internal (Use Obj.getSchema)
+ */
+// TODO(burdon): Narrow type.
+// TODO(dmaretskyi): For echo objects, this always returns the root schema.
+export const getSchema = (obj: unknown | undefined): Schema.Schema.AnyNoContext | undefined => {
+  if (obj) {
+    return (obj as any)[SchemaId];
+  }
+};
+
+/**
+ * @internal
+ */
+export const setSchema = (obj: any, schema: Schema.Schema.AnyNoContext): void => {
+  Object.defineProperty(obj, SchemaId, {
+    value: schema,
+    writable: false,
+    enumerable: false,
+    configurable: false,
+  });
+};

package/src/internal/types/version.ts

@@ -0,0 +1,20 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+/**
+ * Version type symbol.
+ */
+export const VersionTypeId: unique symbol = Symbol.for('@dxos/echo/VersionTypeId');
+
+export type VersionTypeId = typeof VersionTypeId;
+
+/**
+ * Represents the object version.
+ * May be backed by Automerge.
+ * Objects with no history are not versioned.
+ */
+export interface Version {
+  [VersionTypeId]: {};
+  automergeHeads?: string[];
+}

package/src/testing/api.test.ts

@@ -0,0 +1,126 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Schema from 'effect/Schema';
+import { describe, test } from 'vitest';
+
+import { raise } from '@dxos/debug';
+
+import { Entity, Obj, Ref, Relation, Type } from '../index';
+
+import { TestSchema } from './test-schema';
+
+describe('Experimental API review', () => {
+  test('type checks', ({ expect }) => {
+    const contact = Obj.make(TestSchema.Person, { name: 'Test' });
+    const schema = Obj.getSchema(contact) ?? raise(new Error('No schema found'));
+
+    expect(Type.getDXN(schema)?.typename).to.eq(TestSchema.Person.typename);
+    expect(Type.getTypename(schema)).to.eq('example.com/type/Person');
+    expect(Type.getVersion(schema)).to.eq('0.1.0');
+    expect(Type.getMeta(schema)).to.deep.eq({
+      kind: Entity.Kind.Object,
+      typename: 'example.com/type/Person',
+      version: '0.1.0',
+    });
+  });
+
+  test('instance checks', ({ expect }) => {
+    const organization = Obj.make(TestSchema.Organization, { name: 'DXOS' });
+    const contact = Obj.make(TestSchema.Person, {
+      name: 'Test',
+      employer: Ref.make(organization),
+    });
+
+    expect(Schema.is(TestSchema.Person)(contact)).to.be.true;
+    expect(Obj.instanceOf(TestSchema.Person, contact)).to.be.true;
+    expect(Obj.instanceOf(TestSchema.Organization, organization)).to.be.true;
+
+    const isPerson = Obj.instanceOf(TestSchema.Person);
+    const x: any = {};
+    if (isPerson(x)) {
+      x.name;
+    }
+  });
+
+  test('default props', ({ expect }) => {
+    const message = Obj.make(TestSchema.Message, TestSchema.MessageStruct.make({}));
+    expect(message.timestamp).to.exist;
+  });
+
+  test('Obj.isObject', ({ expect }) => {
+    const guy = Obj.make(TestSchema.Person, { name: 'Test' });
+    expect(Obj.isObject(guy)).to.be.true;
+    expect(Obj.isObject(null)).to.be.false;
+    expect(Obj.isObject(undefined)).to.be.false;
+    expect(Obj.isObject(1)).to.be.false;
+    expect(Obj.isObject('string')).to.be.false;
+  });
+
+  test('create relation', ({ expect }) => {
+    const person = Obj.make(TestSchema.Person, { name: 'Test' });
+    const organization = Obj.make(TestSchema.Organization, { name: 'DXOS' });
+    const worksFor = Relation.make(TestSchema.EmployedBy, {
+      [Relation.Source]: person,
+      [Relation.Target]: organization,
+      role: 'CEO',
+    });
+    expect(Relation.getSource(worksFor)).to.eq(person);
+    expect(Relation.getTarget(worksFor)).to.eq(organization);
+  });
+
+  test('version', ({ expect }) => {
+    const person = Obj.make(TestSchema.Person, { name: 'Test' });
+    const version = Obj.version(person);
+    expect(Obj.isVersion(version)).to.be.true;
+    expect(Obj.versionValid(version)).to.be.false;
+  });
+
+  // TODO(burdon): Implement test.
+  test.skip('type narrowing', () => {
+    const any: Entity.Unknown = null as any;
+
+    {
+      if (Obj.isObject(any)) {
+        any;
+      } else {
+        any;
+      }
+    }
+
+    {
+      if (Relation.isRelation(any)) {
+        any;
+      } else {
+        any;
+      }
+    }
+  });
+
+  /**
+   * Test that Obj.make return types are clean and don't expose internal markers like OfKind.
+   * The return type of Obj.make(Schema, props) should be assignable to Obj.Obj<InstanceType>.
+   */
+  describe('Obj.make return type', () => {
+    test('Obj.make with custom schema should be assignable to Obj.Obj<Person>', ({ expect }) => {
+      {
+        const expando = Obj.make(TestSchema.Expando, { content: 'Test' });
+        expect(Obj.isObject(expando)).to.be.true;
+        expect(expando.content).to.eq('Test');
+
+        // This should work: assign to Obj.Obj of the schema's instance type
+        const _typed: Obj.Obj<TestSchema.Expando> = expando;
+      }
+
+      {
+        const person = Obj.make(TestSchema.Person, { name: 'Test' });
+        expect(Obj.isObject(person)).to.be.true;
+        expect(person.name).to.eq('Test');
+
+        // This should work: assign to Obj.Obj of the schema's instance type
+        const _typed: Obj.Obj<TestSchema.Person> = person;
+      }
+    });
+  });
+});