@dxos/echo 0.8.4-main.c4373fc → 0.8.4-main.c85a9c8dae

package/src/internal/schema/echo-schema.ts

@@ -8,10 +8,12 @@ import * as SchemaAST from 'effect/SchemaAST';
 import { invariant } from '@dxos/invariant';
 import { type ObjectId } from '@dxos/keys';
 
-import { type SchemaMeta, SchemaMetaSymbol, type TypeAnnotation, getTypeAnnotation } from '../ast';
-import { toEffectSchema, toJsonSchema } from '../json';
-import { type JsonSchemaType } from '../json-schema';
-import { type TypedObject, type TypedObjectPrototype } from '../object';
+import type * as Type from '../../Type';
+import { type SchemaMeta, SchemaMetaSymbol, type TypeAnnotation, getTypeAnnotation } from '../annotations';
+import { type JsonSchemaType, toEffectSchema, toJsonSchema } from '../json-schema';
+import { type TypedObject, type TypedObjectPrototype, getSnapshot } from '../object';
+import { ChangeId } from '../proxy';
+import { EntityKind, SchemaKindId } from '../types';
 
 import {
   addFieldsToSchema,
@@ -20,14 +22,14 @@ import {
   updateFieldNameInSchema,
   updateFieldsInSchema,
 } from './manipulation';
-import { getSnapshot } from './snapshot';
-import { StoredSchema } from './stored-schema';
+import { PersistentSchema } from './persistent-schema';
 
 /**
  * Base schema type.
  */
 // TODO(burdon): Merge with ImmutableSchema.
 export interface BaseSchema<A = any, I = any> extends TypedObject<A, I> {
+  // TODO(burdon): Different from mutable?
   get readonly(): boolean;
   // TODO(burdon): Change to external function.
   get mutable(): EchoSchema<A, I>;
@@ -83,6 +85,10 @@ export class ImmutableSchema<A = any, I = any> implements BaseSchema<A, I> {
   // TypedObject
   //
 
+  get kind(): EntityKind {
+    return this._objectAnnotation.kind;
+  }
+
   get typename(): string {
     return this._objectAnnotation.typename;
   }
@@ -124,8 +130,10 @@ const EchoSchemaConstructor = (): TypedObjectPrototype => {
    */
   return class {
     private static get _schema() {
-      // The field is DynamicEchoSchema in runtime, but is serialized as StoredEchoSchema in automerge.
-      return Schema.Union(StoredSchema, Schema.instanceOf(EchoSchema)).annotations(StoredSchema.ast.annotations);
+      // The field is DynamicEchoSchema in runtime, but is serialized as PersisentSchema in automerge.
+      return Schema.Union(PersistentSchema, Schema.instanceOf(EchoSchema)).annotations(
+        PersistentSchema.ast.annotations,
+      );
     }
 
     static readonly [Schema.TypeId] = schemaVariance;
@@ -159,7 +167,7 @@ const schemaVariance = {
 };
 
 /**
- * Represents a schema that is stored in the ECHO database.
+ * Represents a schema that is persisted in the ECHO database.
  * Schema can me mutable or readonly (specified by the {@link EchoSchema.readonly} field).
  *
  * Schema that can be modified at runtime via the API.
@@ -170,20 +178,26 @@ const schemaVariance = {
  *
  * @example
  * ```ts
- * export class TableType extends Schema.Struct({...}).pipe(Type.Obj({ typename: 'example.org/type/Table', version: '0.1.0' })){
+ * export class TableType extends Schema.Struct({...}).pipe(Type.object({ typename: 'example.org/type/Table', version: '0.1.0' })){
  *   title: Schema.String,
  *   schema: Schema.optional(ref(EchoSchema)),
  *   props: Schema.mutable(S.Array(TablePropSchema)),
  * }) {}
  * ```
  *
- * The ECHO API will translate any references to StoredSchema objects to be resolved as EchoSchema objects.
+ * The ECHO API will translate any references to PersistentSchema objects to be resolved as EchoSchema objects.
  */
 export class EchoSchema<A = any, I = any> extends EchoSchemaConstructor() implements BaseSchema<A, I> {
   private _schema: Schema.Schema.AnyNoContext | undefined;
   private _isDirty = true;
 
-  constructor(private readonly _storedSchema: StoredSchema) {
+  /**
+   * Schema kind key that marks this as an ECHO schema.
+   * Makes EchoSchema satisfy the Type.AnyObj type.
+   */
+  readonly [SchemaKindId]: EntityKind.Object = EntityKind.Object;
+
+  constructor(private readonly _persistentSchema: PersistentSchema) {
     super();
   }
 
@@ -196,11 +210,11 @@ export class EchoSchema<A = any, I = any> extends EchoSchemaConstructor() implem
   }
 
   public get Type() {
-    return this._storedSchema as A;
+    return this._persistentSchema as A;
   }
 
   public get Encoded() {
-    return this._storedSchema as I;
+    return this._persistentSchema as I;
   }
 
   public get Context() {
@@ -227,12 +241,17 @@ export class EchoSchema<A = any, I = any> extends EchoSchemaConstructor() implem
   // BaseSchema
   //
 
+  public get kind(): EntityKind.Object {
+    // Dynamic schemas are always object schemas.
+    return EntityKind.Object;
+  }
+
   public get typename(): string {
-    return this._storedSchema.typename;
+    return this._persistentSchema.typename;
   }
 
   public get version(): string {
-    return this._storedSchema.version;
+    return this._persistentSchema.version;
   }
 
   public get readonly(): boolean {
@@ -247,10 +266,10 @@ export class EchoSchema<A = any, I = any> extends EchoSchemaConstructor() implem
   }
 
   /**
-   * @reactive
+   * Returns the JSON schema for the schema.
    */
   public get jsonSchema(): JsonSchemaType {
-    return this._storedSchema.jsonSchema;
+    return this._persistentSchema.jsonSchema;
   }
 
   /**
@@ -269,25 +288,25 @@ export class EchoSchema<A = any, I = any> extends EchoSchemaConstructor() implem
    * Id of the ECHO object containing the schema.
    */
   public get id(): ObjectId {
-    return this._storedSchema.id;
+    return this._persistentSchema.id;
   }
 
   /**
    * Short name of the schema.
    */
   public get name(): string | undefined {
-    return this._storedSchema.name;
+    return this._persistentSchema.name;
   }
 
   public get [SchemaMetaSymbol](): SchemaMeta {
-    return { id: this.id, typename: this.typename, version: this._storedSchema.version };
+    return { id: this.id, typename: this.typename, version: this._persistentSchema.version };
   }
 
   /**
-   * Reference to the underlying stored schema object.
+   * Reference to the underlying persistent schema object.
    */
-  public get storedSchema(): StoredSchema {
-    return this._storedSchema;
+  public get persistentSchema(): Type.PersistentType {
+    return this._persistentSchema as Type.PersistentType;
   }
 
   public getProperties(): SchemaAST.PropertySignature[] {
@@ -307,8 +326,10 @@ export class EchoSchema<A = any, I = any> extends EchoSchemaConstructor() implem
    */
   public updateTypename(typename: string): void {
     const updated = setTypenameInSchema(this._getSchema(), typename);
-    this._storedSchema.typename = typename;
-    this._storedSchema.jsonSchema = toJsonSchema(updated);
+    this._change((schema) => {
+      schema.typename = typename;
+      schema.jsonSchema = toJsonSchema(updated);
+    });
   }
 
   /**
@@ -316,7 +337,9 @@ export class EchoSchema<A = any, I = any> extends EchoSchemaConstructor() implem
    */
   public addFields(fields: Schema.Struct.Fields): void {
     const extended = addFieldsToSchema(this._getSchema(), fields);
-    this._storedSchema.jsonSchema = toJsonSchema(extended);
+    this._change((schema) => {
+      schema.jsonSchema = toJsonSchema(extended);
+    });
   }
 
   /**
@@ -324,7 +347,9 @@ export class EchoSchema<A = any, I = any> extends EchoSchemaConstructor() implem
    */
   public updateFields(fields: Schema.Struct.Fields): void {
     const updated = updateFieldsInSchema(this._getSchema(), fields);
-    this._storedSchema.jsonSchema = toJsonSchema(updated);
+    this._change((schema) => {
+      schema.jsonSchema = toJsonSchema(updated);
+    });
   }
 
   /**
@@ -332,7 +357,9 @@ export class EchoSchema<A = any, I = any> extends EchoSchemaConstructor() implem
    */
   public updateFieldPropertyName({ before, after }: { before: PropertyKey; after: PropertyKey }): void {
     const renamed = updateFieldNameInSchema(this._getSchema(), { before, after });
-    this._storedSchema.jsonSchema = toJsonSchema(renamed);
+    this._change((schema) => {
+      schema.jsonSchema = toJsonSchema(renamed);
+    });
   }
 
   /**
@@ -340,7 +367,21 @@ export class EchoSchema<A = any, I = any> extends EchoSchemaConstructor() implem
    */
   public removeFields(fieldNames: string[]): void {
     const removed = removeFieldsFromSchema(this._getSchema(), fieldNames);
-    this._storedSchema.jsonSchema = toJsonSchema(removed);
+    this._change((schema) => {
+      schema.jsonSchema = toJsonSchema(removed);
+    });
+  }
+
+  /**
+   * Wrapper for Obj.change that handles the change context for the persistent schema.
+   */
+  private _change(callback: (schema: PersistentSchema) => void): void {
+    const changeFn = (this._persistentSchema as any)[ChangeId];
+    if (changeFn) {
+      changeFn(callback);
+    } else {
+      callback(this._persistentSchema);
+    }
   }
 
   //
@@ -348,7 +389,7 @@ export class EchoSchema<A = any, I = any> extends EchoSchemaConstructor() implem
   //
 
   /**
-   * Called by EchoSchemaRegistry on update.
+   * Called by DatabaseSchemaRegistry on update.
    */
   _invalidate(): void {
     this._isDirty = true;
@@ -359,12 +400,12 @@ export class EchoSchema<A = any, I = any> extends EchoSchemaConstructor() implem
    */
   _rebuild(): void {
     if (this._isDirty || this._schema == null) {
-      this._schema = toEffectSchema(getSnapshot(this._storedSchema.jsonSchema));
+      this._schema = toEffectSchema(getSnapshot(this._persistentSchema).jsonSchema);
       this._isDirty = false;
     }
   }
 
-  private _getSchema(): Schema.Schema.AnyNoContext {
+  _getSchema(): Schema.Schema.AnyNoContext {
     this._rebuild();
     return this._schema!;
   }

package/src/internal/schema/index.ts

@@ -3,8 +3,5 @@
 //
 
 export * from './echo-schema';
-export * from './snapshot';
-export * from './stored-schema';
-export * from './runtime-schema-registry';
 export * from './manipulation';
-export * from './echo-schema';
+export * from './persistent-schema';

package/src/internal/schema/manipulation.ts

@@ -6,8 +6,9 @@ import * as Schema from 'effect/Schema';
 import * as SchemaAST from 'effect/SchemaAST';
 
 import { invariant } from '@dxos/invariant';
+import { DXN } from '@dxos/keys';
 
-import { type TypeAnnotation, TypeAnnotationId } from '../ast';
+import { type TypeAnnotation, TypeAnnotationId, TypeIdentifierAnnotationId } from '../annotations';
 
 // TODO(ZaymonFC): Do this one at a time. This might be dangerous.
 export const addFieldsToSchema = (
@@ -82,5 +83,10 @@ export const setTypenameInSchema = (
       typename,
       version: existingAnnotation.version,
     } satisfies TypeAnnotation,
+    [SchemaAST.JSONSchemaAnnotationId]: {
+      ...(schema.ast.annotations[SchemaAST.JSONSchemaAnnotationId] ?? {}),
+      $id: schema.ast.annotations[TypeIdentifierAnnotationId] ?? DXN.fromTypename(typename).toString(),
+      typename,
+    },
   });
 };

package/src/internal/schema/persistent-schema.ts

@@ -0,0 +1,33 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import * as Schema from 'effect/Schema';
+
+import { IconAnnotation } from '../annotations';
+import { LabelAnnotation, TypenameSchema, VersionSchema } from '../annotations';
+import { EchoObjectSchema } from '../entities';
+import { JsonSchemaType } from '../json-schema';
+
+/**
+ * Persistent representation of a schema.
+ */
+// TODO(burdon): Move.
+export const PersistentSchema = Schema.Struct({
+  name: Schema.optional(Schema.String),
+  typename: TypenameSchema,
+  version: VersionSchema,
+  jsonSchema: JsonSchemaType,
+}).pipe(
+  EchoObjectSchema({
+    typename: 'dxos.org/type/Schema',
+    version: '0.1.0',
+  }),
+  LabelAnnotation.set(['name']),
+  IconAnnotation.set({
+    icon: 'ph--brackets-curly--regular',
+    hue: 'blue',
+  }),
+);
+
+export interface PersistentSchema extends Schema.Schema.Type<typeof PersistentSchema> {}

package/src/internal/types/base.ts

@@ -0,0 +1,43 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import * as Schema from 'effect/Schema';
+import * as SchemaAST from 'effect/SchemaAST';
+
+import { type ObjectId } from '@dxos/keys';
+
+import { type ATTR_META, type ObjectMeta } from './meta';
+
+/**
+ * Base type for all data objects (reactive, ECHO, and other raw objects).
+ * NOTE: This describes the base type for all database objects.
+ * It is stricter than `T extends {}` or `T extends object`.
+ */
+// TODO(burdon): Prefer Record<string, unknown>.
+export type AnyProperties = Record<string, any>;
+
+/**
+ * Canonical type for all ECHO entities (objects and relations).
+ * @depreacted Remove, use Entity.Unknown instead.
+ */
+// TODO(wittjosiah): Remove. Prefer higher level types (e.g. Entity.Unknown).
+export interface AnyEntity {
+  readonly id: ObjectId;
+}
+
+export type ExcludeId<T extends AnyProperties> = Omit<T, 'id'>;
+
+export type PropertyKey<T extends AnyProperties> = Extract<keyof ExcludeId<T>, string>;
+
+// TODO(dmaretskyi): Remove. This should be using the symbol type.
+type WithMeta = { [ATTR_META]?: ObjectMeta };
+
+/**
+ * The raw object should not include the ECHO id, but may include metadata.
+ */
+export const RawObject = <S extends Schema.Schema.AnyNoContext>(
+  schema: S,
+): Schema.Schema<ExcludeId<Schema.Schema.Type<S>> & WithMeta, Schema.Schema.Encoded<S>> => {
+  return Schema.make(SchemaAST.omit(schema.ast, ['id']));
+};

package/src/internal/types/entity.ts

@@ -0,0 +1,54 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Schema from 'effect/Schema';
+
+// NOTE: String literals are used instead of unique symbols for both KindId and SchemaKindId.
+//   Unique symbols cause TS4023 "cannot be named" errors when external packages
+//   try to export types that reference this key (e.g., `export const Graph = ...`).
+//   TypeScript cannot emit declaration files that reference unique symbols from
+//   external modules. Using a string literal allows the type to be inlined in
+//   declaration files, making the API portable across package boundaries.
+
+/**
+ * String key used to identify the kind of an entity instance (object or relation).
+ */
+export const KindId = '~@dxos/echo/Kind' as const;
+export type KindId = typeof KindId;
+
+/**
+ * String key used to identify the kind of a schema (object schema or relation schema).
+ * Parallels KindId which identifies instance kinds.
+ */
+export const SchemaKindId = '~@dxos/echo/SchemaKind' as const;
+export type SchemaKindId = typeof SchemaKindId;
+
+/**
+ * String key used to brand snapshot types.
+ * Snapshots have SnapshotKindId instead of KindId, making them
+ * distinguishable from reactive objects at the type level.
+ */
+export const SnapshotKindId = '~@dxos/echo/SnapshotKind' as const;
+export type SnapshotKindId = typeof SnapshotKindId;
+
+/**
+ * Kinds of entities stored in ECHO: objects and relations.
+ */
+export enum EntityKind {
+  Object = 'object',
+  Relation = 'relation',
+}
+
+export const EntityKindSchema = Schema.Enums(EntityKind);
+
+/**
+ * Typename for generic object references (Type.Obj / Ref.Ref(Obj.Unknown)).
+ * Used when referencing any object without a specific schema.
+ */
+export const ANY_OBJECT_TYPENAME = 'dxos.org/schema/AnyObject';
+
+/**
+ * Version for generic object references.
+ */
+export const ANY_OBJECT_VERSION = '0.0.0';

package/src/internal/types/index.ts

@@ -2,5 +2,8 @@
 // Copyright 2024 DXOS.org
 //
 
-export * from './types';
-export type * from './util';
+export * from './base';
+export * from './entity';
+export * from './meta';
+export * from './typename';
+export * from './version';

package/src/internal/types/meta.ts

@@ -0,0 +1,68 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import * as Schema from 'effect/Schema';
+
+import { ForeignKey } from '@dxos/echo-protocol';
+import { invariant } from '@dxos/invariant';
+import { type Comparator, intersection } from '@dxos/util';
+
+import type * as Entity from '../../Entity';
+
+import { type AnyProperties } from './base';
+
+/**
+ * Property name for meta when object is serialized to JSON.
+ */
+export const ATTR_META = '@meta';
+
+/**
+ * Metadata section.
+ */
+export const MetaId: Entity.Meta = Symbol.for('@dxos/echo/Meta') as any;
+
+//
+// ObjectMeta
+//
+
+// TODO(dmaretskyi): Rename to ObjectMeta
+export const ObjectMetaSchema = Schema.Struct({
+  keys: Schema.Array(ForeignKey),
+
+  /**
+   * A set of tags.
+   * Tags are arbitrary application-defined strings.
+   * ECHO makes no assumptions about the tag structure.
+   */
+  // TODO(dmaretskyi): Has to be optional for compatibility with old data.
+  // Defaulting to an empty array is possible but requires a bit more work.
+  // TODO(dmaretskyi): In automerge this should be a map of { [tag]: boolean } for uniqueness and conflict resolution.
+  tags: Schema.optional(Schema.Array(Schema.String)),
+});
+
+export type ObjectMeta = Schema.Schema.Type<typeof ObjectMetaSchema>;
+
+/*
+ * Get metadata from object.
+ * Only callable on the object root.
+ *
+ * @internal (use Obj.getMeta or Relation.getMeta)
+ */
+// TODO(burdon): Refine type to BaseObj.
+export const getMeta = (obj: AnyProperties): ObjectMeta => {
+  const metadata = (obj as any)[MetaId];
+  invariant(metadata, 'ObjectMeta not found.');
+  return metadata;
+};
+
+//
+// Foreign keys
+//
+
+export const foreignKey = (source: string, id: string): ForeignKey => ({ source, id });
+export const foreignKeyEquals = (a: ForeignKey, b: ForeignKey) => a.source === b.source && a.id === b.id;
+
+// TODO(dmaretskyi): Move to echo-schema.
+export const compareForeignKeys: Comparator<AnyProperties> = (a: AnyProperties, b: AnyProperties) =>
+  intersection(getMeta(a).keys, getMeta(b).keys, foreignKeyEquals).length > 0;

package/src/internal/types/typename.ts

@@ -0,0 +1,55 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import type * as Schema from 'effect/Schema';
+
+/**
+ * Property name for typename when object is serialized to JSON.
+ */
+export const ATTR_TYPE = '@type';
+
+/**
+ * DXN to the object type.
+ */
+export const TypeId = Symbol.for('@dxos/echo/Type');
+
+/**
+ * Reference to the object schema.
+ */
+export const SchemaId = Symbol.for('@dxos/echo/Schema');
+
+/**
+ * Property name for parent when object is serialized to JSON.
+ */
+export const ATTR_PARENT = '@parent';
+
+/**
+ * Reference to the object parent.
+ */
+export const ParentId = Symbol.for('@dxos/echo/Parent');
+
+/**
+ * Returns the schema for the given object if one is defined.
+ *
+ * @internal (Use Obj.getSchema)
+ */
+// TODO(burdon): Narrow type.
+// TODO(dmaretskyi): For echo objects, this always returns the root schema.
+export const getSchema = (obj: unknown | undefined): Schema.Schema.AnyNoContext | undefined => {
+  if (obj) {
+    return (obj as any)[SchemaId];
+  }
+};
+
+/**
+ * @internal
+ */
+export const setSchema = (obj: any, schema: Schema.Schema.AnyNoContext): void => {
+  Object.defineProperty(obj, SchemaId, {
+    value: schema,
+    writable: false,
+    enumerable: false,
+    configurable: false,
+  });
+};

package/src/internal/types/version.ts

@@ -0,0 +1,19 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+/**
+ * Version type identifier.
+ */
+export const VersionTypeId = '~@dxos/echo/VersionTypeId' as const;
+export type VersionTypeId = typeof VersionTypeId;
+
+/**
+ * Represents the object version.
+ * May be backed by Automerge.
+ * Objects with no history are not versioned.
+ */
+export interface Version {
+  [VersionTypeId]: {};
+  automergeHeads?: string[];
+}