@dxos/echo 0.8.4-main.84f28bd → 0.8.4-main.937b3ca

package/src/internal/schema/echo-schema.ts

@@ -0,0 +1,423 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import * as Schema from 'effect/Schema';
+import * as SchemaAST from 'effect/SchemaAST';
+
+import { invariant } from '@dxos/invariant';
+import { type ObjectId } from '@dxos/keys';
+
+import { type SchemaMeta, SchemaMetaSymbol, type TypeAnnotation, getTypeAnnotation } from '../annotations';
+import { type JsonSchemaType, toEffectSchema, toJsonSchema } from '../json-schema';
+import { type TypedObject, type TypedObjectPrototype, getSnapshot } from '../object';
+import { ChangeId } from '../proxy';
+import { EntityKind, SchemaKindId } from '../types';
+
+import {
+  addFieldsToSchema,
+  removeFieldsFromSchema,
+  setTypenameInSchema,
+  updateFieldNameInSchema,
+  updateFieldsInSchema,
+} from './manipulation';
+import { PersistentSchema } from './persistent-schema';
+
+/**
+ * Base schema type.
+ */
+// TODO(burdon): Merge with ImmutableSchema.
+export interface BaseSchema<A = any, I = any> extends TypedObject<A, I> {
+  // TODO(burdon): Different from mutable?
+  get readonly(): boolean;
+  // TODO(burdon): Change to external function.
+  get mutable(): EchoSchema<A, I>;
+  get snapshot(): Schema.Schema<A, I>;
+  get jsonSchema(): JsonSchemaType;
+}
+
+/**
+ * Immutable schema type.
+ * @deprecated Use `Schema.Schema.AnyNoContext` instead.
+ */
+// TODO(burdon): Common abstract base class?
+export class ImmutableSchema<A = any, I = any> implements BaseSchema<A, I> {
+  private readonly _objectAnnotation: TypeAnnotation;
+  constructor(private readonly _schema: Schema.Schema<A, I>) {
+    this._objectAnnotation = getTypeAnnotation(this._schema)!;
+    invariant(this._objectAnnotation);
+  }
+
+  //
+  // Effect Schema (push to abstract base class).
+  //
+
+  public get [Schema.TypeId]() {
+    return schemaVariance;
+  }
+
+  public get Type() {
+    return this._schema.Type;
+  }
+
+  public get Encoded() {
+    return this._schema.Encoded;
+  }
+
+  public get Context() {
+    return this._schema.Context;
+  }
+
+  public get ast(): SchemaAST.AST {
+    return this._schema.ast;
+  }
+
+  public get annotations() {
+    return this._schema.annotations;
+  }
+
+  public get pipe() {
+    return this._schema.pipe;
+  }
+
+  //
+  // TypedObject
+  //
+
+  get kind(): EntityKind {
+    return this._objectAnnotation.kind;
+  }
+
+  get typename(): string {
+    return this._objectAnnotation.typename;
+  }
+
+  get version(): string {
+    return this._objectAnnotation.version;
+  }
+
+  //
+  // BaseSchema
+  //
+
+  get readonly(): boolean {
+    return true;
+  }
+
+  get snapshot(): Schema.Schema.AnyNoContext {
+    return this._schema;
+  }
+
+  // TODO(burdon): Change from getter since this is expensive.
+  get jsonSchema(): JsonSchemaType {
+    return toJsonSchema(this._schema);
+  }
+
+  get mutable(): EchoSchema {
+    throw new Error('Schema is readonly.');
+  }
+}
+
+/**
+ * Defines an effect-schema for the `EchoSchema` type.
+ *
+ * This is here so that `EchoSchema` class can be used as a part of another schema definition (e.g., `ref(EchoSchema)`).
+ */
+const EchoSchemaConstructor = (): TypedObjectPrototype => {
+  /**
+   * Return class definition satisfying Schema.Schema.
+   */
+  return class {
+    private static get _schema() {
+      // The field is DynamicEchoSchema in runtime, but is serialized as PersisentSchema in automerge.
+      return Schema.Union(PersistentSchema, Schema.instanceOf(EchoSchema)).annotations(
+        PersistentSchema.ast.annotations,
+      );
+    }
+
+    static readonly [Schema.TypeId] = schemaVariance;
+
+    static get ast() {
+      const schema = this._schema;
+      return schema.ast;
+    }
+
+    static get annotations() {
+      const schema = this._schema;
+      return schema.annotations.bind(schema);
+    }
+
+    static get pipe() {
+      const schema = this._schema;
+      return schema.pipe.bind(schema);
+    }
+  } as any;
+};
+
+export const isMutable = (schema: Schema.Schema.AnyNoContext): schema is EchoSchema => {
+  return schema instanceof EchoSchema;
+};
+
+// NOTE: Keep in this file.
+const schemaVariance = {
+  _A: (_: any) => _,
+  _I: (_: any) => _,
+  _R: (_: never) => _,
+};
+
+/**
+ * Represents a schema that is persisted in the ECHO database.
+ * Schema can me mutable or readonly (specified by the {@link EchoSchema.readonly} field).
+ *
+ * Schema that can be modified at runtime via the API.
+ * Is an instance of effect-schema (`Schema.Schema.AnyNoContext`) so it can be used in the same way as a regular schema.
+ * IMPORTANT: The schema AST will change reactively when the schema is updated, including synced updates from remote peers.
+ *
+ * The class constructor is a schema instance itself, and can be used in the echo object definitions:
+ *
+ * @example
+ * ```ts
+ * export class TableType extends Schema.Struct({...}).pipe(Type.object({ typename: 'example.org/type/Table', version: '0.1.0' })){
+ *   title: Schema.String,
+ *   schema: Schema.optional(ref(EchoSchema)),
+ *   props: Schema.mutable(S.Array(TablePropSchema)),
+ * }) {}
+ * ```
+ *
+ * The ECHO API will translate any references to PersistentSchema objects to be resolved as EchoSchema objects.
+ */
+export class EchoSchema<A = any, I = any> extends EchoSchemaConstructor() implements BaseSchema<A, I> {
+  private _schema: Schema.Schema.AnyNoContext | undefined;
+  private _isDirty = true;
+
+  /**
+   * Schema kind key that marks this as an ECHO schema.
+   * Makes EchoSchema satisfy the Type.Obj.Any type.
+   */
+  readonly [SchemaKindId]: EntityKind.Object = EntityKind.Object;
+
+  constructor(private readonly _persistentSchema: PersistentSchema) {
+    super();
+  }
+
+  //
+  // Effect Schema (push to abstract base class).
+  //
+
+  public get [Schema.TypeId]() {
+    return schemaVariance;
+  }
+
+  public get Type() {
+    return this._persistentSchema as A;
+  }
+
+  public get Encoded() {
+    return this._persistentSchema as I;
+  }
+
+  public get Context() {
+    const schema = this._getSchema();
+    return schema.Context;
+  }
+
+  public get ast() {
+    const schema = this._getSchema();
+    return schema.ast;
+  }
+
+  public get annotations() {
+    const schema = this._getSchema();
+    return schema.annotations.bind(schema);
+  }
+
+  public get pipe(): Schema.Schema.AnyNoContext['pipe'] {
+    const schema = this._getSchema();
+    return schema.pipe.bind(schema);
+  }
+
+  //
+  // BaseSchema
+  //
+
+  public get kind(): EntityKind.Object {
+    // Dynamic schemas are always object schemas.
+    return EntityKind.Object;
+  }
+
+  public get typename(): string {
+    return this._persistentSchema.typename;
+  }
+
+  public get version(): string {
+    return this._persistentSchema.version;
+  }
+
+  public get readonly(): boolean {
+    return false;
+  }
+
+  /**
+   * Returns an immutable schema snapshot of the current state of the schema.
+   */
+  public get snapshot(): Schema.Schema.AnyNoContext {
+    return this._getSchema();
+  }
+
+  /**
+   * Returns the JSON schema for the schema.
+   */
+  public get jsonSchema(): JsonSchemaType {
+    return this._persistentSchema.jsonSchema;
+  }
+
+  /**
+   * Returns a mutable schema.
+   */
+  public get mutable(): EchoSchema {
+    invariant(!this.readonly, 'Schema is not mutable');
+    return this;
+  }
+
+  //
+  // Mutable Schema
+  //
+
+  /**
+   * Id of the ECHO object containing the schema.
+   */
+  public get id(): ObjectId {
+    return this._persistentSchema.id;
+  }
+
+  /**
+   * Short name of the schema.
+   */
+  public get name(): string | undefined {
+    return this._persistentSchema.name;
+  }
+
+  public get [SchemaMetaSymbol](): SchemaMeta {
+    return { id: this.id, typename: this.typename, version: this._persistentSchema.version };
+  }
+
+  /**
+   * Reference to the underlying persistent schema object.
+   */
+  public get persistentSchema(): PersistentSchema {
+    return this._persistentSchema;
+  }
+
+  public getProperties(): SchemaAST.PropertySignature[] {
+    const ast = this._getSchema().ast;
+    invariant(SchemaAST.isTypeLiteral(ast));
+    return [...ast.propertySignatures].filter((p) => p.name !== 'id').map(unwrapOptionality);
+  }
+
+  //
+  // Mutation methods.
+  // TODO(burdon): Create separate interface for dynamic schema.
+  // TODO(burdon): Deprecate direct manipulation? Use JSONSchema directly.
+  //
+
+  /**
+   * @throws Error if the schema is readonly.
+   */
+  public updateTypename(typename: string): void {
+    const updated = setTypenameInSchema(this._getSchema(), typename);
+    this._change((schema) => {
+      schema.typename = typename;
+      schema.jsonSchema = toJsonSchema(updated);
+    });
+  }
+
+  /**
+   * @throws Error if the schema is readonly.
+   */
+  public addFields(fields: Schema.Struct.Fields): void {
+    const extended = addFieldsToSchema(this._getSchema(), fields);
+    this._change((schema) => {
+      schema.jsonSchema = toJsonSchema(extended);
+    });
+  }
+
+  /**
+   * @throws Error if the schema is readonly.
+   */
+  public updateFields(fields: Schema.Struct.Fields): void {
+    const updated = updateFieldsInSchema(this._getSchema(), fields);
+    this._change((schema) => {
+      schema.jsonSchema = toJsonSchema(updated);
+    });
+  }
+
+  /**
+   * @throws Error if the schema is readonly.
+   */
+  public updateFieldPropertyName({ before, after }: { before: PropertyKey; after: PropertyKey }): void {
+    const renamed = updateFieldNameInSchema(this._getSchema(), { before, after });
+    this._change((schema) => {
+      schema.jsonSchema = toJsonSchema(renamed);
+    });
+  }
+
+  /**
+   * @throws Error if the schema is readonly.
+   */
+  public removeFields(fieldNames: string[]): void {
+    const removed = removeFieldsFromSchema(this._getSchema(), fieldNames);
+    this._change((schema) => {
+      schema.jsonSchema = toJsonSchema(removed);
+    });
+  }
+
+  /**
+   * Wrapper for Obj.change that handles the change context for the persistent schema.
+   */
+  private _change(callback: (schema: PersistentSchema) => void): void {
+    const changeFn = (this._persistentSchema as any)[ChangeId];
+    if (changeFn) {
+      changeFn(callback);
+    } else {
+      callback(this._persistentSchema);
+    }
+  }
+
+  //
+  // Internals
+  //
+
+  /**
+   * Called by DatabaseSchemaRegistry on update.
+   */
+  _invalidate(): void {
+    this._isDirty = true;
+  }
+
+  /**
+   * Rebuilds this schema if it is dirty.
+   */
+  _rebuild(): void {
+    if (this._isDirty || this._schema == null) {
+      this._schema = toEffectSchema(getSnapshot(this._persistentSchema.jsonSchema));
+      this._isDirty = false;
+    }
+  }
+
+  _getSchema(): Schema.Schema.AnyNoContext {
+    this._rebuild();
+    return this._schema!;
+  }
+}
+
+// TODO(burdon): Move to effect.
+const unwrapOptionality = (property: SchemaAST.PropertySignature): SchemaAST.PropertySignature => {
+  if (!SchemaAST.isUnion(property.type)) {
+    return property;
+  }
+
+  return {
+    ...property,
+    type: property.type.types.find((type) => !SchemaAST.isUndefinedKeyword(type))!,
+  } as any;
+};

package/src/internal/schema/index.ts

@@ -0,0 +1,7 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+export * from './echo-schema';
+export * from './manipulation';
+export * from './persistent-schema';

package/src/internal/schema/manipulation.ts

@@ -0,0 +1,92 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import * as Schema from 'effect/Schema';
+import * as SchemaAST from 'effect/SchemaAST';
+
+import { invariant } from '@dxos/invariant';
+import { DXN } from '@dxos/keys';
+
+import { type TypeAnnotation, TypeAnnotationId, TypeIdentifierAnnotationId } from '../annotations';
+
+// TODO(ZaymonFC): Do this one at a time. This might be dangerous.
+export const addFieldsToSchema = (
+  schema: Schema.Schema.AnyNoContext,
+  fields: Schema.Struct.Fields,
+): Schema.Schema.AnyNoContext => {
+  const schemaExtension = Schema.partial(Schema.Struct(fields));
+  return Schema.extend(schema, schemaExtension).annotations(
+    schema.ast.annotations,
+  ) as any as Schema.Schema.AnyNoContext;
+};
+
+export const updateFieldsInSchema = (
+  schema: Schema.Schema.AnyNoContext,
+  fields: Schema.Struct.Fields,
+): Schema.Schema.AnyNoContext => {
+  const ast = schema.ast as SchemaAST.TypeLiteral;
+  invariant(SchemaAST.isTypeLiteral(ast));
+
+  const updatedProperties = [...ast.propertySignatures];
+  const propertiesToUpdate = (Schema.partial(Schema.Struct(fields)).ast as SchemaAST.TypeLiteral).propertySignatures;
+  for (const property of propertiesToUpdate) {
+    const index = updatedProperties.findIndex((p) => p.name === property.name);
+    if (index !== -1) {
+      updatedProperties[index] = property;
+    } else {
+      updatedProperties.push(property);
+    }
+  }
+
+  return Schema.make(new SchemaAST.TypeLiteral(updatedProperties, ast.indexSignatures, ast.annotations));
+};
+
+export const removeFieldsFromSchema = (
+  schema: Schema.Schema.AnyNoContext,
+  fieldNames: string[],
+): Schema.Schema.AnyNoContext => {
+  return Schema.make(SchemaAST.omit(schema.ast, fieldNames)).annotations(schema.ast.annotations);
+};
+
+export const updateFieldNameInSchema = (
+  schema: Schema.Schema.AnyNoContext,
+  { before, after }: { before: PropertyKey; after: PropertyKey },
+): Schema.Schema.AnyNoContext => {
+  const ast = schema.ast as SchemaAST.TypeLiteral;
+  invariant(SchemaAST.isTypeLiteral(ast));
+
+  return Schema.make(
+    new SchemaAST.TypeLiteral(
+      ast.propertySignatures.map((p) =>
+        p.name === before
+          ? new SchemaAST.PropertySignature(after, p.type, p.isOptional, p.isReadonly, p.annotations)
+          : p,
+      ),
+      ast.indexSignatures,
+      ast.annotations,
+    ),
+  );
+};
+
+export const setTypenameInSchema = (
+  schema: Schema.Schema.AnyNoContext,
+  typename: string,
+): Schema.Schema.AnyNoContext => {
+  const existingAnnotation = schema.ast.annotations[TypeAnnotationId] as TypeAnnotation;
+  invariant(existingAnnotation, `Missing ${String(TypeAnnotationId)}`);
+
+  return schema.annotations({
+    ...schema.ast.annotations,
+    [TypeAnnotationId]: {
+      kind: existingAnnotation.kind,
+      typename,
+      version: existingAnnotation.version,
+    } satisfies TypeAnnotation,
+    [SchemaAST.JSONSchemaAnnotationId]: {
+      ...(schema.ast.annotations[SchemaAST.JSONSchemaAnnotationId] ?? {}),
+      $id: schema.ast.annotations[TypeIdentifierAnnotationId] ?? DXN.fromTypename(typename).toString(),
+      typename,
+    },
+  });
+};

package/src/internal/schema/persistent-schema.ts

@@ -0,0 +1,27 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import * as Schema from 'effect/Schema';
+
+import { TypenameSchema, VersionSchema } from '../annotations';
+import { EchoObjectSchema } from '../entities';
+import { JsonSchemaType } from '../json-schema';
+
+/**
+ * Persistent representation of a schema.
+ */
+// TODO(burdon): Move.
+export const PersistentSchema = Schema.Struct({
+  name: Schema.optional(Schema.String),
+  typename: TypenameSchema,
+  version: VersionSchema,
+  jsonSchema: JsonSchemaType,
+}).pipe(
+  EchoObjectSchema({
+    typename: 'dxos.org/type/Schema',
+    version: '0.1.0',
+  }),
+);
+
+export interface PersistentSchema extends Schema.Schema.Type<typeof PersistentSchema> {}

package/src/internal/types/base.ts

@@ -0,0 +1,43 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import * as Schema from 'effect/Schema';
+import * as SchemaAST from 'effect/SchemaAST';
+
+import { type ObjectId } from '@dxos/keys';
+
+import { type ATTR_META, type ObjectMeta } from './meta';
+
+/**
+ * Base type for all data objects (reactive, ECHO, and other raw objects).
+ * NOTE: This describes the base type for all database objects.
+ * It is stricter than `T extends {}` or `T extends object`.
+ */
+// TODO(burdon): Make internal.
+// TODO(burdon): Prefer Record<string, unknown>.
+export type AnyProperties = Record<string, any>;
+
+/**
+ * Canonical type for all ECHO entities (objects and relations).
+ */
+// TODO(wittjosiah): Remove. Prefer higher level types (e.g. Entity.Unknown).
+export interface AnyEntity {
+  readonly id: ObjectId;
+}
+
+export type ExcludeId<T extends AnyProperties> = Omit<T, 'id'>;
+
+export type PropertyKey<T extends AnyProperties> = Extract<keyof ExcludeId<T>, string>;
+
+// TODO(dmaretskyi): Remove. This should be using the symbol type.
+type WithMeta = { [ATTR_META]?: ObjectMeta };
+
+/**
+ * The raw object should not include the ECHO id, but may include metadata.
+ */
+export const RawObject = <S extends Schema.Schema.AnyNoContext>(
+  schema: S,
+): Schema.Schema<ExcludeId<Schema.Schema.Type<S>> & WithMeta, Schema.Schema.Encoded<S>> => {
+  return Schema.make(SchemaAST.omit(schema.ast, ['id']));
+};

package/src/internal/types/entity.ts

@@ -0,0 +1,54 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Schema from 'effect/Schema';
+
+// NOTE: String literals are used instead of unique symbols for both KindId and SchemaKindId.
+//   Unique symbols cause TS4023 "cannot be named" errors when external packages
+//   try to export types that reference this key (e.g., `export const Graph = ...`).
+//   TypeScript cannot emit declaration files that reference unique symbols from
+//   external modules. Using a string literal allows the type to be inlined in
+//   declaration files, making the API portable across package boundaries.
+
+/**
+ * String key used to identify the kind of an entity instance (object or relation).
+ */
+export const KindId = '~@dxos/echo/Kind' as const;
+export type KindId = typeof KindId;
+
+/**
+ * String key used to identify the kind of a schema (object schema or relation schema).
+ * Parallels KindId which identifies instance kinds.
+ */
+export const SchemaKindId = '~@dxos/echo/SchemaKind' as const;
+export type SchemaKindId = typeof SchemaKindId;
+
+/**
+ * String key used to brand snapshot types.
+ * Snapshots have SnapshotKindId instead of KindId, making them
+ * distinguishable from reactive objects at the type level.
+ */
+export const SnapshotKindId = '~@dxos/echo/SnapshotKind' as const;
+export type SnapshotKindId = typeof SnapshotKindId;
+
+/**
+ * Kinds of entities stored in ECHO: objects and relations.
+ */
+export enum EntityKind {
+  Object = 'object',
+  Relation = 'relation',
+}
+
+export const EntityKindSchema = Schema.Enums(EntityKind);
+
+/**
+ * Typename for generic object references (Type.Obj / Type.Ref(Type.Obj)).
+ * Used when referencing any object without a specific schema.
+ */
+export const ANY_OBJECT_TYPENAME = 'dxos.org/schema/AnyObject';
+
+/**
+ * Version for generic object references.
+ */
+export const ANY_OBJECT_VERSION = '0.0.0';

package/src/internal/types/index.ts

@@ -0,0 +1,9 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+export * from './base';
+export * from './entity';
+export * from './meta';
+export * from './typename';
+export * from './version';