@dxos/echo 0.8.4-main.5ea62a8 → 0.8.4-main.66e292d

package/src/internal/schema/echo-schema.ts

@@ -0,0 +1,385 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import * as Schema from 'effect/Schema';
+import * as SchemaAST from 'effect/SchemaAST';
+
+import { invariant } from '@dxos/invariant';
+import { type ObjectId } from '@dxos/keys';
+
+import { type SchemaMeta, SchemaMetaSymbol, type TypeAnnotation, getTypeAnnotation } from '../annotations';
+import { type JsonSchemaType, toEffectSchema, toJsonSchema } from '../json-schema';
+import { type TypedObject, type TypedObjectPrototype } from '../object';
+
+import {
+  addFieldsToSchema,
+  removeFieldsFromSchema,
+  setTypenameInSchema,
+  updateFieldNameInSchema,
+  updateFieldsInSchema,
+} from './manipulation';
+import { PersistentSchema } from './persistent-schema';
+import { getSnapshot } from './snapshot';
+
+/**
+ * Base schema type.
+ */
+// TODO(burdon): Merge with ImmutableSchema.
+export interface BaseSchema<A = any, I = any> extends TypedObject<A, I> {
+  // TODO(burdon): Different from mutable?
+  get readonly(): boolean;
+  // TODO(burdon): Change to external function.
+  get mutable(): EchoSchema<A, I>;
+  get snapshot(): Schema.Schema<A, I>;
+  get jsonSchema(): JsonSchemaType;
+}
+
+/**
+ * Immutable schema type.
+ * @deprecated Use `Schema.Schema.AnyNoContext` instead.
+ */
+// TODO(burdon): Common abstract base class?
+export class ImmutableSchema<A = any, I = any> implements BaseSchema<A, I> {
+  private readonly _objectAnnotation: TypeAnnotation;
+  constructor(private readonly _schema: Schema.Schema<A, I>) {
+    this._objectAnnotation = getTypeAnnotation(this._schema)!;
+    invariant(this._objectAnnotation);
+  }
+
+  //
+  // Effect Schema (push to abstract base class).
+  //
+
+  public get [Schema.TypeId]() {
+    return schemaVariance;
+  }
+
+  public get Type() {
+    return this._schema.Type;
+  }
+
+  public get Encoded() {
+    return this._schema.Encoded;
+  }
+
+  public get Context() {
+    return this._schema.Context;
+  }
+
+  public get ast(): SchemaAST.AST {
+    return this._schema.ast;
+  }
+
+  public get annotations() {
+    return this._schema.annotations;
+  }
+
+  public get pipe() {
+    return this._schema.pipe;
+  }
+
+  //
+  // TypedObject
+  //
+
+  get typename(): string {
+    return this._objectAnnotation.typename;
+  }
+
+  get version(): string {
+    return this._objectAnnotation.version;
+  }
+
+  //
+  // BaseSchema
+  //
+
+  get readonly(): boolean {
+    return true;
+  }
+
+  get snapshot(): Schema.Schema.AnyNoContext {
+    return this._schema;
+  }
+
+  // TODO(burdon): Change from getter since this is expensive.
+  get jsonSchema(): JsonSchemaType {
+    return toJsonSchema(this._schema);
+  }
+
+  get mutable(): EchoSchema {
+    throw new Error('Schema is readonly.');
+  }
+}
+
+/**
+ * Defines an effect-schema for the `EchoSchema` type.
+ *
+ * This is here so that `EchoSchema` class can be used as a part of another schema definition (e.g., `ref(EchoSchema)`).
+ */
+const EchoSchemaConstructor = (): TypedObjectPrototype => {
+  /**
+   * Return class definition satisfying Schema.Schema.
+   */
+  return class {
+    private static get _schema() {
+      // The field is DynamicEchoSchema in runtime, but is serialized as PersisentSchema in automerge.
+      return Schema.Union(PersistentSchema, Schema.instanceOf(EchoSchema)).annotations(
+        PersistentSchema.ast.annotations,
+      );
+    }
+
+    static readonly [Schema.TypeId] = schemaVariance;
+
+    static get ast() {
+      const schema = this._schema;
+      return schema.ast;
+    }
+
+    static get annotations() {
+      const schema = this._schema;
+      return schema.annotations.bind(schema);
+    }
+
+    static get pipe() {
+      const schema = this._schema;
+      return schema.pipe.bind(schema);
+    }
+  } as any;
+};
+
+export const isMutable = (schema: Schema.Schema.AnyNoContext): schema is EchoSchema => {
+  return schema instanceof EchoSchema;
+};
+
+// NOTE: Keep in this file.
+const schemaVariance = {
+  _A: (_: any) => _,
+  _I: (_: any) => _,
+  _R: (_: never) => _,
+};
+
+/**
+ * Represents a schema that is persisted in the ECHO database.
+ * Schema can me mutable or readonly (specified by the {@link EchoSchema.readonly} field).
+ *
+ * Schema that can be modified at runtime via the API.
+ * Is an instance of effect-schema (`Schema.Schema.AnyNoContext`) so it can be used in the same way as a regular schema.
+ * IMPORTANT: The schema AST will change reactively when the schema is updated, including synced updates from remote peers.
+ *
+ * The class constructor is a schema instance itself, and can be used in the echo object definitions:
+ *
+ * @example
+ * ```ts
+ * export class TableType extends Schema.Struct({...}).pipe(Type.Obj({ typename: 'example.org/type/Table', version: '0.1.0' })){
+ *   title: Schema.String,
+ *   schema: Schema.optional(ref(EchoSchema)),
+ *   props: Schema.mutable(S.Array(TablePropSchema)),
+ * }) {}
+ * ```
+ *
+ * The ECHO API will translate any references to PersistentSchema objects to be resolved as EchoSchema objects.
+ */
+export class EchoSchema<A = any, I = any> extends EchoSchemaConstructor() implements BaseSchema<A, I> {
+  private _schema: Schema.Schema.AnyNoContext | undefined;
+  private _isDirty = true;
+
+  constructor(private readonly _persistentSchema: PersistentSchema) {
+    super();
+  }
+
+  //
+  // Effect Schema (push to abstract base class).
+  //
+
+  public get [Schema.TypeId]() {
+    return schemaVariance;
+  }
+
+  public get Type() {
+    return this._persistentSchema as A;
+  }
+
+  public get Encoded() {
+    return this._persistentSchema as I;
+  }
+
+  public get Context() {
+    const schema = this._getSchema();
+    return schema.Context;
+  }
+
+  public get ast() {
+    const schema = this._getSchema();
+    return schema.ast;
+  }
+
+  public get annotations() {
+    const schema = this._getSchema();
+    return schema.annotations.bind(schema);
+  }
+
+  public get pipe(): Schema.Schema.AnyNoContext['pipe'] {
+    const schema = this._getSchema();
+    return schema.pipe.bind(schema);
+  }
+
+  //
+  // BaseSchema
+  //
+
+  public get typename(): string {
+    return this._persistentSchema.typename;
+  }
+
+  public get version(): string {
+    return this._persistentSchema.version;
+  }
+
+  public get readonly(): boolean {
+    return false;
+  }
+
+  /**
+   * Returns an immutable schema snapshot of the current state of the schema.
+   */
+  public get snapshot(): Schema.Schema.AnyNoContext {
+    return this._getSchema();
+  }
+
+  /**
+   * @reactive
+   */
+  public get jsonSchema(): JsonSchemaType {
+    return this._persistentSchema.jsonSchema;
+  }
+
+  /**
+   * Returns a mutable schema.
+   */
+  public get mutable(): EchoSchema {
+    invariant(!this.readonly, 'Schema is not mutable');
+    return this;
+  }
+
+  //
+  // Mutable Schema
+  //
+
+  /**
+   * Id of the ECHO object containing the schema.
+   */
+  public get id(): ObjectId {
+    return this._persistentSchema.id;
+  }
+
+  /**
+   * Short name of the schema.
+   */
+  public get name(): string | undefined {
+    return this._persistentSchema.name;
+  }
+
+  public get [SchemaMetaSymbol](): SchemaMeta {
+    return { id: this.id, typename: this.typename, version: this._persistentSchema.version };
+  }
+
+  /**
+   * Reference to the underlying persistent schema object.
+   */
+  public get persistentSchema(): PersistentSchema {
+    return this._persistentSchema;
+  }
+
+  public getProperties(): SchemaAST.PropertySignature[] {
+    const ast = this._getSchema().ast;
+    invariant(SchemaAST.isTypeLiteral(ast));
+    return [...ast.propertySignatures].filter((p) => p.name !== 'id').map(unwrapOptionality);
+  }
+
+  //
+  // Mutation methods.
+  // TODO(burdon): Create separate interface for dynamic schema.
+  // TODO(burdon): Deprecate direct manipulation? Use JSONSchema directly.
+  //
+
+  /**
+   * @throws Error if the schema is readonly.
+   */
+  public updateTypename(typename: string): void {
+    const updated = setTypenameInSchema(this._getSchema(), typename);
+    this._persistentSchema.typename = typename;
+    this._persistentSchema.jsonSchema = toJsonSchema(updated);
+  }
+
+  /**
+   * @throws Error if the schema is readonly.
+   */
+  public addFields(fields: Schema.Struct.Fields): void {
+    const extended = addFieldsToSchema(this._getSchema(), fields);
+    this._persistentSchema.jsonSchema = toJsonSchema(extended);
+  }
+
+  /**
+   * @throws Error if the schema is readonly.
+   */
+  public updateFields(fields: Schema.Struct.Fields): void {
+    const updated = updateFieldsInSchema(this._getSchema(), fields);
+    this._persistentSchema.jsonSchema = toJsonSchema(updated);
+  }
+
+  /**
+   * @throws Error if the schema is readonly.
+   */
+  public updateFieldPropertyName({ before, after }: { before: PropertyKey; after: PropertyKey }): void {
+    const renamed = updateFieldNameInSchema(this._getSchema(), { before, after });
+    this._persistentSchema.jsonSchema = toJsonSchema(renamed);
+  }
+
+  /**
+   * @throws Error if the schema is readonly.
+   */
+  public removeFields(fieldNames: string[]): void {
+    const removed = removeFieldsFromSchema(this._getSchema(), fieldNames);
+    this._persistentSchema.jsonSchema = toJsonSchema(removed);
+  }
+
+  //
+  // Internals
+  //
+
+  /**
+   * Called by EchoSchemaRegistry on update.
+   */
+  _invalidate(): void {
+    this._isDirty = true;
+  }
+
+  /**
+   * Rebuilds this schema if it is dirty.
+   */
+  _rebuild(): void {
+    if (this._isDirty || this._schema == null) {
+      this._schema = toEffectSchema(getSnapshot(this._persistentSchema.jsonSchema));
+      this._isDirty = false;
+    }
+  }
+
+  _getSchema(): Schema.Schema.AnyNoContext {
+    this._rebuild();
+    return this._schema!;
+  }
+}
+
+// TODO(burdon): Move to effect.
+const unwrapOptionality = (property: SchemaAST.PropertySignature): SchemaAST.PropertySignature => {
+  if (!SchemaAST.isUnion(property.type)) {
+    return property;
+  }
+
+  return {
+    ...property,
+    type: property.type.types.find((type) => !SchemaAST.isUndefinedKeyword(type))!,
+  } as any;
+};

package/src/internal/schema/index.ts

@@ -0,0 +1,9 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+export * from './echo-schema';
+export * from './manipulation';
+export * from './persistent-schema';
+export * from './runtime-schema-registry';
+export * from './snapshot';

package/src/internal/schema/manipulation.ts

@@ -0,0 +1,92 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import * as Schema from 'effect/Schema';
+import * as SchemaAST from 'effect/SchemaAST';
+
+import { invariant } from '@dxos/invariant';
+import { DXN } from '@dxos/keys';
+
+import { type TypeAnnotation, TypeAnnotationId, TypeIdentifierAnnotationId } from '../annotations';
+
+// TODO(ZaymonFC): Do this one at a time. This might be dangerous.
+export const addFieldsToSchema = (
+  schema: Schema.Schema.AnyNoContext,
+  fields: Schema.Struct.Fields,
+): Schema.Schema.AnyNoContext => {
+  const schemaExtension = Schema.partial(Schema.Struct(fields));
+  return Schema.extend(schema, schemaExtension).annotations(
+    schema.ast.annotations,
+  ) as any as Schema.Schema.AnyNoContext;
+};
+
+export const updateFieldsInSchema = (
+  schema: Schema.Schema.AnyNoContext,
+  fields: Schema.Struct.Fields,
+): Schema.Schema.AnyNoContext => {
+  const ast = schema.ast as SchemaAST.TypeLiteral;
+  invariant(SchemaAST.isTypeLiteral(ast));
+
+  const updatedProperties = [...ast.propertySignatures];
+  const propertiesToUpdate = (Schema.partial(Schema.Struct(fields)).ast as SchemaAST.TypeLiteral).propertySignatures;
+  for (const property of propertiesToUpdate) {
+    const index = updatedProperties.findIndex((p) => p.name === property.name);
+    if (index !== -1) {
+      updatedProperties[index] = property;
+    } else {
+      updatedProperties.push(property);
+    }
+  }
+
+  return Schema.make(new SchemaAST.TypeLiteral(updatedProperties, ast.indexSignatures, ast.annotations));
+};
+
+export const removeFieldsFromSchema = (
+  schema: Schema.Schema.AnyNoContext,
+  fieldNames: string[],
+): Schema.Schema.AnyNoContext => {
+  return Schema.make(SchemaAST.omit(schema.ast, fieldNames)).annotations(schema.ast.annotations);
+};
+
+export const updateFieldNameInSchema = (
+  schema: Schema.Schema.AnyNoContext,
+  { before, after }: { before: PropertyKey; after: PropertyKey },
+): Schema.Schema.AnyNoContext => {
+  const ast = schema.ast as SchemaAST.TypeLiteral;
+  invariant(SchemaAST.isTypeLiteral(ast));
+
+  return Schema.make(
+    new SchemaAST.TypeLiteral(
+      ast.propertySignatures.map((p) =>
+        p.name === before
+          ? new SchemaAST.PropertySignature(after, p.type, p.isOptional, p.isReadonly, p.annotations)
+          : p,
+      ),
+      ast.indexSignatures,
+      ast.annotations,
+    ),
+  );
+};
+
+export const setTypenameInSchema = (
+  schema: Schema.Schema.AnyNoContext,
+  typename: string,
+): Schema.Schema.AnyNoContext => {
+  const existingAnnotation = schema.ast.annotations[TypeAnnotationId] as TypeAnnotation;
+  invariant(existingAnnotation, `Missing ${String(TypeAnnotationId)}`);
+
+  return schema.annotations({
+    ...schema.ast.annotations,
+    [TypeAnnotationId]: {
+      kind: existingAnnotation.kind,
+      typename,
+      version: existingAnnotation.version,
+    } satisfies TypeAnnotation,
+    [SchemaAST.JSONSchemaAnnotationId]: {
+      ...(schema.ast.annotations[SchemaAST.JSONSchemaAnnotationId] ?? {}),
+      $id: schema.ast.annotations[TypeIdentifierAnnotationId] ?? DXN.fromTypename(typename).toString(),
+      typename,
+    },
+  });
+};

package/src/internal/schema/persistent-schema.ts

@@ -0,0 +1,28 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import * as Schema from 'effect/Schema';
+
+import { TypenameSchema, VersionSchema } from '../annotations';
+import { EchoObjectSchema } from '../entities';
+import { JsonSchemaType } from '../json-schema';
+
+/**
+ * Persistent representation of a schema.
+ */
+// TODO(burdon): Move.
+const PersistentEchoSchema = Schema.Struct({
+  name: Schema.optional(Schema.String),
+  typename: TypenameSchema,
+  version: VersionSchema,
+  jsonSchema: JsonSchemaType,
+}).pipe(
+  EchoObjectSchema({
+    typename: 'dxos.org/type/Schema',
+    version: '0.1.0',
+  }),
+);
+export interface PersistentSchema extends Schema.Schema.Type<typeof PersistentEchoSchema> {}
+export interface PersistentSchemaEncoded extends Schema.Schema.Encoded<typeof PersistentEchoSchema> {}
+export const PersistentSchema: Schema.Schema<PersistentSchema, PersistentSchemaEncoded> = PersistentEchoSchema;

package/src/internal/schema/runtime-schema-registry.ts

@@ -0,0 +1,78 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+import type * as Schema from 'effect/Schema';
+
+import { raise } from '@dxos/debug';
+import { invariant } from '@dxos/invariant';
+import { type DXN } from '@dxos/keys';
+import { defaultMap } from '@dxos/util';
+
+import { getSchemaTypename, getSchemaVersion } from '../annotations';
+
+import { PersistentSchema } from './persistent-schema';
+
+/**
+ * Runtime registry of static schema objects (i.e., not Dynamic .
+ */
+// TODO(burdon): Reconcile with EchoSchemaRegistry.
+export class RuntimeSchemaRegistry {
+  private readonly _registry = new Map<string, Schema.Schema.AnyNoContext[]>();
+
+  constructor() {
+    this._registry.set(getSchemaTypename(PersistentSchema)!, [PersistentSchema]);
+  }
+
+  get schemas(): Schema.Schema.AnyNoContext[] {
+    return Array.from(this._registry.values()).flat();
+  }
+
+  hasSchema<S extends Schema.Schema.AnyNoContext>(schema: S): boolean {
+    const typename = getSchemaTypename(schema);
+    const version = getSchemaVersion(schema);
+    invariant(typename, 'Invalid schema');
+
+    const schemas = this._registry.get(typename);
+    return schemas?.some((schema) => getSchemaVersion(schema) === version) ?? false;
+  }
+
+  getSchemaByDXN(dxn: DXN): Schema.Schema.AnyNoContext | undefined {
+    const components = dxn.asTypeDXN();
+    if (!components) {
+      return undefined;
+    }
+
+    const { type, version } = components;
+    const allSchemas = this._registry.get(type) ?? [];
+    if (version) {
+      return allSchemas.find((s) => getSchemaVersion(s) === version);
+    } else {
+      // If no version is specified, return the earliest version for backwards compatibility.
+      // TODO(dmaretskyi): Probably not correct to compare lexicographically, but it's good enough for now.
+      return allSchemas.sort((a, b) =>
+        (getSchemaVersion(a) ?? '0.0.0').localeCompare(getSchemaVersion(b) ?? '0.0.0'),
+      )[0];
+    }
+  }
+
+  /**
+   * @deprecated Use getSchemaByDXN.
+   */
+  getSchema(typename: string): Schema.Schema.AnyNoContext | undefined {
+    return this._registry.get(typename)?.[0];
+  }
+
+  addSchema(types: readonly Schema.Schema.AnyNoContext[]): void {
+    types.forEach((schema) => {
+      const typename = getSchemaTypename(schema) ?? raise(new TypeError('Schema has no typename'));
+      const version = getSchemaVersion(schema) ?? raise(new TypeError('Schema has no version'));
+      const versions = defaultMap(this._registry, typename, () => []);
+      if (versions.some((schema) => getSchemaVersion(schema) === version)) {
+        throw new Error(`Schema version already registered: ${typename}:${version}`);
+      }
+
+      versions.push(schema);
+    });
+  }
+}

package/src/internal/schema/snapshot.ts

@@ -0,0 +1,25 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+/**
+ * Returns a non-reactive snapshot of the given live object.
+ * @deprecated Use `getSnapshot` from `@dxos/live-object` instead.
+ */
+// TODO(wittjosiah): Types.
+export const getSnapshot = (object: any): any => {
+  if (typeof object !== 'object') {
+    return object;
+  }
+
+  if (Array.isArray(object)) {
+    return object.map(getSnapshot);
+  }
+
+  const result: any = {};
+  for (const key in object) {
+    result[key] = getSnapshot(object[key]);
+  }
+
+  return result;
+};

package/src/internal/types/base.ts

@@ -0,0 +1,58 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import * as Schema from 'effect/Schema';
+import * as SchemaAST from 'effect/SchemaAST';
+
+import { type ObjectId } from '@dxos/keys';
+
+import { type ATTR_META, type ObjectMeta } from './meta';
+
+/**
+ * Marker interface for object with an `id`.
+ */
+export interface HasId {
+  readonly id: ObjectId;
+}
+
+/**
+ * Object that has an associated typename.
+ * The typename is retrievable using {@link getTypename}.
+ * The object can be used with {@link isInstanceOf} to check if it is an instance of a schema.
+ */
+export type HasTypename = {};
+
+/**
+ * Base type for all data objects (reactive, ECHO, and other raw objects).
+ * NOTE: This describes the base type for all database objects.
+ * It is stricter than `T extends {}` or `T extends object`.
+ */
+// TODO(burdon): Make internal.
+// TODO(burdon): Prefer Record<string, unknown>.
+// TODO(burdon): Exclude "id", etc. from keys.
+export type AnyProperties = Record<string, any>;
+
+/**
+ * Canonical type for all ECHO objects.
+ */
+export interface AnyEchoObject extends HasId, HasTypename {}
+
+// TODO(dmaretskyi): Remove; this type effectively disables type safety due to `any`.
+export type WithId<T extends AnyProperties = AnyProperties> = T & HasId;
+
+export type ExcludeId<T extends AnyProperties> = Omit<T, 'id'>;
+
+export type PropertyKey<T extends AnyProperties> = Extract<keyof ExcludeId<T>, string>;
+
+// TODO(dmaretskyi): Remove. This should be using the symbol type.
+export type WithMeta = { [ATTR_META]?: ObjectMeta };
+
+/**
+ * The raw object should not include the ECHO id, but may include metadata.
+ */
+export const RawObject = <S extends Schema.Schema.AnyNoContext>(
+  schema: S,
+): Schema.Schema<ExcludeId<Schema.Schema.Type<S>> & WithMeta, Schema.Schema.Encoded<S>> => {
+  return Schema.make(SchemaAST.omit(schema.ast, ['id']));
+};

package/src/internal/types/entity.ts

@@ -0,0 +1,23 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Schema from 'effect/Schema';
+
+/**
+ * Entity kind.
+ */
+const EntityKindId = Symbol.for('@dxos/echo/EntityKind');
+
+/**
+ * Kinds of entities stored in ECHO: objects and relations.
+ */
+export enum EntityKind {
+  Object = 'object',
+  Relation = 'relation',
+}
+
+export const EntityKindSchema = Schema.Enums(EntityKind);
+
+export const KindId: unique symbol = EntityKindId as any;
+export type KindId = typeof KindId;