@dxos/echo 0.8.4-main.a4bbb77 → 0.8.4-main.abd8ff62ef

package/src/internal/Type/echo-schema.ts

@@ -0,0 +1,423 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import * as Schema from 'effect/Schema';
+import * as SchemaAST from 'effect/SchemaAST';
+
+import { invariant } from '@dxos/invariant';
+import { type ObjectId } from '@dxos/keys';
+
+import type * as Type from '../../Type';
+import { type SchemaMeta, SchemaMetaSymbol, type TypeAnnotation, getTypeAnnotation } from '../Annotation';
+import { ChangeId } from '../common/proxy';
+import { EntityKind, SchemaKindId } from '../common/types';
+import { type JsonSchemaType, toEffectSchema, toJsonSchema } from '../JsonSchema';
+import { type TypedObject, type TypedObjectPrototype, getSnapshot } from '../Obj';
+import {
+  addFieldsToSchema,
+  removeFieldsFromSchema,
+  setTypenameInSchema,
+  updateFieldNameInSchema,
+  updateFieldsInSchema,
+} from './manipulation';
+import { PersistentSchema } from './persistent-schema';
+
+/**
+ * Base schema type.
+ */
+// TODO(burdon): Merge with ImmutableSchema.
+export interface BaseSchema<A = any, I = any> extends TypedObject<A, I> {
+  // TODO(burdon): Different from mutable?
+  get readonly(): boolean;
+  // TODO(burdon): Change to external function.
+  get mutable(): EchoSchema<A, I>;
+  get snapshot(): Schema.Schema<A, I>;
+  get jsonSchema(): JsonSchemaType;
+}
+
+/**
+ * Immutable schema type.
+ * @deprecated Use `Schema.Schema.AnyNoContext` instead.
+ */
+// TODO(burdon): Common abstract base class?
+export class ImmutableSchema<A = any, I = any> implements BaseSchema<A, I> {
+  private readonly _objectAnnotation: TypeAnnotation;
+  constructor(private readonly _schema: Schema.Schema<A, I>) {
+    this._objectAnnotation = getTypeAnnotation(this._schema)!;
+    invariant(this._objectAnnotation);
+  }
+
+  //
+  // Effect Schema (push to abstract base class).
+  //
+
+  public get [Schema.TypeId]() {
+    return schemaVariance;
+  }
+
+  public get Type() {
+    return this._schema.Type;
+  }
+
+  public get Encoded() {
+    return this._schema.Encoded;
+  }
+
+  public get Context() {
+    return this._schema.Context;
+  }
+
+  public get ast(): SchemaAST.AST {
+    return this._schema.ast;
+  }
+
+  public get annotations() {
+    return this._schema.annotations;
+  }
+
+  public get pipe() {
+    return this._schema.pipe;
+  }
+
+  //
+  // TypedObject
+  //
+
+  get kind(): EntityKind {
+    return this._objectAnnotation.kind;
+  }
+
+  get typename(): string {
+    return this._objectAnnotation.typename;
+  }
+
+  get version(): string {
+    return this._objectAnnotation.version;
+  }
+
+  //
+  // BaseSchema
+  //
+
+  get readonly(): boolean {
+    return true;
+  }
+
+  get snapshot(): Schema.Schema.AnyNoContext {
+    return this._schema;
+  }
+
+  // TODO(burdon): Change from getter since this is expensive.
+  get jsonSchema(): JsonSchemaType {
+    return toJsonSchema(this._schema);
+  }
+
+  get mutable(): EchoSchema {
+    throw new Error('Schema is readonly.');
+  }
+}
+
+/**
+ * Defines an effect-schema for the `EchoSchema` type.
+ *
+ * This is here so that `EchoSchema` class can be used as a part of another schema definition (e.g., `ref(EchoSchema)`).
+ */
+const EchoSchemaConstructor = (): TypedObjectPrototype => {
+  /**
+   * Return class definition satisfying Schema.Schema.
+   */
+  return class {
+    private static get _schema() {
+      // The field is DynamicEchoSchema in runtime, but is serialized as PersisentSchema in automerge.
+      return Schema.Union(PersistentSchema, Schema.instanceOf(EchoSchema)).annotations(
+        PersistentSchema.ast.annotations,
+      );
+    }
+
+    static readonly [Schema.TypeId] = schemaVariance;
+
+    static get ast() {
+      const schema = this._schema;
+      return schema.ast;
+    }
+
+    static get annotations() {
+      const schema = this._schema;
+      return schema.annotations.bind(schema);
+    }
+
+    static get pipe() {
+      const schema = this._schema;
+      return schema.pipe.bind(schema);
+    }
+  } as any;
+};
+
+export const isMutable = (schema: Schema.Schema.AnyNoContext): schema is EchoSchema => {
+  return schema instanceof EchoSchema;
+};
+
+// NOTE: Keep in this file.
+const schemaVariance = {
+  _A: (_: any) => _,
+  _I: (_: any) => _,
+  _R: (_: never) => _,
+};
+
+/**
+ * Represents a schema that is persisted in the ECHO database.
+ * Schema can me mutable or readonly (specified by the {@link EchoSchema.readonly} field).
+ *
+ * Schema that can be modified at runtime via the API.
+ * Is an instance of effect-schema (`Schema.Schema.AnyNoContext`) so it can be used in the same way as a regular schema.
+ * IMPORTANT: The schema AST will change reactively when the schema is updated, including synced updates from remote peers.
+ *
+ * The class constructor is a schema instance itself, and can be used in the echo object definitions:
+ *
+ * @example
+ * ```ts
+ * export class TableType extends Schema.Struct({...}).pipe(Type.object({ typename: 'com.example.type.table', version: '0.1.0' })){
+ *   title: Schema.String,
+ *   schema: Schema.optional(ref(EchoSchema)),
+ *   props: Schema.mutable(S.Array(TablePropSchema)),
+ * }) {}
+ * ```
+ *
+ * The ECHO API will translate any references to PersistentSchema objects to be resolved as EchoSchema objects.
+ */
+export class EchoSchema<A = any, I = any> extends EchoSchemaConstructor() implements BaseSchema<A, I> {
+  private _schema: Schema.Schema.AnyNoContext | undefined;
+  private _isDirty = true;
+
+  /**
+   * Schema kind key that marks this as an ECHO schema.
+   * Makes EchoSchema satisfy the Type.AnyObj type.
+   */
+  readonly [SchemaKindId]: EntityKind.Object = EntityKind.Object;
+
+  constructor(private readonly _persistentSchema: PersistentSchema) {
+    super();
+  }
+
+  //
+  // Effect Schema (push to abstract base class).
+  //
+
+  public get [Schema.TypeId]() {
+    return schemaVariance;
+  }
+
+  public get Type() {
+    return this._persistentSchema as A;
+  }
+
+  public get Encoded() {
+    return this._persistentSchema as I;
+  }
+
+  public get Context() {
+    const schema = this._getSchema();
+    return schema.Context;
+  }
+
+  public get ast() {
+    const schema = this._getSchema();
+    return schema.ast;
+  }
+
+  public get annotations() {
+    const schema = this._getSchema();
+    return schema.annotations.bind(schema);
+  }
+
+  public get pipe(): Schema.Schema.AnyNoContext['pipe'] {
+    const schema = this._getSchema();
+    return schema.pipe.bind(schema);
+  }
+
+  //
+  // BaseSchema
+  //
+
+  public get kind(): EntityKind.Object {
+    // Dynamic schemas are always object schemas.
+    return EntityKind.Object;
+  }
+
+  public get typename(): string {
+    return this._persistentSchema.typename;
+  }
+
+  public get version(): string {
+    return this._persistentSchema.version;
+  }
+
+  public get readonly(): boolean {
+    return false;
+  }
+
+  /**
+   * Returns an immutable schema snapshot of the current state of the schema.
+   */
+  public get snapshot(): Schema.Schema.AnyNoContext {
+    return this._getSchema();
+  }
+
+  /**
+   * Returns the JSON schema for the schema.
+   */
+  public get jsonSchema(): JsonSchemaType {
+    return this._persistentSchema.jsonSchema;
+  }
+
+  /**
+   * Returns a mutable schema.
+   */
+  public get mutable(): EchoSchema {
+    invariant(!this.readonly, 'Schema is not mutable');
+    return this;
+  }
+
+  //
+  // Mutable Schema
+  //
+
+  /**
+   * Id of the ECHO object containing the schema.
+   */
+  public get id(): ObjectId {
+    return this._persistentSchema.id;
+  }
+
+  /**
+   * Short name of the schema.
+   */
+  public get name(): string | undefined {
+    return this._persistentSchema.name;
+  }
+
+  public get [SchemaMetaSymbol](): SchemaMeta {
+    return { id: this.id, typename: this.typename, version: this._persistentSchema.version };
+  }
+
+  /**
+   * Reference to the underlying persistent schema object.
+   */
+  public get persistentSchema(): Type.PersistentType {
+    return this._persistentSchema as Type.PersistentType;
+  }
+
+  public getProperties(): SchemaAST.PropertySignature[] {
+    const ast = this._getSchema().ast;
+    invariant(SchemaAST.isTypeLiteral(ast));
+    return [...ast.propertySignatures].filter((p) => p.name !== 'id').map(unwrapOptionality);
+  }
+
+  //
+  // Mutation methods.
+  // TODO(burdon): Create separate interface for dynamic schema.
+  // TODO(burdon): Deprecate direct manipulation? Use JSONSchema directly.
+  //
+
+  /**
+   * @throws Error if the schema is readonly.
+   */
+  public updateTypename(typename: string): void {
+    const updated = setTypenameInSchema(this._getSchema(), typename);
+    this._change((schema) => {
+      schema.typename = typename;
+      schema.jsonSchema = toJsonSchema(updated);
+    });
+  }
+
+  /**
+   * @throws Error if the schema is readonly.
+   */
+  public addFields(fields: Schema.Struct.Fields): void {
+    const extended = addFieldsToSchema(this._getSchema(), fields);
+    this._change((schema) => {
+      schema.jsonSchema = toJsonSchema(extended);
+    });
+  }
+
+  /**
+   * @throws Error if the schema is readonly.
+   */
+  public updateFields(fields: Schema.Struct.Fields): void {
+    const updated = updateFieldsInSchema(this._getSchema(), fields);
+    this._change((schema) => {
+      schema.jsonSchema = toJsonSchema(updated);
+    });
+  }
+
+  /**
+   * @throws Error if the schema is readonly.
+   */
+  public updateFieldPropertyName({ before, after }: { before: PropertyKey; after: PropertyKey }): void {
+    const renamed = updateFieldNameInSchema(this._getSchema(), { before, after });
+    this._change((schema) => {
+      schema.jsonSchema = toJsonSchema(renamed);
+    });
+  }
+
+  /**
+   * @throws Error if the schema is readonly.
+   */
+  public removeFields(fieldNames: string[]): void {
+    const removed = removeFieldsFromSchema(this._getSchema(), fieldNames);
+    this._change((schema) => {
+      schema.jsonSchema = toJsonSchema(removed);
+    });
+  }
+
+  /**
+   * Wrapper for Obj.update that handles the change context for the persistent schema.
+   */
+  private _change(callback: (schema: PersistentSchema) => void): void {
+    const changeFn = (this._persistentSchema as any)[ChangeId];
+    if (changeFn) {
+      changeFn(callback);
+    } else {
+      callback(this._persistentSchema);
+    }
+  }
+
+  //
+  // Internals
+  //
+
+  /**
+   * Called by DatabaseSchemaRegistry on update.
+   */
+  _invalidate(): void {
+    this._isDirty = true;
+  }
+
+  /**
+   * Rebuilds this schema if it is dirty.
+   */
+  _rebuild(): void {
+    if (this._isDirty || this._schema == null) {
+      this._schema = toEffectSchema(getSnapshot(this._persistentSchema).jsonSchema);
+      this._isDirty = false;
+    }
+  }
+
+  _getSchema(): Schema.Schema.AnyNoContext {
+    this._rebuild();
+    return this._schema!;
+  }
+}
+
+// TODO(burdon): Move to effect.
+const unwrapOptionality = (property: SchemaAST.PropertySignature): SchemaAST.PropertySignature => {
+  if (!SchemaAST.isUnion(property.type)) {
+    return property;
+  }
+
+  return {
+    ...property,
+    type: property.type.types.find((type) => !SchemaAST.isUndefinedKeyword(type))!,
+  } as any;
+};

package/src/internal/Type/index.ts

@@ -0,0 +1,7 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+export * from './echo-schema';
+export * from './manipulation';
+export * from './persistent-schema';

package/src/internal/Type/manipulation.ts

@@ -0,0 +1,92 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import * as Schema from 'effect/Schema';
+import * as SchemaAST from 'effect/SchemaAST';
+
+import { invariant } from '@dxos/invariant';
+import { DXN } from '@dxos/keys';
+
+import { type TypeAnnotation, TypeAnnotationId, TypeIdentifierAnnotationId } from '../Annotation';
+
+// TODO(ZaymonFC): Do this one at a time. This might be dangerous.
+export const addFieldsToSchema = (
+  schema: Schema.Schema.AnyNoContext,
+  fields: Schema.Struct.Fields,
+): Schema.Schema.AnyNoContext => {
+  const schemaExtension = Schema.partial(Schema.Struct(fields));
+  return Schema.extend(schema, schemaExtension).annotations(
+    schema.ast.annotations,
+  ) as any as Schema.Schema.AnyNoContext;
+};
+
+export const updateFieldsInSchema = (
+  schema: Schema.Schema.AnyNoContext,
+  fields: Schema.Struct.Fields,
+): Schema.Schema.AnyNoContext => {
+  const ast = schema.ast as SchemaAST.TypeLiteral;
+  invariant(SchemaAST.isTypeLiteral(ast));
+
+  const updatedProperties = [...ast.propertySignatures];
+  const propertiesToUpdate = (Schema.partial(Schema.Struct(fields)).ast as SchemaAST.TypeLiteral).propertySignatures;
+  for (const property of propertiesToUpdate) {
+    const index = updatedProperties.findIndex((p) => p.name === property.name);
+    if (index !== -1) {
+      updatedProperties[index] = property;
+    } else {
+      updatedProperties.push(property);
+    }
+  }
+
+  return Schema.make(new SchemaAST.TypeLiteral(updatedProperties, ast.indexSignatures, ast.annotations));
+};
+
+export const removeFieldsFromSchema = (
+  schema: Schema.Schema.AnyNoContext,
+  fieldNames: string[],
+): Schema.Schema.AnyNoContext => {
+  return Schema.make(SchemaAST.omit(schema.ast, fieldNames)).annotations(schema.ast.annotations);
+};
+
+export const updateFieldNameInSchema = (
+  schema: Schema.Schema.AnyNoContext,
+  { before, after }: { before: PropertyKey; after: PropertyKey },
+): Schema.Schema.AnyNoContext => {
+  const ast = schema.ast as SchemaAST.TypeLiteral;
+  invariant(SchemaAST.isTypeLiteral(ast));
+
+  return Schema.make(
+    new SchemaAST.TypeLiteral(
+      ast.propertySignatures.map((p) =>
+        p.name === before
+          ? new SchemaAST.PropertySignature(after, p.type, p.isOptional, p.isReadonly, p.annotations)
+          : p,
+      ),
+      ast.indexSignatures,
+      ast.annotations,
+    ),
+  );
+};
+
+export const setTypenameInSchema = (
+  schema: Schema.Schema.AnyNoContext,
+  typename: string,
+): Schema.Schema.AnyNoContext => {
+  const existingAnnotation = schema.ast.annotations[TypeAnnotationId] as TypeAnnotation;
+  invariant(existingAnnotation, `Missing ${String(TypeAnnotationId)}`);
+
+  return schema.annotations({
+    ...schema.ast.annotations,
+    [TypeAnnotationId]: {
+      kind: existingAnnotation.kind,
+      typename,
+      version: existingAnnotation.version,
+    } satisfies TypeAnnotation,
+    [SchemaAST.JSONSchemaAnnotationId]: {
+      ...(schema.ast.annotations[SchemaAST.JSONSchemaAnnotationId] ?? {}),
+      $id: schema.ast.annotations[TypeIdentifierAnnotationId] ?? DXN.fromTypename(typename).toString(),
+      typename,
+    },
+  });
+};

package/src/internal/Type/persistent-schema.ts

@@ -0,0 +1,33 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import * as Schema from 'effect/Schema';
+
+import { IconAnnotation } from '../Annotation';
+import { LabelAnnotation, TypenameSchema, VersionSchema } from '../Annotation';
+import { EchoObjectSchema } from '../Entity';
+import { JsonSchemaType } from '../JsonSchema';
+
+/**
+ * Persistent representation of a schema.
+ */
+// TODO(burdon): Move.
+export const PersistentSchema = Schema.Struct({
+  name: Schema.optional(Schema.String),
+  typename: TypenameSchema,
+  version: VersionSchema,
+  jsonSchema: JsonSchemaType,
+}).pipe(
+  EchoObjectSchema({
+    typename: 'org.dxos.type.schema',
+    version: '0.1.0',
+  }),
+  LabelAnnotation.set(['name']),
+  IconAnnotation.set({
+    icon: 'ph--database--regular',
+    hue: 'green',
+  }),
+);
+
+export interface PersistentSchema extends Schema.Schema.Type<typeof PersistentSchema> {}

package/src/internal/common/README.md

@@ -0,0 +1,102 @@
+## Refactor
+
+1. Remove access from outisde of @dxos/echo-db
+
+- [x] completely restructure @dxos/echo src/internal
+- [x] Remove import "." and ".."! (create lint rule).
+- [x] Unify FOUR different nests of test schema.
+- [x] Remove @deprecated from internal methods and mark @internal (e.g., getSchemaDXN).
+  - NOTE: Internal methods should not use the import \* from Obj/Type APIs.
+- [x] import ObjectId => @dxos/keys
+- [x] Entity.Any = Obj.Any | Relation.Any
+  - NOTE: Relation does not extend (in not polymorphic with) Obj.
+- [x] import LabelAnnotation => Annotation.LabelAnnotation
+- [x] import Expando => @dxos/schema (Expando.Expando)
+- [x] live => Obj.make
+- [x] Rename live => makeObject
+- [x] Rename {EchoObject, EchoRelation} => {EchoObjecSchema, EchoRelationSchema}
+- [x] Rename AnyProperties => AnyProperties
+- [x] Created Entity.Any (=> AnyEchoObject).
+- [x] TypeFormat => TypeFormat
+- [x] JsonSchemaType defs
+- [x] Fix database.add() input/output types and search for "Obj.Any = db.add" (also QueryResult types).
+- [x] TODO(burdon): FIX!!!
+- [x] Fix failing tests.
+- [x] Reconcile types/version with entities/model/version
+- [x] Expando moved to @dxos/schema
+- [x] Narrow QueryResult and match Schema and Object generics.
+
+2. Clean-up
+
+- [x] QueryResult namespace
+- [x] SchemaRegistry interface
+- [x] Schema registry should return Type.AnyEntity instead of Schema.Schema.AnyNoContext.
+- [x] Hypergraph interface
+- [x] Obj.getDatabase
+- [x] SpaceAction.AddObject target should be a db not a space.
+- [x] Add Relation.MakeProps
+- [x] Add Obj.Unknown and Relation.Unknown to match Entity.Unknown.
+- [x] TODO(wittjosiah): Should be Type.obj<...> or equivalent.
+- [x] TODO(wittjosiah): Find a simpler way to define this type.
+- [x] Ref.Array.targets doesn't satisfy Obj.Any because it uses AnyEchoObject.
+- [x] Remove echo-db/AnyLiveObject<T> => Obj.Obj<T>
+- [x] Rename AnyEchoObject => AnyEntity
+- [x] Remove WithId => AnyEntity
+- [x] Remove WithMeta => AnyEntity
+- [x] Obj.Any => Obj.Unknown
+- [x] Relation.Any => Relation.Unknown
+- [x] Ref.Any => Ref.Unknown
+- [x] Obj.AnyProps => Obj.Any
+- [x] Factor Expando out of @dxos/echo (moved to @dxos/schema)
+- [x] Obj.instanceOf works with Expando (tested in @dxos/schema)
+- [x] Obj.Snapshot should be same shape as Obj.Obj but with a different brand
+- [x] Mutators should only work on object after it is made mutable
+
+- [ ] Review usage of Obj.Any, see if it could be stricter
+- [ ] Type.Obj should validate using the echo object brand
+- [ ] Mutable could be a branded type to fix "NOTE: TypeScript's structural typing allows readonly objects to be passed to `Mutable<T>`"
+- [ ] space properties, queues, messagins preventing getSpace from being removed
+- [ ] Schema registry should use Query.Query.
+- [ ] Move EchoSchemaRegistry into hypergraph
+- [ ] Narrow QueryResult and match Schema and Object generics.
+- [ ] BaseSchema
+- [ ] Directly import JSONPath, etc. from @dxos/effect.
+- [ ] Promote parts of src/internal/ref to Ref.ts
+- [ ] JsonPath, JsonProp, getValue, setValue => Json.Path?
+- [ ] DISCUSS: Standradize $ suffix to disambuguate imports (GPT recommended).
+
+3. Audit usage from @dxos/echo-db
+
+- [ ] Ability to extract Struct from Type.Obj
+- [ ] created/updated system props (const { created } = Obj.getTimestamps)?
+- [ ] QueryFn, QueryOptions => Database
+- [ ] Datatbase.query() options?
+- [ ] Expando type is used as a fallback in many plance.
+- [ ] Standardize '@automerge/automerge' imports (A vs. next).
+
+## NOTES
+
+- Marking types as @internal (even for unexported types) erases type information.
+
+## 0.9.0
+
+- [ ] Database, Queue, Space, Type, namespaces.
+- [ ] Support class variant for types.
+- [ ] Support defaults.
+- [ ] Support effect Date, Timestamp formats.
+- [ ] Metadata for created, updated timestamps.
+- [ ] Effect team design review.
+- [ ] @category annotations to group types in the API.
+- [ ] TSdoc and LLM training for function generation.
+- [ ] Rename TypeFormat => Primitive?
+- [ ] Don't re-export effect?
+
+## Issues
+
+- [ ] Build error when changing from TypedObject using Ref$: Type 'Task' does not satisfy the constraint 'WithId' (see echo-schema/testing/schema.ts)
+- [ ] Reconcile all schema variants (Base, Immutable, TypedObject, EchoObject, etc.)
+- [ ] Consolidate getters (getType, getSchema, getTypename, getSchemaTypename, etc.)
+- [ ] ReactiveObject should specify id property? Reconcile AnyProperties, ReactiveObject, HasId, WithId, etc.
+- [ ] Can we us S.is(MyType) to detect objects with our types system? (Branding?)
+- [ ] Type.Expando doesn't work with AtomQuery result type (have to use Obj.Any instead).
+- [ ] Obj.Any doesn't work with Obj.update callback types (have to use `any` for the mutable parameter).

package/src/internal/common/api/index.ts

@@ -0,0 +1,15 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+/**
+ * Shared API helpers for Obj and Relation modules.
+ * These implementations use generic Entity types.
+ * Obj and Relation modules re-export with strongly-typed signatures.
+ *
+ * TODO(wittjosiah): Could this become the new "internal" where we slowly refactor
+ * existing internal modules into here? This would provide a cleaner boundary
+ * and prevent external packages from using internals directly.
+ */
+
+export * from './meta';