@dxos/echo 0.8.4-main.84f28bd → 0.8.4-main.937b3ca

package/src/internal/README.md

@@ -0,0 +1,102 @@
+## Refactor
+
+1. Remove access from outisde of @dxos/echo-db
+
+- [x] completely restructure @dxos/echo src/internal
+- [x] Remove import "." and ".."! (create lint rule).
+- [x] Unify FOUR different nests of test schema.
+- [x] Remove @deprecated from internal methods and mark @internal (e.g., getSchemaDXN).
+  - NOTE: Internal methods should not use the import \* from Obj/Type APIs.
+- [x] import ObjectId => @dxos/keys
+- [x] Entity.Any = Obj.Any | Relation.Any
+  - NOTE: Relation does not extend (in not polymorphic with) Obj.
+- [x] import LabelAnnotation => Annotation.LabelAnnotation
+- [x] import Expando => @dxos/schema (Expando.Expando)
+- [x] live => Obj.make
+- [x] Rename live => makeObject
+- [x] Rename {EchoObject, EchoRelation} => {EchoObjecSchema, EchoRelationSchema}
+- [x] Rename AnyProperties => AnyProperties
+- [x] Created Entity.Any (=> AnyEchoObject).
+- [x] TypeFormat => TypeFormat
+- [x] JsonSchemaType defs
+- [x] Fix database.add() input/output types and search for "Obj.Any = db.add" (also QueryResult types).
+- [x] TODO(burdon): FIX!!!
+- [x] Fix failing tests.
+- [x] Reconcile types/version with entities/model/version
+- [x] Expando moved to @dxos/schema
+- [x] Narrow QueryResult and match Schema and Object generics.
+
+2. Clean-up
+
+- [x] QueryResult namespace
+- [x] SchemaRegistry interface
+- [x] Schema registry should return Type.Entity.Any instead of Schema.Schema.AnyNoContext.
+- [x] Hypergraph interface
+- [x] Obj.getDatabase
+- [x] SpaceAction.AddObject target should be a db not a space.
+- [x] Add Relation.MakeProps
+- [x] Add Obj.Unknown and Relation.Unknown to match Entity.Unknown.
+- [x] TODO(wittjosiah): Should be Type.obj<...> or equivalent.
+- [x] TODO(wittjosiah): Find a simpler way to define this type.
+- [x] Ref.Array.targets doesn't satisfy Obj.Any because it uses AnyEchoObject.
+- [x] Remove echo-db/AnyLiveObject<T> => Obj.Obj<T>
+- [x] Rename AnyEchoObject => AnyEntity
+- [x] Remove WithId => AnyEntity
+- [x] Remove WithMeta => AnyEntity
+- [x] Obj.Any => Obj.Unknown
+- [x] Relation.Any => Relation.Unknown
+- [x] Ref.Any => Ref.Unknown
+- [x] Obj.AnyProps => Obj.Any
+- [x] Factor Expando out of @dxos/echo (moved to @dxos/schema)
+- [x] Obj.instanceOf works with Expando (tested in @dxos/schema)
+- [x] Obj.Snapshot should be same shape as Obj.Obj but with a different brand
+- [x] Mutators should only work on object after it is made mutable
+
+- [ ] Review usage of Obj.Any, see if it could be stricter
+- [ ] Type.Obj should validate using the echo object brand
+- [ ] Mutable could be a branded type to fix "NOTE: TypeScript's structural typing allows readonly objects to be passed to `Mutable<T>`"
+- [ ] space properties, queues, messagins preventing getSpace from being removed
+- [ ] Schema registry should use Query.Query.
+- [ ] Move EchoSchemaRegistry into hypergraph
+- [ ] Narrow QueryResult and match Schema and Object generics.
+- [ ] BaseSchema
+- [ ] Directly import JSONPath, etc. from @dxos/effect.
+- [ ] Promote parts of src/internal/ref to Ref.ts
+- [ ] JsonPath, JsonProp, getValue, setValue => Json.Path?
+- [ ] DISCUSS: Standradize $ suffix to disambuguate imports (GPT recommended).
+
+3. Audit usage from @dxos/echo-db
+
+- [ ] Ability to extract Struct from Type.Obj
+- [ ] created/updated system props (const { created } = Obj.getTimestamps)?
+- [ ] QueryFn, QueryOptions => Database
+- [ ] Datatbase.query() options?
+- [ ] Expando type is used as a fallback in many plance.
+- [ ] Standardize '@automerge/automerge' imports (A vs. next).
+
+## NOTES
+
+- Marking types as @internal (even for unexported types) erases type information.
+
+## 0.9.0
+
+- [ ] Database, Queue, Space, Type, namespaces.
+- [ ] Support class variant for types.
+- [ ] Support defaults.
+- [ ] Support effect Date, Timestamp formats.
+- [ ] Metadata for created, updated timestamps.
+- [ ] Effect team design review.
+- [ ] @category annotations to group types in the API.
+- [ ] TSdoc and LLM training for function generation.
+- [ ] Rename TypeFormat => Primitive?
+- [ ] Don't re-export effect?
+
+## Issues
+
+- [ ] Build error when changing from TypedObject using Ref$: Type 'Task' does not satisfy the constraint 'WithId' (see echo-schema/testing/schema.ts)
+- [ ] Reconcile all schema variants (Base, Immutable, TypedObject, EchoObject, etc.)
+- [ ] Consolidate getters (getType, getSchema, getTypename, getSchemaTypename, etc.)
+- [ ] ReactiveObject should specify id property? Reconcile AnyProperties, ReactiveObject, HasId, WithId, etc.
+- [ ] Can we us S.is(MyType) to detect objects with our types system? (Branding?)
+- [ ] Type.Expando doesn't work with AtomQuery result type (have to use Obj.Any instead).
+- [ ] Obj.Any doesn't work with Obj.change callback types (have to use `any` for the mutable parameter).

package/src/internal/annotations/annotations.test.ts

@@ -0,0 +1,96 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Schema from 'effect/Schema';
+import { describe, test } from 'vitest';
+
+import { EchoObjectSchema } from '../entities';
+
+import { LabelAnnotation, TypenameSchema, VersionSchema, getLabelWithSchema } from './annotations';
+
+// TODO(dmaretskyi): Use one of the testing schemas.
+const TestObject = Schema.Struct({
+  name: Schema.optional(Schema.String),
+  fallbackName: Schema.optional(Schema.String),
+  other: Schema.String,
+}).pipe(LabelAnnotation.set(['name', 'fallbackName']));
+
+type TestObject = Schema.Schema.Type<typeof TestObject>;
+
+const TestEchoSchema = TestObject.pipe(
+  EchoObjectSchema({
+    typename: 'dxos.org/type/Test',
+    version: '0.1.0',
+  }),
+);
+
+type TestEchoSchema = Schema.Schema.Type<typeof TestEchoSchema>;
+
+describe('annotations', () => {
+  describe('Typename', () => {
+    test('should validate typename', ({ expect }) => {
+      // Valid.
+      expect(TypenameSchema.make('dxos.org/type/foo')).to.exist;
+      expect(TypenameSchema.make('dxos.org/type/foo-bar')).to.exist;
+      expect(TypenameSchema.make('dxos.org/type/foo_bar')).to.exist;
+
+      // Invalid.
+      expect(() => TypenameSchema.make('dxn:dxos.org')).to.throw();
+      expect(() => TypenameSchema.make('2dxos.org')).to.throw();
+      expect(() => TypenameSchema.make('dxos org')).to.throw();
+    });
+
+    test('should validate version', ({ expect }) => {
+      // Valid.
+      expect(VersionSchema.make('0.1.0')).to.exist;
+
+      // Invalid.
+      expect(() => VersionSchema.make('0.1.x')).to.throw();
+      expect(() => VersionSchema.make('0.1.0-alpha')).to.throw();
+    });
+  });
+
+  describe('getLabelWithSchema', () => {
+    test('should return first available label value', ({ expect }) => {
+      const obj: TestObject = {
+        name: 'Primary Name',
+        fallbackName: 'Fallback Name',
+        other: 'Other',
+      };
+
+      expect(getLabelWithSchema(TestObject, obj)).toEqual('Primary Name');
+    });
+
+    test('should fallback to second path if first is undefined', ({ expect }) => {
+      const obj: TestObject = {
+        name: undefined,
+        fallbackName: 'Fallback Name',
+        other: 'Other',
+      };
+
+      expect(getLabelWithSchema(TestObject, obj)).toEqual('Fallback Name');
+    });
+
+    test('should return undefined if no label paths resolve', ({ expect }) => {
+      const obj: TestObject = {
+        name: undefined,
+        fallbackName: undefined,
+        other: 'Other',
+      };
+
+      expect(getLabelWithSchema(TestObject, obj)).toBeUndefined();
+    });
+
+    test('should return label from echo object', ({ expect }) => {
+      const obj: TestEchoSchema = {
+        id: 'test',
+        name: 'Primary Name',
+        fallbackName: 'Fallback Name',
+        other: 'Other',
+      };
+
+      expect(getLabelWithSchema(TestEchoSchema, obj)).toEqual('Primary Name');
+    });
+  });
+});

package/src/internal/annotations/annotations.ts

@@ -0,0 +1,460 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import * as Function from 'effect/Function';
+import * as Option from 'effect/Option';
+import * as Schema from 'effect/Schema';
+import * as SchemaAST from 'effect/SchemaAST';
+
+import { raise } from '@dxos/debug';
+import { type JsonPath, getField } from '@dxos/effect';
+import { assertArgument, invariant } from '@dxos/invariant';
+import { DXN } from '@dxos/keys';
+import { type Primitive } from '@dxos/util';
+
+import { type AnyProperties, EntityKind, TypeId, getSchema } from '../types';
+
+import { createAnnotationHelper } from './util';
+
+/**
+ * @internal
+ */
+export const FIELD_PATH_ANNOTATION = 'path';
+
+/**
+ * Sets the path for the field.
+ * @param path Data source path in the json path format. This is the field path in the source object.
+ */
+// TODO(burdon): Field, vs. path vs. property.
+export const FieldPath = (path: string) => PropertyMeta(FIELD_PATH_ANNOTATION, path);
+
+//
+// Type
+//
+
+/**
+ * ECHO identifier (for a stored schema).
+ * Must be a `dxn:echo:` URI.
+ */
+export const TypeIdentifierAnnotationId = Symbol.for('@dxos/schema/annotation/TypeIdentifier');
+
+export const getTypeIdentifierAnnotation = (schema: Schema.Schema.All) =>
+  Function.flow(
+    SchemaAST.getAnnotation<string>(TypeIdentifierAnnotationId),
+    Option.getOrElse(() => undefined),
+  )(schema.ast);
+
+/**
+ * @returns DXN of the schema.
+ *
+ * For non-stored schema returns `dxn:type:`.
+ * For stored schema returns `dxn:echo:`.
+ */
+export const getSchemaDXN = (schema: Schema.Schema.All): DXN | undefined => {
+  assertArgument(Schema.isSchema(schema), 'schema', 'invalid schema');
+  const id = getTypeIdentifierAnnotation(schema);
+  if (id) {
+    return DXN.parse(id);
+  }
+
+  // TODO(dmaretskyi): Add support for dynamic schema.
+  const objectAnnotation = getTypeAnnotation(schema);
+  if (!objectAnnotation) {
+    return undefined;
+  }
+
+  return DXN.fromTypenameAndVersion(objectAnnotation.typename, objectAnnotation.version);
+};
+
+/**
+ * @param input schema or a typename string.
+ * @return type DXN.
+ */
+export const getTypeDXNFromSpecifier = (input: Schema.Schema.All | string): DXN => {
+  if (Schema.isSchema(input)) {
+    return getSchemaDXN(input) ?? raise(new TypeError('Schema has no DXN'));
+  } else {
+    assertArgument(typeof input === 'string', 'input');
+    assertArgument(!input.startsWith('dxn:'), 'input');
+    return DXN.fromTypename(input);
+  }
+};
+
+//
+// TypeAnnotation
+//
+
+/**
+ * Fully qualified globally unique typename.
+ * Example: `dxos.org/type/Message`
+ */
+// TODO(burdon): Reconcile with short DXN format.
+// TODO(burdon): Change "/type" => "/schema" throughout.
+export const TypenameSchema = Schema.String.pipe(Schema.pattern(/^[a-zA-Z]\w+\.[a-zA-Z]\w{1,}\/[\w/_-]+$/)).annotations(
+  {
+    description: 'Fully qualified globally unique typename',
+    example: 'dxos.org/type/Message',
+  },
+);
+
+/**
+ * Semantic version format: `major.minor.patch`
+ * Example: `1.0.0`
+ */
+export const VersionSchema = Schema.String.pipe(Schema.pattern(/^\d+.\d+.\d+$/)).annotations({
+  description: 'Semantic version format: `major.minor.patch`',
+  example: '1.0.0',
+});
+
+export const TypeMeta = Schema.Struct({
+  typename: TypenameSchema,
+  version: VersionSchema,
+});
+
+export interface TypeMeta extends Schema.Schema.Type<typeof TypeMeta> {}
+
+/**
+ * Entity type.
+ */
+export const TypeAnnotationId = Symbol.for('@dxos/schema/annotation/Type');
+
+/**
+ * Payload stored under {@link TypeAnnotationId}.
+ */
+export const TypeAnnotation = Schema.extend(
+  TypeMeta,
+  Schema.Struct({
+    kind: Schema.Enums(EntityKind),
+
+    /**
+     * If this is a relation, the schema of the source object.
+     * Must be present if entity kind is {@link EntityKind.Relation}.
+     */
+    sourceSchema: Schema.optional(DXN.Schema),
+
+    /**
+     * If this is a relation, the schema of the target object.
+     * Must be present if entity kind is {@link EntityKind.Relation}.
+     */
+    targetSchema: Schema.optional(DXN.Schema),
+  }),
+);
+
+export interface TypeAnnotation extends Schema.Schema.Type<typeof TypeAnnotation> {}
+
+/**
+ * @returns {@link TypeAnnotation} from a schema.
+ * Schema must have been created with {@link TypedObject} or {@link TypedLink} or manually assigned an appropriate annotation.
+ */
+export const getTypeAnnotation = (schema: Schema.Schema.All): TypeAnnotation | undefined => {
+  assertArgument(schema != null && schema.ast != null, 'schema', 'invalid schema');
+  return Function.flow(
+    SchemaAST.getAnnotation<TypeAnnotation>(TypeAnnotationId),
+    Option.getOrElse(() => undefined),
+  )(schema.ast);
+};
+
+/**
+ * @returns {@link EntityKind} from a schema.
+ */
+export const getEntityKind = (schema: Schema.Schema.All): EntityKind | undefined => getTypeAnnotation(schema)?.kind;
+
+/**
+ * @internal
+ * @returns Schema typename (without dxn: prefix or version number).
+ */
+export const getSchemaTypename = (schema: Schema.Schema.All): string | undefined => getTypeAnnotation(schema)?.typename;
+
+/**
+ * @internal
+ * @returns Schema version in semver format.
+ */
+export const getSchemaVersion = (schema: Schema.Schema.All): string | undefined => getTypeAnnotation(schema)?.version;
+
+/**
+ * Gets the typename of the object without the version.
+ * Returns only the name portion, not the DXN.
+ * @example "example.org/type/Contact"
+ *
+ * @internal (use Obj.getTypename)
+ */
+export const getTypename = (obj: AnyProperties): string | undefined => {
+  const schema = getSchema(obj);
+  if (schema != null) {
+    // Try to extract typename from DXN.
+    return getSchemaTypename(schema);
+  } else {
+    const type = getTypeDXN(obj);
+    return type?.asTypeDXN()?.type;
+  }
+};
+
+/**
+ * @internal (use Type.setTypename)
+ */
+// TODO(dmaretskyi): Rename setTypeDXN.
+export const setTypename = (obj: any, typename: DXN): void => {
+  invariant(typename instanceof DXN, 'Invalid type.');
+  Object.defineProperty(obj, TypeId, {
+    value: typename,
+    writable: false,
+    enumerable: false,
+    configurable: false,
+  });
+};
+
+/**
+ * @returns Object type as {@link DXN}.
+ * @returns undefined if the object doesn't have a type.
+ * @example `dxn:example.com/type/Person:1.0.0`
+ *
+ * @internal (use Obj.getTypeDXN)
+ */
+// TODO(burdon): Narrow type.
+export const getTypeDXN = (obj: AnyProperties): DXN | undefined => {
+  if (!obj) {
+    return undefined;
+  }
+
+  const type = (obj as any)[TypeId];
+  if (!type) {
+    return undefined;
+  }
+
+  invariant(type instanceof DXN, 'Invalid object.');
+  return type;
+};
+
+/**
+ * Checks if the object is an instance of the schema.
+ * Only typename is compared, the schema version is ignored.
+ *
+ * The following cases are considered to mean that the object is an instance of the schema:
+ *  - Object was created with this exact schema.
+ *  - Object was created with a different version of this schema.
+ *  - Object was created with a different schema (maybe dynamic) that has the same typename.
+ */
+// TODO(burdon): Can we use `Schema.is`?
+export const isInstanceOf = <Schema extends Schema.Schema.AnyNoContext>(
+  schema: Schema,
+  object: any,
+): object is Schema.Schema.Type<Schema> => {
+  if (object == null) {
+    return false;
+  }
+
+  const schemaDXN = getSchemaDXN(schema);
+  if (!schemaDXN) {
+    throw new Error('Schema must have an object annotation.');
+  }
+
+  const type = getTypeDXN(object);
+  if (type && DXN.equals(type, schemaDXN)) {
+    return true;
+  }
+
+  const typename = getTypename(object);
+  if (!typename) {
+    return false;
+  }
+
+  const typeDXN = schemaDXN.asTypeDXN();
+  if (!typeDXN) {
+    return false;
+  }
+
+  return typeDXN.type === typename;
+};
+
+//
+// PropertyMeta
+//
+
+/**
+ * PropertyMeta (metadata for dynamic schema properties).
+ * For user-defined annotations.
+ */
+export const PropertyMetaAnnotationId = Symbol.for('@dxos/schema/annotation/PropertyMeta');
+
+export type PropertyMetaValue = Primitive | Record<string, Primitive> | Primitive[];
+
+export type PropertyMetaAnnotation = {
+  [name: string]: PropertyMetaValue;
+};
+
+// TODO(wittjosiah): Align with other annotations.
+// TODO(wittjosiah): Why is this separate from FormatAnnotation?
+export const PropertyMeta = (name: string, value: PropertyMetaValue) => {
+  return <A, I, R>(self: Schema.Schema<A, I, R>): Schema.Schema<A, I, R> => {
+    const existingMeta = self.ast.annotations[PropertyMetaAnnotationId] as PropertyMetaAnnotation;
+    return self.annotations({
+      [PropertyMetaAnnotationId]: {
+        ...existingMeta,
+        [name]: value,
+      },
+    });
+  };
+};
+
+export const getPropertyMetaAnnotation = <T>(prop: SchemaAST.PropertySignature, name: string) =>
+  Function.pipe(
+    SchemaAST.getAnnotation<PropertyMetaAnnotation>(PropertyMetaAnnotationId)(prop.type),
+    Option.map((meta) => meta[name] as T),
+    Option.getOrElse(() => undefined),
+  );
+
+//
+// Reference
+//
+
+/**
+ * Schema reference.
+ */
+export const ReferenceAnnotationId = Symbol.for('@dxos/schema/annotation/Reference');
+export type ReferenceAnnotationValue = TypeAnnotation;
+export const ReferenceAnnotation = createAnnotationHelper<ReferenceAnnotationValue>(ReferenceAnnotationId);
+
+/**
+ * SchemaMeta.
+ */
+export const SchemaMetaSymbol = Symbol.for('@dxos/schema/SchemaMeta');
+export type SchemaMeta = TypeMeta & { id: string };
+
+/**
+ * Identifies a schema as a schema for a hidden system type.
+ */
+export const SystemTypeAnnotationId = Symbol.for('@dxos/schema/annotation/SystemType');
+export const SystemTypeAnnotation = createAnnotationHelper<boolean>(SystemTypeAnnotationId);
+
+/**
+ * Identifies label property or JSON path expression.
+ * Either a string or an array of strings representing field accessors each matched in priority order.
+ */
+export const LabelAnnotationId = Symbol.for('@dxos/schema/annotation/Label');
+export const LabelAnnotation = createAnnotationHelper<string[]>(LabelAnnotationId);
+
+/**
+ * Returns the label for a given object based on {@link LabelAnnotationId}.
+ * Lower-level version that requires explicit schema parameter.
+ */
+// TODO(burdon): Convert to JsonPath?
+export const getLabelWithSchema = <S extends Schema.Schema.Any>(
+  schema: S,
+  object: Schema.Schema.Type<S>,
+): string | undefined => {
+  const annotation = LabelAnnotation.get(schema).pipe(Option.getOrElse(() => ['name']));
+  for (const accessor of annotation) {
+    assertArgument(
+      typeof accessor === 'string',
+      'accessor',
+      'Label annotation must be a string or an array of strings',
+    );
+    const value = getField(object, accessor as JsonPath);
+    switch (typeof value) {
+      case 'string':
+      case 'number':
+      case 'boolean':
+      case 'bigint':
+      case 'symbol':
+        return value.toString();
+      case 'undefined':
+      case 'object':
+      case 'function':
+        continue;
+    }
+  }
+
+  return undefined;
+};
+
+/**
+ * Sets the label for a given object based on {@link LabelAnnotationId}.
+ * Lower-level version that requires explicit schema parameter.
+ */
+export const setLabelWithSchema = <S extends Schema.Schema.Any>(
+  schema: S,
+  object: Schema.Schema.Type<S>,
+  label: string,
+) => {
+  const annotation = LabelAnnotation.get(schema).pipe(
+    Option.map((field) => field[0]),
+    Option.getOrElse(() => 'name'),
+  );
+  object[annotation] = label;
+};
+
+/**
+ * Identifies description property or JSON path expression.
+ * A string representing field accessor.
+ */
+export const DescriptionAnnotationId = Symbol.for('@dxos/schema/annotation/Description');
+export const DescriptionAnnotation = createAnnotationHelper<string>(DescriptionAnnotationId);
+
+/**
+ * Returns the description for a given object based on {@link DescriptionAnnotationId}.
+ * Lower-level version that requires explicit schema parameter.
+ */
+// TODO(burdon): Convert to JsonPath?
+export const getDescriptionWithSchema = <S extends Schema.Schema.Any>(
+  schema: S,
+  object: Schema.Schema.Type<S>,
+): string | undefined => {
+  const accessor = DescriptionAnnotation.get(schema).pipe(Option.getOrElse(() => 'description'));
+  assertArgument(typeof accessor === 'string', 'accessor', 'Description annotation must be a string');
+  const value = getField(object, accessor as JsonPath);
+  switch (typeof value) {
+    case 'string':
+    case 'number':
+    case 'boolean':
+    case 'bigint':
+    case 'symbol':
+      return value.toString();
+    case 'undefined':
+    case 'object':
+    case 'function':
+    default:
+      return undefined;
+  }
+};
+
+/**
+ * Sets the description for a given object based on {@link DescriptionAnnotationId}.
+ * Lower-level version that requires explicit schema parameter.
+ */
+export const setDescriptionWithSchema = <S extends Schema.Schema.Any>(
+  schema: S,
+  object: Schema.Schema.Type<S>,
+  description: string,
+) => {
+  const accessor = DescriptionAnnotation.get(schema).pipe(Option.getOrElse(() => 'description'));
+  object[accessor] = description;
+};
+
+/**
+ * Identifies if a property should be included in a form or not.
+ * By default, all properties are included in forms, so this is opt-out.
+ */
+// TODO(burdon): UI concern.
+export const FormInputAnnotationId = Symbol.for('@dxos/schema/annotation/FormInput');
+export const FormInputAnnotation = createAnnotationHelper<boolean>(FormInputAnnotationId);
+
+/**
+ * Default field to be used on referenced schema to lookup the value.
+ */
+export const FieldLookupAnnotationId = Symbol.for('@dxos/schema/annotation/FieldLookup');
+
+/**
+ * Generate test data.
+ */
+export const GeneratorAnnotationId = Symbol.for('@dxos/schema/annotation/Generator');
+
+export type GeneratorAnnotationValue =
+  | string
+  | {
+      generator: string;
+      args?: any[];
+      probability?: number;
+    };
+
+export const GeneratorAnnotation = createAnnotationHelper<GeneratorAnnotationValue>(GeneratorAnnotationId);

package/src/internal/annotations/index.ts

@@ -0,0 +1,6 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+export * from './annotations';
+export * from './util';