@dxos/echo 0.8.4-main.fd6878d → 0.8.4-staging.60fe92afc8

package/src/internal/Type/compose.test.ts

@@ -0,0 +1,43 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import * as Schema from 'effect/Schema';
+import { describe, test } from 'vitest';
+
+import { DXN } from '@dxos/keys';
+
+import { FieldPath } from '../Annotation';
+import { EchoObjectSchema } from '../Entity';
+import { FormatAnnotation, TypeFormat } from '../Format';
+import { ECHO_ANNOTATIONS_NS_KEY, toJsonSchema } from '../JsonSchema';
+import { composeSchema } from './compose';
+
+describe('schema composition', () => {
+  test('schema composition', ({ expect }) => {
+    const BaseType = Schema.Struct({
+      name: Schema.String,
+      email: Schema.String,
+    }).pipe(EchoObjectSchema(DXN.make('com.example.person', '0.1.0')));
+
+    const OverlaySchema = Schema.Struct({
+      email: Schema.String.pipe(FieldPath('$.email'), FormatAnnotation.set(TypeFormat.Email)),
+    });
+
+    const baseSchema = toJsonSchema(BaseType);
+    const overlaySchema = toJsonSchema(OverlaySchema);
+    const composedSchema = composeSchema(baseSchema, overlaySchema);
+    expect(composedSchema.properties).to.deep.eq({
+      email: {
+        type: 'string',
+        format: TypeFormat.Email,
+        // TODO(dmaretskyi): Should use the new field.
+        [ECHO_ANNOTATIONS_NS_KEY]: {
+          meta: {
+            path: '$.email',
+          },
+        },
+      },
+    });
+  });
+});

package/src/internal/Type/compose.ts

@@ -0,0 +1,41 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import type * as Types from 'effect/Types';
+
+import { invariant } from '@dxos/invariant';
+
+import { type JsonSchemaType } from '../JsonSchema';
+
+/**
+ * Creates a composite schema from the source and projection schemas.
+ */
+// TODO(burdon): Use effect schema projections.
+// TODO(burdon): Can avoid having to call this every time we modify any property on the view?
+export const composeSchema = (
+  source: Types.Mutable<JsonSchemaType>,
+  target: Types.Mutable<JsonSchemaType>,
+): Types.Mutable<JsonSchemaType> => {
+  const result = structuredClone(target);
+  invariant('type' in result && result.type === 'object', 'source schema must be an object');
+  invariant('type' in source && source.type === 'object', 'target schema must be an object');
+
+  for (const prop in result.properties) {
+    const propSchema = source.properties![prop]; // TODO(dmaretskyi): Find by json-path instead.
+    const annotations = (propSchema as JsonSchemaType)?.annotations?.meta;
+    if (annotations) {
+      const resultProp = result.properties[prop] as Record<string, any>;
+      resultProp.annotations ??= {};
+      resultProp.annotations.meta ??= {};
+      for (const key in annotations) {
+        resultProp.annotations.meta[key] ??= {};
+        Object.assign(resultProp.annotations.meta[key], annotations[key], {
+          ...resultProp.annotations.meta[key],
+        });
+      }
+    }
+  }
+
+  return result;
+};

package/src/internal/Type/index.ts

@@ -0,0 +1,6 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+export * from './manipulation';
+export * from './type-schema';

package/src/internal/Type/manipulation.ts

@@ -0,0 +1,67 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import * as Schema from 'effect/Schema';
+import * as SchemaAST from 'effect/SchemaAST';
+
+import { invariant } from '@dxos/invariant';
+
+// TODO(ZaymonFC): Do this one at a time. This might be dangerous.
+export const addFieldsToSchema = (
+  schema: Schema.Schema.AnyNoContext,
+  fields: Schema.Struct.Fields,
+): Schema.Schema.AnyNoContext => {
+  const schemaExtension = Schema.partial(Schema.Struct(fields));
+  return Schema.extend(schema, schemaExtension).annotations(
+    schema.ast.annotations,
+  ) as any as Schema.Schema.AnyNoContext;
+};
+
+export const updateFieldsInSchema = (
+  schema: Schema.Schema.AnyNoContext,
+  fields: Schema.Struct.Fields,
+): Schema.Schema.AnyNoContext => {
+  const ast = schema.ast as SchemaAST.TypeLiteral;
+  invariant(SchemaAST.isTypeLiteral(ast));
+
+  const updatedProperties = [...ast.propertySignatures];
+  const propertiesToUpdate = (Schema.partial(Schema.Struct(fields)).ast as SchemaAST.TypeLiteral).propertySignatures;
+  for (const property of propertiesToUpdate) {
+    const index = updatedProperties.findIndex((p) => p.name === property.name);
+    if (index !== -1) {
+      updatedProperties[index] = property;
+    } else {
+      updatedProperties.push(property);
+    }
+  }
+
+  return Schema.make(new SchemaAST.TypeLiteral(updatedProperties, ast.indexSignatures, ast.annotations));
+};
+
+export const removeFieldsFromSchema = (
+  schema: Schema.Schema.AnyNoContext,
+  fieldNames: string[],
+): Schema.Schema.AnyNoContext => {
+  return Schema.make(SchemaAST.omit(schema.ast, fieldNames)).annotations(schema.ast.annotations);
+};
+
+export const updateFieldNameInSchema = (
+  schema: Schema.Schema.AnyNoContext,
+  { before, after }: { before: PropertyKey; after: PropertyKey },
+): Schema.Schema.AnyNoContext => {
+  const ast = schema.ast as SchemaAST.TypeLiteral;
+  invariant(SchemaAST.isTypeLiteral(ast));
+
+  return Schema.make(
+    new SchemaAST.TypeLiteral(
+      ast.propertySignatures.map((p) =>
+        p.name === before
+          ? new SchemaAST.PropertySignature(after, p.type, p.isOptional, p.isReadonly, p.annotations)
+          : p,
+      ),
+      ast.indexSignatures,
+      ast.annotations,
+    ),
+  );
+};

package/src/internal/Type/type-schema.ts

@@ -0,0 +1,60 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import * as Schema from 'effect/Schema';
+
+import { IconAnnotation, LabelAnnotation } from '../Annotation';
+import { EntityKind, KindId, SchemaKindId, StaticTypeSchemaSlot } from '../common/types';
+import { EchoTypeKindSchema, TypeMetaSchemaDXN } from '../Entity';
+import { JsonSchemaType } from '../JsonSchema';
+
+/**
+ * Raw struct backing {@link TypeSchema}. Exposed only so `TypeSchema`
+ * (the TS type) can derive its data fields via `Schema.Schema.Type<typeof ...>`;
+ * runtime callers should use {@link TypeSchema} (the piped, branded entity).
+ *
+ * `typename` and `version` are NOT data fields — they live in `EntityMeta.key` /
+ * `EntityMeta.version` (the canonical registry-provenance pair, queryable via
+ * `Filter.key(...)`). The same `jsonSchema` payload also embeds them so a
+ * standalone JSON-Schema export remains self-describing, but the schema-registry
+ * reads/writes them through meta.
+ */
+const TypeSchemaStruct = Schema.Struct({
+  name: Schema.optional(Schema.String),
+  jsonSchema: JsonSchemaType,
+});
+
+/**
+ * Persistent representation of a schema.
+ */
+export const TypeSchema = TypeSchemaStruct.pipe(
+  LabelAnnotation.set(['name']),
+  IconAnnotation.set({
+    icon: 'ph--database--regular',
+    hue: 'green',
+  }),
+  EchoTypeKindSchema(TypeMetaSchemaDXN),
+);
+
+/**
+ * Persistent representation of a schema — the runtime shape that
+ * `db.addType(Type.makeObjectFromJsonSchema(...))` produces
+ * and `Filter.type(Type.Type).run()` returns.
+ *
+ * Structurally identical to a static `Type.Type` entity: the entity-handler's
+ * `get` trap exposes `[SchemaKindId]` (derived from `system.kind` and
+ * `data.jsonSchema.entityKind`) and rebuilds `[StaticTypeSchemaSlot]` lazily
+ * from `jsonSchema`, so a persisted instance satisfies the public `Type<A>`
+ * interface without casting.
+ */
+export type TypeSchema = Schema.Schema.Type<typeof TypeSchemaStruct> & {
+  /** Object identifier — injected by `EchoTypeKindSchema` and stamped at construction. */
+  readonly id: string;
+  /** Entity-kind discriminator (object/relation/type) carried on every entity instance. */
+  readonly [KindId]: EntityKind.Type;
+  /** Kind of schema described by this meta-instance — always `EntityKind.Type` for `Type.Type` itself. */
+  readonly [SchemaKindId]: EntityKind.Type;
+  /** Effect Schema rebuilt lazily from `jsonSchema`; satisfies `Type.getSchema(...)` without an extra cast. */
+  readonly [StaticTypeSchemaSlot]: Schema.Schema.AnyNoContext;
+};

package/src/internal/common/README.md

@@ -0,0 +1,102 @@
+## Refactor
+
+1. Remove access from outisde of @dxos/echo-db
+
+- [x] completely restructure @dxos/echo src/internal
+- [x] Remove import "." and ".."! (create lint rule).
+- [x] Unify FOUR different nests of test schema.
+- [x] Remove @deprecated from internal methods and mark @internal (e.g., getSchemaDXN).
+  - NOTE: Internal methods should not use the import \* from Obj/Type APIs.
+- [x] import EntityId => @dxos/keys
+- [x] Entity.Any = Obj.Any | Relation.Any
+  - NOTE: Relation does not extend (in not polymorphic with) Obj.
+- [x] import LabelAnnotation => Annotation.LabelAnnotation
+- [x] import Expando => @dxos/schema (Expando.Expando)
+- [x] live => Obj.make
+- [x] Rename live => makeObject
+- [x] Rename {EchoObject, EchoRelation} => {EchoObjecSchema, EchoRelationSchema}
+- [x] Rename AnyProperties => AnyProperties
+- [x] Created Entity.Any (=> AnyEchoObject).
+- [x] TypeFormat => TypeFormat
+- [x] JsonSchemaType defs
+- [x] Fix database.add() input/output types and search for "Obj.Any = db.add" (also QueryResult types).
+- [x] TODO(burdon): FIX!!!
+- [x] Fix failing tests.
+- [x] Reconcile types/version with entities/model/version
+- [x] Expando moved to @dxos/schema
+- [x] Narrow QueryResult and match Schema and Object generics.
+
+2. Clean-up
+
+- [x] QueryResult namespace
+- [x] SchemaRegistry interface
+- [x] Schema registry should return Type.AnyEntity instead of Schema.Schema.AnyNoContext.
+- [x] Hypergraph interface
+- [x] Obj.getDatabase
+- [x] SpaceAction.AddObject target should be a db not a space.
+- [x] Add Relation.MakeProps
+- [x] Add Obj.Unknown and Relation.Unknown to match Entity.Unknown.
+- [x] TODO(wittjosiah): Should be Type.obj<...> or equivalent.
+- [x] TODO(wittjosiah): Find a simpler way to define this type.
+- [x] Ref.Array.targets doesn't satisfy Obj.Any because it uses AnyEchoObject.
+- [x] Remove echo-db/AnyLiveObject<T> => Obj.Obj<T>
+- [x] Rename AnyEchoObject => AnyEntity
+- [x] Remove WithId => AnyEntity
+- [x] Remove WithMeta => AnyEntity
+- [x] Obj.Any => Obj.Unknown
+- [x] Relation.Any => Relation.Unknown
+- [x] Ref.Any => Ref.Unknown
+- [x] Obj.AnyProps => Obj.Any
+- [x] Factor Expando out of @dxos/echo (moved to @dxos/schema)
+- [x] Obj.instanceOf works with Expando (tested in @dxos/schema)
+- [x] Obj.Snapshot should be same shape as Obj.Obj but with a different brand
+- [x] Mutators should only work on object after it is made mutable
+
+- [ ] Review usage of Obj.Any, see if it could be stricter
+- [ ] Type.Obj should validate using the echo object brand
+- [ ] Mutable could be a branded type to fix "NOTE: TypeScript's structural typing allows readonly objects to be passed to `Mutable<T>`"
+- [ ] space properties, queues, messagins preventing getSpace from being removed
+- [ ] Schema registry should use Query.Query.
+- [ ] Move EchoSchemaRegistry into hypergraph
+- [ ] Narrow QueryResult and match Schema and Object generics.
+- [ ] BaseSchema
+- [ ] Directly import JSONPath, etc. from @dxos/effect.
+- [ ] Promote parts of src/internal/ref to Ref.ts
+- [ ] JsonPath, JsonProp, getValue, setValue => Json.Path?
+- [ ] DISCUSS: Standradize $ suffix to disambuguate imports (GPT recommended).
+
+3. Audit usage from @dxos/echo-db
+
+- [ ] Ability to extract Struct from Type.Obj
+- [ ] created/updated system props (const { created } = Obj.getTimestamps)?
+- [ ] QueryFn, QueryOptions => Database
+- [ ] Datatbase.query() options?
+- [ ] Expando type is used as a fallback in many plance.
+- [ ] Standardize '@automerge/automerge' imports (A vs. next).
+
+## NOTES
+
+- Marking types as @internal (even for unexported types) erases type information.
+
+## 0.9.0
+
+- [ ] Database, Queue, Space, Type, namespaces.
+- [ ] Support class variant for types.
+- [ ] Support defaults.
+- [ ] Support effect Date, Timestamp formats.
+- [ ] Metadata for created, updated timestamps.
+- [ ] Effect team design review.
+- [ ] @category annotations to group types in the API.
+- [ ] TSdoc and LLM training for function generation.
+- [ ] Rename TypeFormat => Primitive?
+- [ ] Don't re-export effect?
+
+## Issues
+
+- [ ] Build error when changing from TypedObject using Ref$: Type 'Task' does not satisfy the constraint 'WithId' (see echo-schema/testing/schema.ts)
+- [ ] Reconcile all schema variants (Base, Immutable, TypedObject, EchoObject, etc.)
+- [ ] Consolidate getters (getType, getSchema, getTypename, getSchemaTypename, etc.)
+- [ ] ReactiveObject should specify id property? Reconcile AnyProperties, ReactiveObject, HasId, WithId, etc.
+- [ ] Can we us S.is(MyType) to detect objects with our types system? (Branding?)
+- [ ] Type.Expando doesn't work with AtomQuery result type (have to use Obj.Any instead).
+- [ ] Obj.Any doesn't work with Obj.update callback types (have to use `any` for the mutable parameter).

package/src/internal/common/api/index.ts

@@ -0,0 +1,15 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+/**
+ * Shared API helpers for Obj and Relation modules.
+ * These implementations use generic Entity types.
+ * Obj and Relation modules re-export with strongly-typed signatures.
+ *
+ * TODO(wittjosiah): Could this become the new "internal" where we slowly refactor
+ * existing internal modules into here? This would provide a cleaner boundary
+ * and prevent external packages from using internals directly.
+ */
+
+export * from './meta';

package/src/internal/common/api/meta.ts

@@ -0,0 +1,90 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import type { ForeignKey } from '@dxos/echo-protocol';
+import { assertArgument, invariant } from '@dxos/invariant';
+import type { DeepReadonly } from '@dxos/util';
+
+import type * as Tag from '../../../Tag';
+import type { Ref } from '../../Ref/ref';
+import { type Mutable } from '../proxy';
+import { type AnyProperties } from '../types';
+import { type EntityMeta, getMeta as getMeta$ } from '../types/meta';
+
+/**
+ * Deeply read-only version of EntityMeta.
+ */
+export type ReadonlyMeta = DeepReadonly<EntityMeta>;
+
+/**
+ * Mutable meta type received in meta mutation callbacks.
+ */
+export type Meta = Mutable<EntityMeta>;
+
+/**
+ * Get the metadata for an entity with validation.
+ * Returns mutable meta when passed a mutable entity (inside change callback).
+ * Returns read-only meta when passed a regular entity or snapshot.
+ *
+ * TODO(burdon): When passed a Snapshot, should return a snapshot of meta, not the live meta proxy.
+ */
+export function getMetaChecked(entity: Mutable<AnyProperties>): Meta;
+export function getMetaChecked(entity: AnyProperties): ReadonlyMeta;
+export function getMetaChecked(entity: AnyProperties): Meta | ReadonlyMeta {
+  assertArgument(entity, 'entity', 'Should be an entity.');
+  const meta = getMeta$(entity);
+  invariant(meta != null, 'Invalid entity.');
+  return meta;
+}
+
+/**
+ * @returns Foreign keys for the entity from the specified source.
+ * Accepts both reactive entities and snapshots.
+ */
+export const getKeys = (entity: AnyProperties, source: string): ForeignKey[] => {
+  assertArgument(entity, 'entity', 'Should be an entity.');
+  const meta = getMetaChecked(entity);
+  invariant(meta != null, 'Invalid entity.');
+  return meta.keys.filter((key) => key.source === source);
+};
+
+/**
+ * Delete all keys from the entity for the specified source.
+ * Must be called within an Obj.update or Relation.update callback.
+ */
+export const deleteKeys = (entity: Mutable<AnyProperties>, source: string) => {
+  const meta = getMetaChecked(entity);
+  for (let i = 0; i < meta.keys.length; i++) {
+    if (meta.keys[i].source === source) {
+      meta.keys.splice(i, 1);
+      i--;
+    }
+  }
+};
+
+/**
+ * Add a tag (a reference to a {@link Tag} object) to the entity. Idempotent.
+ * Must be called within an Obj.update or Relation.update callback.
+ */
+export const addTag = (entity: Mutable<AnyProperties>, tag: Ref<Tag.Tag>) => {
+  const meta = getMetaChecked(entity);
+  // Two refs to the same target are not `===`; dedupe by URI.
+  if (!meta.tags.some((existing) => existing.uri === tag.uri)) {
+    meta.tags.push(tag);
+  }
+};
+
+/**
+ * Remove a tag (a reference to a {@link Tag} object) from the entity. No-op when not present.
+ * Must be called within an Obj.update or Relation.update callback.
+ */
+export const removeTag = (entity: Mutable<AnyProperties>, tag: Ref<Tag.Tag>) => {
+  const meta = getMetaChecked(entity);
+  for (let i = 0; i < meta.tags.length; i++) {
+    if (meta.tags[i].uri === tag.uri) {
+      meta.tags.splice(i, 1);
+      i--;
+    }
+  }
+};

package/src/internal/common/index.ts

@@ -0,0 +1,7 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+export * from './api';
+export * from './proxy';
+export * from './types';

package/src/internal/common/proxy/change-context.ts

@@ -0,0 +1,138 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { batchEvents } from './event-batch';
+import { EventId } from './symbols';
+
+/**
+ * Generic change context tracking.
+ * Only one object can be in a change context at a time (synchronous changes).
+ *
+ * This module provides a unified change context mechanism used by both:
+ * - TypedReactiveHandler (for non-database objects, using target as key)
+ * - EchoReactiveHandler (for database objects, using ObjectCore as key)
+ */
+
+/**
+ * The object currently in a change context.
+ * Can be a target object (for typed reactive) or ObjectCore (for database objects).
+ */
+let currentChangeContext: object | null = null;
+
+/**
+ * The primary object that has pending notifications, if any.
+ * This uses the contextKey (target or ObjectCore).
+ */
+let pendingNotificationKey: object | null = null;
+
+/**
+ * Additional objects (owner chain) that need notifications.
+ * These are separate from the primary notification because they use EventId directly.
+ */
+const pendingOwnerNotifications = new Set<object>();
+
+/**
+ * Enter a change context for the given key.
+ * While in a change context, mutations are allowed on the associated object.
+ * Nested Obj.update calls are not supported.
+ *
+ * @param key - The key to enter the change context for (target object or ObjectCore).
+ * @returns A cleanup function that exits the change context.
+ */
+export const enterChangeContext = (key: object): (() => void) => {
+  currentChangeContext = key;
+  return () => {
+    currentChangeContext = null;
+  };
+};
+
+/**
+ * Check if the given key is currently in a change context.
+ *
+ * @param key - The key to check (target object or ObjectCore).
+ * @returns True if the key is in a change context, false otherwise.
+ */
+export const isInChangeContext = (key: object): boolean => {
+  return currentChangeContext === key;
+};
+
+/**
+ * Queue a notification for the given key to be fired when the change context exits.
+ *
+ * @param key - The key to queue a notification for.
+ */
+export const queueNotification = (key: object): void => {
+  if (currentChangeContext === key) {
+    pendingNotificationKey = key;
+  }
+};
+
+/**
+ * Queue an owner notification. Owner notifications are for objects in the ownership
+ * chain that should be notified when a nested object changes.
+ * These objects have EventId and emit directly.
+ * Skip if the target is already the current change context (to avoid duplicate notifications).
+ *
+ * @param target - The owner target that has EventId.
+ */
+export const queueOwnerNotification = (target: object): void => {
+  // Skip if this is the object already being changed (primary notification handles it).
+  if (currentChangeContext !== null && target !== currentChangeContext) {
+    pendingOwnerNotifications.add(target);
+  }
+};
+
+/**
+ * Check if there are any pending notifications for the given key.
+ *
+ * @param key - The key to check.
+ * @returns True if there are pending notifications, false otherwise.
+ */
+export const hasPendingNotifications = (key: object): boolean => {
+  return pendingNotificationKey === key;
+};
+
+/**
+ * Clear any pending notifications for the given key.
+ *
+ * @param key - The key to clear notifications for.
+ */
+export const clearPendingNotifications = (key: object): void => {
+  if (pendingNotificationKey === key) {
+    pendingNotificationKey = null;
+  }
+};
+
+/**
+ * Execute a callback within a change context.
+ * This is the shared implementation used by both TypedReactiveHandler and EchoReactiveHandler.
+ *
+ * @param contextKey - The key for the change context (target for typed, ObjectCore for db).
+ * @param eventTarget - The object that has the EventId for notifications.
+ * @param proxy - The proxy object to pass to the callback.
+ * @param callback - The callback to execute with mutations allowed.
+ */
+export const executeChange = (
+  contextKey: object,
+  eventTarget: object,
+  proxy: any,
+  callback: (proxy: any) => void,
+): void => {
+  const exitContext = enterChangeContext(contextKey);
+  try {
+    batchEvents(() => callback(proxy));
+  } finally {
+    exitContext();
+    // Fire primary notification.
+    if (hasPendingNotifications(contextKey)) {
+      clearPendingNotifications(contextKey);
+      (eventTarget as any)[EventId]?.emit();
+    }
+    // Fire owner chain notifications.
+    for (const ownerTarget of pendingOwnerNotifications) {
+      (ownerTarget as any)[EventId]?.emit();
+    }
+    pendingOwnerNotifications.clear();
+  }
+};