@dxos/echo 0.8.4-main.fffef41 → 0.8.4-staging.60fe92afc8

package/src/internal/{projection → Type}/compose.test.ts

@@ -5,23 +5,23 @@
 import * as Schema from 'effect/Schema';
 import { describe, test } from 'vitest';
 
-import { FieldPath } from '../ast';
-import { FormatAnnotation, FormatEnum } from '../formats';
-import { toJsonSchema } from '../json';
-import { ECHO_ANNOTATIONS_NS_KEY } from '../json-schema';
-import { TypedObject } from '../object';
+import { DXN } from '@dxos/keys';
 
+import { FieldPath } from '../Annotation';
+import { EchoObjectSchema } from '../Entity';
+import { FormatAnnotation, TypeFormat } from '../Format';
+import { ECHO_ANNOTATIONS_NS_KEY, toJsonSchema } from '../JsonSchema';
 import { composeSchema } from './compose';
 
 describe('schema composition', () => {
   test('schema composition', ({ expect }) => {
-    class BaseType extends TypedObject({ typename: 'example.com/Person', version: '0.1.0' })({
+    const BaseType = Schema.Struct({
       name: Schema.String,
       email: Schema.String,
-    }) {}
+    }).pipe(EchoObjectSchema(DXN.make('com.example.person', '0.1.0')));
 
     const OverlaySchema = Schema.Struct({
-      email: Schema.String.pipe(FieldPath('$.email'), FormatAnnotation.set(FormatEnum.Email)),
+      email: Schema.String.pipe(FieldPath('$.email'), FormatAnnotation.set(TypeFormat.Email)),
     });
 
     const baseSchema = toJsonSchema(BaseType);
@@ -30,7 +30,7 @@ describe('schema composition', () => {
     expect(composedSchema.properties).to.deep.eq({
       email: {
         type: 'string',
-        format: FormatEnum.Email,
+        format: TypeFormat.Email,
         // TODO(dmaretskyi): Should use the new field.
         [ECHO_ANNOTATIONS_NS_KEY]: {
           meta: {

package/src/internal/{projection → Type}/compose.ts

@@ -2,18 +2,22 @@
 // Copyright 2024 DXOS.org
 //
 
+import type * as Types from 'effect/Types';
+
 import { invariant } from '@dxos/invariant';
 
-import { type JsonSchemaType } from '../json-schema';
-import { getSnapshot } from '../schema';
+import { type JsonSchemaType } from '../JsonSchema';
 
 /**
  * Creates a composite schema from the source and projection schemas.
  */
 // TODO(burdon): Use effect schema projections.
 // TODO(burdon): Can avoid having to call this every time we modify any property on the view?
-export const composeSchema = (source: JsonSchemaType, target: JsonSchemaType): JsonSchemaType => {
-  const result: JsonSchemaType = getSnapshot(target);
+export const composeSchema = (
+  source: Types.Mutable<JsonSchemaType>,
+  target: Types.Mutable<JsonSchemaType>,
+): Types.Mutable<JsonSchemaType> => {
+  const result = structuredClone(target);
   invariant('type' in result && result.type === 'object', 'source schema must be an object');
   invariant('type' in source && source.type === 'object', 'target schema must be an object');
 
@@ -21,12 +25,13 @@ export const composeSchema = (source: JsonSchemaType, target: JsonSchemaType): J
     const propSchema = source.properties![prop]; // TODO(dmaretskyi): Find by json-path instead.
     const annotations = (propSchema as JsonSchemaType)?.annotations?.meta;
     if (annotations) {
-      (result.properties[prop] as JsonSchemaType).annotations ??= {};
-      (result.properties[prop] as JsonSchemaType).annotations!.meta ??= {};
+      const resultProp = result.properties[prop] as Record<string, any>;
+      resultProp.annotations ??= {};
+      resultProp.annotations.meta ??= {};
       for (const key in annotations) {
-        (result.properties[prop] as JsonSchemaType).annotations!.meta![key] ??= {};
-        Object.assign((result.properties[prop] as JsonSchemaType).annotations!.meta![key], annotations[key], {
-          ...(result.properties[prop] as JsonSchemaType).annotations!.meta![key],
+        resultProp.annotations.meta[key] ??= {};
+        Object.assign(resultProp.annotations.meta[key], annotations[key], {
+          ...resultProp.annotations.meta[key],
         });
       }
     }

package/src/internal/Type/index.ts

@@ -0,0 +1,6 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+export * from './manipulation';
+export * from './type-schema';

package/src/internal/{schema → Type}/manipulation.ts

@@ -6,9 +6,6 @@ import * as Schema from 'effect/Schema';
 import * as SchemaAST from 'effect/SchemaAST';
 
 import { invariant } from '@dxos/invariant';
-import { DXN } from '@dxos/keys';
-
-import { type TypeAnnotation, TypeAnnotationId, TypeIdentifierAnnotationId } from '../ast';
 
 // TODO(ZaymonFC): Do this one at a time. This might be dangerous.
 export const addFieldsToSchema = (
@@ -68,25 +65,3 @@ export const updateFieldNameInSchema = (
     ),
   );
 };
-
-export const setTypenameInSchema = (
-  schema: Schema.Schema.AnyNoContext,
-  typename: string,
-): Schema.Schema.AnyNoContext => {
-  const existingAnnotation = schema.ast.annotations[TypeAnnotationId] as TypeAnnotation;
-  invariant(existingAnnotation, `Missing ${String(TypeAnnotationId)}`);
-
-  return schema.annotations({
-    ...schema.ast.annotations,
-    [TypeAnnotationId]: {
-      kind: existingAnnotation.kind,
-      typename,
-      version: existingAnnotation.version,
-    } satisfies TypeAnnotation,
-    [SchemaAST.JSONSchemaAnnotationId]: {
-      ...(schema.ast.annotations[SchemaAST.JSONSchemaAnnotationId] ?? {}),
-      $id: schema.ast.annotations[TypeIdentifierAnnotationId] ?? DXN.fromTypename(typename).toString(),
-      typename,
-    },
-  });
-};

package/src/internal/Type/type-schema.ts

@@ -0,0 +1,60 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import * as Schema from 'effect/Schema';
+
+import { IconAnnotation, LabelAnnotation } from '../Annotation';
+import { EntityKind, KindId, SchemaKindId, StaticTypeSchemaSlot } from '../common/types';
+import { EchoTypeKindSchema, TypeMetaSchemaDXN } from '../Entity';
+import { JsonSchemaType } from '../JsonSchema';
+
+/**
+ * Raw struct backing {@link TypeSchema}. Exposed only so `TypeSchema`
+ * (the TS type) can derive its data fields via `Schema.Schema.Type<typeof ...>`;
+ * runtime callers should use {@link TypeSchema} (the piped, branded entity).
+ *
+ * `typename` and `version` are NOT data fields — they live in `EntityMeta.key` /
+ * `EntityMeta.version` (the canonical registry-provenance pair, queryable via
+ * `Filter.key(...)`). The same `jsonSchema` payload also embeds them so a
+ * standalone JSON-Schema export remains self-describing, but the schema-registry
+ * reads/writes them through meta.
+ */
+const TypeSchemaStruct = Schema.Struct({
+  name: Schema.optional(Schema.String),
+  jsonSchema: JsonSchemaType,
+});
+
+/**
+ * Persistent representation of a schema.
+ */
+export const TypeSchema = TypeSchemaStruct.pipe(
+  LabelAnnotation.set(['name']),
+  IconAnnotation.set({
+    icon: 'ph--database--regular',
+    hue: 'green',
+  }),
+  EchoTypeKindSchema(TypeMetaSchemaDXN),
+);
+
+/**
+ * Persistent representation of a schema — the runtime shape that
+ * `db.addType(Type.makeObjectFromJsonSchema(...))` produces
+ * and `Filter.type(Type.Type).run()` returns.
+ *
+ * Structurally identical to a static `Type.Type` entity: the entity-handler's
+ * `get` trap exposes `[SchemaKindId]` (derived from `system.kind` and
+ * `data.jsonSchema.entityKind`) and rebuilds `[StaticTypeSchemaSlot]` lazily
+ * from `jsonSchema`, so a persisted instance satisfies the public `Type<A>`
+ * interface without casting.
+ */
+export type TypeSchema = Schema.Schema.Type<typeof TypeSchemaStruct> & {
+  /** Object identifier — injected by `EchoTypeKindSchema` and stamped at construction. */
+  readonly id: string;
+  /** Entity-kind discriminator (object/relation/type) carried on every entity instance. */
+  readonly [KindId]: EntityKind.Type;
+  /** Kind of schema described by this meta-instance — always `EntityKind.Type` for `Type.Type` itself. */
+  readonly [SchemaKindId]: EntityKind.Type;
+  /** Effect Schema rebuilt lazily from `jsonSchema`; satisfies `Type.getSchema(...)` without an extra cast. */
+  readonly [StaticTypeSchemaSlot]: Schema.Schema.AnyNoContext;
+};

package/src/internal/common/README.md

@@ -0,0 +1,102 @@
+## Refactor
+
+1. Remove access from outisde of @dxos/echo-db
+
+- [x] completely restructure @dxos/echo src/internal
+- [x] Remove import "." and ".."! (create lint rule).
+- [x] Unify FOUR different nests of test schema.
+- [x] Remove @deprecated from internal methods and mark @internal (e.g., getSchemaDXN).
+  - NOTE: Internal methods should not use the import \* from Obj/Type APIs.
+- [x] import EntityId => @dxos/keys
+- [x] Entity.Any = Obj.Any | Relation.Any
+  - NOTE: Relation does not extend (in not polymorphic with) Obj.
+- [x] import LabelAnnotation => Annotation.LabelAnnotation
+- [x] import Expando => @dxos/schema (Expando.Expando)
+- [x] live => Obj.make
+- [x] Rename live => makeObject
+- [x] Rename {EchoObject, EchoRelation} => {EchoObjecSchema, EchoRelationSchema}
+- [x] Rename AnyProperties => AnyProperties
+- [x] Created Entity.Any (=> AnyEchoObject).
+- [x] TypeFormat => TypeFormat
+- [x] JsonSchemaType defs
+- [x] Fix database.add() input/output types and search for "Obj.Any = db.add" (also QueryResult types).
+- [x] TODO(burdon): FIX!!!
+- [x] Fix failing tests.
+- [x] Reconcile types/version with entities/model/version
+- [x] Expando moved to @dxos/schema
+- [x] Narrow QueryResult and match Schema and Object generics.
+
+2. Clean-up
+
+- [x] QueryResult namespace
+- [x] SchemaRegistry interface
+- [x] Schema registry should return Type.AnyEntity instead of Schema.Schema.AnyNoContext.
+- [x] Hypergraph interface
+- [x] Obj.getDatabase
+- [x] SpaceAction.AddObject target should be a db not a space.
+- [x] Add Relation.MakeProps
+- [x] Add Obj.Unknown and Relation.Unknown to match Entity.Unknown.
+- [x] TODO(wittjosiah): Should be Type.obj<...> or equivalent.
+- [x] TODO(wittjosiah): Find a simpler way to define this type.
+- [x] Ref.Array.targets doesn't satisfy Obj.Any because it uses AnyEchoObject.
+- [x] Remove echo-db/AnyLiveObject<T> => Obj.Obj<T>
+- [x] Rename AnyEchoObject => AnyEntity
+- [x] Remove WithId => AnyEntity
+- [x] Remove WithMeta => AnyEntity
+- [x] Obj.Any => Obj.Unknown
+- [x] Relation.Any => Relation.Unknown
+- [x] Ref.Any => Ref.Unknown
+- [x] Obj.AnyProps => Obj.Any
+- [x] Factor Expando out of @dxos/echo (moved to @dxos/schema)
+- [x] Obj.instanceOf works with Expando (tested in @dxos/schema)
+- [x] Obj.Snapshot should be same shape as Obj.Obj but with a different brand
+- [x] Mutators should only work on object after it is made mutable
+
+- [ ] Review usage of Obj.Any, see if it could be stricter
+- [ ] Type.Obj should validate using the echo object brand
+- [ ] Mutable could be a branded type to fix "NOTE: TypeScript's structural typing allows readonly objects to be passed to `Mutable<T>`"
+- [ ] space properties, queues, messagins preventing getSpace from being removed
+- [ ] Schema registry should use Query.Query.
+- [ ] Move EchoSchemaRegistry into hypergraph
+- [ ] Narrow QueryResult and match Schema and Object generics.
+- [ ] BaseSchema
+- [ ] Directly import JSONPath, etc. from @dxos/effect.
+- [ ] Promote parts of src/internal/ref to Ref.ts
+- [ ] JsonPath, JsonProp, getValue, setValue => Json.Path?
+- [ ] DISCUSS: Standradize $ suffix to disambuguate imports (GPT recommended).
+
+3. Audit usage from @dxos/echo-db
+
+- [ ] Ability to extract Struct from Type.Obj
+- [ ] created/updated system props (const { created } = Obj.getTimestamps)?
+- [ ] QueryFn, QueryOptions => Database
+- [ ] Datatbase.query() options?
+- [ ] Expando type is used as a fallback in many plance.
+- [ ] Standardize '@automerge/automerge' imports (A vs. next).
+
+## NOTES
+
+- Marking types as @internal (even for unexported types) erases type information.
+
+## 0.9.0
+
+- [ ] Database, Queue, Space, Type, namespaces.
+- [ ] Support class variant for types.
+- [ ] Support defaults.
+- [ ] Support effect Date, Timestamp formats.
+- [ ] Metadata for created, updated timestamps.
+- [ ] Effect team design review.
+- [ ] @category annotations to group types in the API.
+- [ ] TSdoc and LLM training for function generation.
+- [ ] Rename TypeFormat => Primitive?
+- [ ] Don't re-export effect?
+
+## Issues
+
+- [ ] Build error when changing from TypedObject using Ref$: Type 'Task' does not satisfy the constraint 'WithId' (see echo-schema/testing/schema.ts)
+- [ ] Reconcile all schema variants (Base, Immutable, TypedObject, EchoObject, etc.)
+- [ ] Consolidate getters (getType, getSchema, getTypename, getSchemaTypename, etc.)
+- [ ] ReactiveObject should specify id property? Reconcile AnyProperties, ReactiveObject, HasId, WithId, etc.
+- [ ] Can we us S.is(MyType) to detect objects with our types system? (Branding?)
+- [ ] Type.Expando doesn't work with AtomQuery result type (have to use Obj.Any instead).
+- [ ] Obj.Any doesn't work with Obj.update callback types (have to use `any` for the mutable parameter).

package/src/internal/common/api/index.ts

@@ -0,0 +1,15 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+/**
+ * Shared API helpers for Obj and Relation modules.
+ * These implementations use generic Entity types.
+ * Obj and Relation modules re-export with strongly-typed signatures.
+ *
+ * TODO(wittjosiah): Could this become the new "internal" where we slowly refactor
+ * existing internal modules into here? This would provide a cleaner boundary
+ * and prevent external packages from using internals directly.
+ */
+
+export * from './meta';

package/src/internal/common/api/meta.ts

@@ -0,0 +1,90 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import type { ForeignKey } from '@dxos/echo-protocol';
+import { assertArgument, invariant } from '@dxos/invariant';
+import type { DeepReadonly } from '@dxos/util';
+
+import type * as Tag from '../../../Tag';
+import type { Ref } from '../../Ref/ref';
+import { type Mutable } from '../proxy';
+import { type AnyProperties } from '../types';
+import { type EntityMeta, getMeta as getMeta$ } from '../types/meta';
+
+/**
+ * Deeply read-only version of EntityMeta.
+ */
+export type ReadonlyMeta = DeepReadonly<EntityMeta>;
+
+/**
+ * Mutable meta type received in meta mutation callbacks.
+ */
+export type Meta = Mutable<EntityMeta>;
+
+/**
+ * Get the metadata for an entity with validation.
+ * Returns mutable meta when passed a mutable entity (inside change callback).
+ * Returns read-only meta when passed a regular entity or snapshot.
+ *
+ * TODO(burdon): When passed a Snapshot, should return a snapshot of meta, not the live meta proxy.
+ */
+export function getMetaChecked(entity: Mutable<AnyProperties>): Meta;
+export function getMetaChecked(entity: AnyProperties): ReadonlyMeta;
+export function getMetaChecked(entity: AnyProperties): Meta | ReadonlyMeta {
+  assertArgument(entity, 'entity', 'Should be an entity.');
+  const meta = getMeta$(entity);
+  invariant(meta != null, 'Invalid entity.');
+  return meta;
+}
+
+/**
+ * @returns Foreign keys for the entity from the specified source.
+ * Accepts both reactive entities and snapshots.
+ */
+export const getKeys = (entity: AnyProperties, source: string): ForeignKey[] => {
+  assertArgument(entity, 'entity', 'Should be an entity.');
+  const meta = getMetaChecked(entity);
+  invariant(meta != null, 'Invalid entity.');
+  return meta.keys.filter((key) => key.source === source);
+};
+
+/**
+ * Delete all keys from the entity for the specified source.
+ * Must be called within an Obj.update or Relation.update callback.
+ */
+export const deleteKeys = (entity: Mutable<AnyProperties>, source: string) => {
+  const meta = getMetaChecked(entity);
+  for (let i = 0; i < meta.keys.length; i++) {
+    if (meta.keys[i].source === source) {
+      meta.keys.splice(i, 1);
+      i--;
+    }
+  }
+};
+
+/**
+ * Add a tag (a reference to a {@link Tag} object) to the entity. Idempotent.
+ * Must be called within an Obj.update or Relation.update callback.
+ */
+export const addTag = (entity: Mutable<AnyProperties>, tag: Ref<Tag.Tag>) => {
+  const meta = getMetaChecked(entity);
+  // Two refs to the same target are not `===`; dedupe by URI.
+  if (!meta.tags.some((existing) => existing.uri === tag.uri)) {
+    meta.tags.push(tag);
+  }
+};
+
+/**
+ * Remove a tag (a reference to a {@link Tag} object) from the entity. No-op when not present.
+ * Must be called within an Obj.update or Relation.update callback.
+ */
+export const removeTag = (entity: Mutable<AnyProperties>, tag: Ref<Tag.Tag>) => {
+  const meta = getMetaChecked(entity);
+  for (let i = 0; i < meta.tags.length; i++) {
+    if (meta.tags[i].uri === tag.uri) {
+      meta.tags.splice(i, 1);
+      i--;
+    }
+  }
+};

package/src/internal/{testing → common}/index.ts

@@ -2,5 +2,6 @@
 // Copyright 2024 DXOS.org
 //
 
+export * from './api';
+export * from './proxy';
 export * from './types';
-export * from './utils';

package/src/internal/common/proxy/change-context.ts

@@ -0,0 +1,138 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { batchEvents } from './event-batch';
+import { EventId } from './symbols';
+
+/**
+ * Generic change context tracking.
+ * Only one object can be in a change context at a time (synchronous changes).
+ *
+ * This module provides a unified change context mechanism used by both:
+ * - TypedReactiveHandler (for non-database objects, using target as key)
+ * - EchoReactiveHandler (for database objects, using ObjectCore as key)
+ */
+
+/**
+ * The object currently in a change context.
+ * Can be a target object (for typed reactive) or ObjectCore (for database objects).
+ */
+let currentChangeContext: object | null = null;
+
+/**
+ * The primary object that has pending notifications, if any.
+ * This uses the contextKey (target or ObjectCore).
+ */
+let pendingNotificationKey: object | null = null;
+
+/**
+ * Additional objects (owner chain) that need notifications.
+ * These are separate from the primary notification because they use EventId directly.
+ */
+const pendingOwnerNotifications = new Set<object>();
+
+/**
+ * Enter a change context for the given key.
+ * While in a change context, mutations are allowed on the associated object.
+ * Nested Obj.update calls are not supported.
+ *
+ * @param key - The key to enter the change context for (target object or ObjectCore).
+ * @returns A cleanup function that exits the change context.
+ */
+export const enterChangeContext = (key: object): (() => void) => {
+  currentChangeContext = key;
+  return () => {
+    currentChangeContext = null;
+  };
+};
+
+/**
+ * Check if the given key is currently in a change context.
+ *
+ * @param key - The key to check (target object or ObjectCore).
+ * @returns True if the key is in a change context, false otherwise.
+ */
+export const isInChangeContext = (key: object): boolean => {
+  return currentChangeContext === key;
+};
+
+/**
+ * Queue a notification for the given key to be fired when the change context exits.
+ *
+ * @param key - The key to queue a notification for.
+ */
+export const queueNotification = (key: object): void => {
+  if (currentChangeContext === key) {
+    pendingNotificationKey = key;
+  }
+};
+
+/**
+ * Queue an owner notification. Owner notifications are for objects in the ownership
+ * chain that should be notified when a nested object changes.
+ * These objects have EventId and emit directly.
+ * Skip if the target is already the current change context (to avoid duplicate notifications).
+ *
+ * @param target - The owner target that has EventId.
+ */
+export const queueOwnerNotification = (target: object): void => {
+  // Skip if this is the object already being changed (primary notification handles it).
+  if (currentChangeContext !== null && target !== currentChangeContext) {
+    pendingOwnerNotifications.add(target);
+  }
+};
+
+/**
+ * Check if there are any pending notifications for the given key.
+ *
+ * @param key - The key to check.
+ * @returns True if there are pending notifications, false otherwise.
+ */
+export const hasPendingNotifications = (key: object): boolean => {
+  return pendingNotificationKey === key;
+};
+
+/**
+ * Clear any pending notifications for the given key.
+ *
+ * @param key - The key to clear notifications for.
+ */
+export const clearPendingNotifications = (key: object): void => {
+  if (pendingNotificationKey === key) {
+    pendingNotificationKey = null;
+  }
+};
+
+/**
+ * Execute a callback within a change context.
+ * This is the shared implementation used by both TypedReactiveHandler and EchoReactiveHandler.
+ *
+ * @param contextKey - The key for the change context (target for typed, ObjectCore for db).
+ * @param eventTarget - The object that has the EventId for notifications.
+ * @param proxy - The proxy object to pass to the callback.
+ * @param callback - The callback to execute with mutations allowed.
+ */
+export const executeChange = (
+  contextKey: object,
+  eventTarget: object,
+  proxy: any,
+  callback: (proxy: any) => void,
+): void => {
+  const exitContext = enterChangeContext(contextKey);
+  try {
+    batchEvents(() => callback(proxy));
+  } finally {
+    exitContext();
+    // Fire primary notification.
+    if (hasPendingNotifications(contextKey)) {
+      clearPendingNotifications(contextKey);
+      (eventTarget as any)[EventId]?.emit();
+    }
+    // Fire owner chain notifications.
+    for (const ownerTarget of pendingOwnerNotifications) {
+      (ownerTarget as any)[EventId]?.emit();
+    }
+    pendingOwnerNotifications.clear();
+  }
+};