@dxos/echo 0.10.0 → 0.11.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dxos/echo",
-  "version": "0.10.0",
+  "version": "0.11.1",
   "description": "ECHO API",
   "homepage": "https://dxos.org",
   "bugs": "https://github.com/dxos/dxos/issues",
@@ -16,122 +16,127 @@
     ".": {
       "source": "./src/index.ts",
       "types": "./dist/types/src/index.d.ts",
-      "default": "./dist/lib/neutral/index.mjs"
+      "import": "./dist/lib/index.mjs"
     },
     "./internal": {
       "source": "./src/internal/index.ts",
       "types": "./dist/types/src/internal/index.d.ts",
-      "default": "./dist/lib/neutral/internal/index.mjs"
+      "import": "./dist/lib/internal.mjs"
     },
     "./testing": {
       "source": "./src/testing/index.ts",
       "types": "./dist/types/src/testing/index.d.ts",
-      "default": "./dist/lib/neutral/testing/index.mjs"
+      "import": "./dist/lib/testing.mjs"
     },
     "./Annotation": {
       "source": "./src/Annotation.ts",
       "types": "./dist/types/src/Annotation.d.ts",
-      "default": "./dist/lib/neutral/Annotation.mjs"
+      "import": "./dist/lib/Annotation.mjs"
+    },
+    "./Blob": {
+      "source": "./src/Blob.ts",
+      "types": "./dist/types/src/Blob.d.ts",
+      "import": "./dist/lib/Blob.mjs"
     },
     "./Database": {
       "source": "./src/Database.ts",
       "types": "./dist/types/src/Database.d.ts",
-      "default": "./dist/lib/neutral/Database.mjs"
+      "import": "./dist/lib/Database.mjs"
     },
     "./Entity": {
       "source": "./src/Entity.ts",
       "types": "./dist/types/src/Entity.d.ts",
-      "default": "./dist/lib/neutral/Entity.mjs"
+      "import": "./dist/lib/Entity.mjs"
     },
     "./Err": {
       "source": "./src/Err.ts",
       "types": "./dist/types/src/Err.d.ts",
-      "default": "./dist/lib/neutral/Err.mjs"
+      "import": "./dist/lib/Err.mjs"
     },
     "./Feed": {
       "source": "./src/Feed.ts",
       "types": "./dist/types/src/Feed.d.ts",
-      "default": "./dist/lib/neutral/Feed.mjs"
+      "import": "./dist/lib/Feed.mjs"
     },
     "./Filter": {
       "source": "./src/Filter.ts",
       "types": "./dist/types/src/Filter.d.ts",
-      "default": "./dist/lib/neutral/Filter.mjs"
+      "import": "./dist/lib/Filter.mjs"
     },
     "./Format": {
       "source": "./src/Format.ts",
       "types": "./dist/types/src/Format.d.ts",
-      "default": "./dist/lib/neutral/Format.mjs"
+      "import": "./dist/lib/Format.mjs"
     },
     "./JsonSchema": {
       "source": "./src/JsonSchema.ts",
       "types": "./dist/types/src/JsonSchema.d.ts",
-      "default": "./dist/lib/neutral/JsonSchema.mjs"
+      "import": "./dist/lib/JsonSchema.mjs"
     },
     "./Key": {
       "source": "./src/Key.ts",
       "types": "./dist/types/src/Key.d.ts",
-      "default": "./dist/lib/neutral/Key.mjs"
+      "import": "./dist/lib/Key.mjs"
     },
     "./Migration": {
       "source": "./src/Migration.ts",
       "types": "./dist/types/src/Migration.d.ts",
-      "default": "./dist/lib/neutral/Migration.mjs"
+      "import": "./dist/lib/Migration.mjs"
     },
     "./Obj": {
       "source": "./src/Obj.ts",
       "types": "./dist/types/src/Obj.d.ts",
-      "default": "./dist/lib/neutral/Obj.mjs"
+      "import": "./dist/lib/Obj.mjs"
     },
     "./Order": {
       "source": "./src/Order.ts",
       "types": "./dist/types/src/Order.d.ts",
-      "default": "./dist/lib/neutral/Order.mjs"
+      "import": "./dist/lib/Order.mjs"
     },
     "./Query": {
       "source": "./src/Query.ts",
       "types": "./dist/types/src/Query.d.ts",
-      "default": "./dist/lib/neutral/Query.mjs"
+      "import": "./dist/lib/Query.mjs"
     },
     "./QueryResult": {
       "source": "./src/QueryResult.ts",
       "types": "./dist/types/src/QueryResult.d.ts",
-      "default": "./dist/lib/neutral/QueryResult.mjs"
+      "import": "./dist/lib/QueryResult.mjs"
     },
     "./Ref": {
       "source": "./src/Ref.ts",
       "types": "./dist/types/src/Ref.d.ts",
-      "default": "./dist/lib/neutral/Ref.mjs"
+      "import": "./dist/lib/Ref.mjs"
     },
     "./Registry": {
       "source": "./src/Registry.ts",
       "types": "./dist/types/src/Registry.d.ts",
-      "default": "./dist/lib/neutral/Registry.mjs"
+      "import": "./dist/lib/Registry.mjs"
     },
     "./Relation": {
       "source": "./src/Relation.ts",
       "types": "./dist/types/src/Relation.d.ts",
-      "default": "./dist/lib/neutral/Relation.mjs"
+      "import": "./dist/lib/Relation.mjs"
     },
     "./Scope": {
       "source": "./src/Scope.ts",
       "types": "./dist/types/src/Scope.d.ts",
-      "default": "./dist/lib/neutral/Scope.mjs"
+      "import": "./dist/lib/Scope.mjs"
     },
     "./Tag": {
       "source": "./src/Tag.ts",
       "types": "./dist/types/src/Tag.d.ts",
-      "default": "./dist/lib/neutral/Tag.mjs"
+      "import": "./dist/lib/Tag.mjs"
     },
     "./Text": {
       "source": "./src/Text.ts",
       "types": "./dist/types/src/Text.d.ts",
-      "default": "./dist/lib/neutral/Text.mjs"
+      "import": "./dist/lib/Text.mjs"
     },
     "./Type": {
       "source": "./src/Type.ts",
       "types": "./dist/types/src/Type.d.ts",
-      "default": "./dist/lib/neutral/Type.mjs"
+      "import": "./dist/lib/Type.mjs"
     }
   },
   "files": [
@@ -142,16 +147,16 @@
     "@effect-atom/atom": "^0.5.3",
     "effect": "3.21.4",
     "semver": "^7.7.4",
-    "@dxos/async": "0.10.0",
-    "@dxos/debug": "0.10.0",
-    "@dxos/echo-protocol": "0.10.0",
-    "@dxos/effect": "0.10.0",
-    "@dxos/errors": "0.10.0",
-    "@dxos/keys": "0.10.0",
-    "@dxos/invariant": "0.10.0",
-    "@dxos/util": "0.10.0",
-    "@dxos/log": "0.10.0",
-    "@dxos/node-std": "0.10.0"
+    "@dxos/async": "0.11.1",
+    "@dxos/debug": "0.11.1",
+    "@dxos/echo-protocol": "0.11.1",
+    "@dxos/effect": "0.11.1",
+    "@dxos/errors": "0.11.1",
+    "@dxos/invariant": "0.11.1",
+    "@dxos/keys": "0.11.1",
+    "@dxos/log": "0.11.1",
+    "@dxos/node-std": "0.11.1",
+    "@dxos/util": "0.11.1"
   },
   "devDependencies": {
     "@types/node": "22.10.2",

package/src/Aggregate.ts

@@ -0,0 +1,105 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+// @import-as-namespace
+
+import type * as EffectTypes from 'effect/Types';
+
+import type { QueryAST } from '@dxos/echo-protocol';
+
+import type * as Order from './Order';
+import type * as Query from './Query';
+
+/**
+ * The aggregate spec sans name; the name is supplied by the `Query.aggregate` record key. A tagged
+ * union per kind so `property`/`limit`/`order` are present exactly when the kind uses them (no
+ * unused optional fields to guard against at read sites).
+ */
+export type Spec =
+  | { kind: 'group'; property: string }
+  | { kind: 'max'; property: string }
+  | { kind: 'min'; property: string }
+  | { kind: 'items'; limit?: number; order?: readonly QueryAST.Order[] }
+  | { kind: 'count' };
+
+export const AggregateTypeId = '~@dxos/echo/Aggregate' as const;
+export type AggregateTypeId = typeof AggregateTypeId;
+
+/**
+ * A per-group aggregate declaration, materialised as a top-level field on the flat result record
+ * `Query.aggregate` produces and orderable via a following `orderBy(Order.property(name))`. Name it
+ * via the record passed to `Query.aggregate({ name: Aggregate.max('created') })`.
+ *
+ * `T` is the query's element type (so a reduced/grouped property is checked against it); `V` is the
+ * value the aggregate produces. {@link group} entries define the grouping keys; a record with no
+ * `group` entries aggregates the entire input into a single row.
+ */
+export interface Aggregate<T, V> {
+  readonly [AggregateTypeId]: { element: T; value: V };
+
+  spec: Spec;
+}
+
+export type Any = Aggregate<any, any>;
+
+/** Extracts the value type a {@link Aggregate} produces. */
+export type ValueOf<A> = A extends Aggregate<any, infer V> ? V : never;
+
+/** Computes the flat result type of `Query.aggregate(aggregates)` for a given aggregate record `A`. */
+export type AggregationResult<A extends Record<string, Any>> = EffectTypes.Simplify<
+  Query.AggregateResult & { readonly [N in keyof A]: ValueOf<A[N]> }
+>;
+
+class AggregateClass<T, V> implements Aggregate<T, V> {
+  private static 'variance': Aggregate<any, any>[AggregateTypeId] = {} as Aggregate<any, any>[AggregateTypeId];
+
+  static is(value: unknown): value is Any {
+    return typeof value === 'object' && value !== null && AggregateTypeId in value;
+  }
+
+  constructor(public readonly spec: Spec) {}
+
+  [AggregateTypeId] = AggregateClass.variance as Aggregate<T, V>[AggregateTypeId];
+}
+
+/**
+ * Group members by a scalar property. The record key names the result field carrying the coerced
+ * group-key value; multiple `group` entries form a composite key. Members whose property is missing,
+ * `null`, `undefined`, or non-scalar group under the `null` key, so the field value is `T[K] | null`.
+ */
+export const group = <T, K extends keyof T & string>(property: K): Aggregate<T, T[K] | null> =>
+  new AggregateClass({ kind: 'group', property });
+
+/**
+ * Aggregate the maximum of a scalar property across the group's members.
+ * `T` is inferred from the `Query.aggregate` context, so `property` is checked against the query's
+ * element type (like {@link Order.property}).
+ */
+export const max = <T, K extends keyof T & string>(property: K): Aggregate<T, T[K] | null> =>
+  new AggregateClass({ kind: 'max', property });
+
+/**
+ * Aggregate the minimum of a scalar property across the group's members.
+ */
+export const min = <T, K extends keyof T & string>(property: K): Aggregate<T, T[K] | null> =>
+  new AggregateClass({ kind: 'min', property });
+
+/**
+ * Collect the group's members, optionally ordered by `order` and capped to `limit` per group.
+ * Opt-in — groups carry no members unless this aggregate is declared.
+ *
+ * `order` is this aggregate's own per-group ordering — independent of any `orderBy` elsewhere in
+ * the query, which orders the whole input stream (and, following `aggregate`, the resulting
+ * groups), not this aggregate's member selection. Without `order`, members keep whatever order
+ * they arrived in from a preceding `orderBy` (unspecified if there is none), so a per-group
+ * top-`limit` moved position in the chain would silently change; declaring `order` here makes the
+ * ordering explicit and local to this aggregate regardless of where it sits.
+ */
+export const items = <T>(options?: { limit?: number; order?: Order.Any[] }): Aggregate<T, T[]> =>
+  new AggregateClass({ kind: 'items', limit: options?.limit, order: options?.order?.map((order) => order.ast) });
+
+/**
+ * Count the group's members. Opt-in — groups carry no count unless this aggregate is declared.
+ */
+export const count = <T>(): Aggregate<T, number> => new AggregateClass({ kind: 'count' });

package/src/Blob.test.ts

@@ -0,0 +1,45 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import * as Schema from 'effect/Schema';
+import { describe, test } from 'vitest';
+
+import * as Blob from './Blob';
+
+describe('Blob', () => {
+  test('inline data schema roundtrip', ({ expect }) => {
+    const bytes = new Uint8Array([1, 2, 3, 4]);
+    const data = Blob.inlineData(bytes);
+    const decoded = Schema.decodeUnknownSync(Blob.BlobData)(data);
+    expect(decoded).toEqual({ _tag: 'inline', bytes });
+  });
+
+  test('external data schema roundtrip', ({ expect }) => {
+    const uri = 'ni:///sha-256;3q2-7w';
+    const data = Blob.externalData(uri);
+    const decoded = Schema.decodeUnknownSync(Blob.BlobData)(data);
+    expect(decoded).toEqual({ _tag: 'external', uri });
+  });
+
+  test('rejects an unknown tag', ({ expect }) => {
+    expect(() => Schema.decodeUnknownSync(Blob.BlobData)({ _tag: 'bogus' })).toThrow();
+  });
+
+  test('make constructs a valid Blob object', ({ expect }) => {
+    const bytes = new Uint8Array([1, 2, 3]);
+    const blob = Blob.make({ type: 'image/png', size: bytes.length, data: Blob.inlineData(bytes) });
+    expect(blob.type).toBe('image/png');
+    expect(blob.size).toBe(bytes.length);
+    expect(blob.data).toEqual({ _tag: 'inline', bytes });
+  });
+
+  test('Storage constants', ({ expect }) => {
+    expect(Blob.Storage.inline).toBe('inline');
+    expect(Blob.Storage.edge).toBe('edge');
+  });
+
+  test('Scheme constants', ({ expect }) => {
+    expect(Blob.Scheme.ni).toBe('ni');
+  });
+});

package/src/Blob.ts

@@ -0,0 +1,205 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+// @import-as-namespace
+
+import * as Effect from 'effect/Effect';
+import * as Option from 'effect/Option';
+import * as Schema from 'effect/Schema';
+
+import { DXN } from '@dxos/keys';
+
+import * as Annotation from './Annotation';
+import * as Database from './Database';
+import * as Err from './Err';
+import * as internal from './internal';
+import * as Obj from './Obj';
+import * as Type from './Type';
+
+/**
+ * Inline blob data: bytes stored directly on the ECHO object.
+ */
+export const InlineData = Schema.TaggedStruct('inline', {
+  bytes: Schema.Uint8ArrayFromSelf.annotations({ jsonSchema: { type: 'string', contentEncoding: 'base64' } }),
+});
+
+/**
+ * External blob data: bytes stored by a registered blob backend, addressed by `uri`.
+ */
+export const ExternalData = Schema.TaggedStruct('external', {
+  /**
+   * URI locating the bytes. The scheme selects the backend at read time (e.g. `ni:///sha-256;…` for
+   * the edge backend — an RFC 6920 content digest, not a backend name — or `wnfs://…` for the
+   * plugin-wnfs extension), resolved via the scheme→backend map built from `registerBlobBackend`
+   * calls. Core code never parses or constructs backend-specific URIs beyond reading the scheme.
+   *
+   * @see {@link https://www.rfc-editor.org/rfc/rfc6920 RFC 6920} for the `ni:` URI format.
+   */
+  uri: Schema.String,
+});
+
+export const BlobData = Schema.Union(InlineData, ExternalData);
+export type BlobData = Schema.Schema.Type<typeof BlobData>;
+
+/**
+ * Runtime schema for a Blob object.
+ *
+ * NOTE: This export shadows the DOM `Blob` global within modules that import it as a namespace.
+ * Code that needs the DOM class should use `globalThis.Blob` explicitly.
+ *
+ * @example
+ * ```ts
+ * const blob = yield* Blob.fromBytes(bytes, { type: 'image/png' });
+ * ```
+ */
+export class Blob extends Type.makeObject<Blob>(DXN.make('org.dxos.type.blob', '0.1.0'))(
+  Schema.Struct({
+    /** MIME type. */
+    type: Schema.optional(Schema.String),
+    size: Schema.Number,
+    data: BlobData,
+  }).pipe(
+    internal.HiddenAnnotation.set(true),
+    Annotation.IconAnnotation.set({ icon: 'ph--file--regular', hue: 'teal' }),
+  ),
+) {}
+
+//
+// Storage constants
+//
+
+/**
+ * Storage backend names, selected at write time via {@link FromBytesOptions.storage}. Call sites
+ * always use these constants, never raw string literals. Extensions (e.g. plugin-wnfs) define
+ * their own storage names in their own package — core code knows nothing of them.
+ *
+ * Distinct from {@link Scheme}: a storage name picks *where* to write, a URI scheme describes
+ * *what the identifier is* at read time. They coincide for backends with no content-addressing
+ * (e.g. wnfs uses `'wnfs'` for both), but the edge backend's storage name (`'edge'`) differs from
+ * its URI scheme (`Scheme.ni`) because the identifier it produces is a content hash, not an
+ * edge-specific address.
+ */
+export const Storage = { inline: 'inline', edge: 'edge' } as const;
+export type Storage = (typeof Storage)[keyof typeof Storage];
+
+/**
+ * URI schemes claimed by core backends (see `BlobBackend.schemes` in `@dxos/echo-protocol`).
+ *
+ * `ni` marks an {@link https://www.rfc-editor.org/rfc/rfc6920 RFC 6920} Named Information URI whose
+ * authority is empty and whose path carries a registered hash-algorithm name plus a base64url-encoded
+ * digest (e.g. `ni:///sha-256;UyaQNQIUxQKgg1jVMKMbg1Yr8Rrb2Y3RaOx2N0mVJhc`, produced by the edge
+ * backend). The scheme names *what the identifier is* — a content digest over the complete blob —
+ * not which backend wrote it, so any backend or cache that can resolve that digest may claim it.
+ * The URI is an identity claim, not a retrievability promise; transport is selected by the registered
+ * backend at read time.
+ *
+ * Extensions (e.g. plugin-wnfs) define their own schemes in their own package.
+ */
+export const Scheme = { ni: 'ni' } as const;
+export type Scheme = (typeof Scheme)[keyof typeof Scheme];
+
+//
+// Factory
+//
+
+export const MAX_INLINE_SIZE = 4 * 1024 * 1024;
+
+/**
+ * Creates a new blob object. Does not add it to the database — see {@link fromBytes}.
+ */
+export const make = (props: Obj.MakeProps<typeof Blob>): Blob => Obj.make(Blob, props);
+
+/**
+ * Creates an inline {@link BlobData} variant that embeds raw bytes on the ECHO object.
+ */
+export const inlineData = (bytes: Uint8Array): BlobData => ({ _tag: 'inline', bytes });
+
+/**
+ * Creates an external {@link BlobData} variant that references bytes held by a registered backend.
+ * @param uri - Backend-scoped URI, e.g. as returned by a `BlobBackend.put` call.
+ */
+export const externalData = (uri: string): BlobData => ({ _tag: 'external', uri });
+
+export interface FromBytesOptions {
+  /** MIME type. */
+  type?: string;
+  /**
+   * Storage backend name. Defaults to the registry's configured default (initially `'inline'`;
+   * `@dxos/client` marks `'edge'` as the default when configured).
+   */
+  storage?: Storage | (string & {});
+}
+
+//
+// Operations
+//
+
+/**
+ * Hashes and uploads `bytes` via the chosen storage backend (a no-op for `inline`), returning an
+ * un-added Blob object. The caller is responsible for adding it to the database.
+ *
+ * @example
+ * ```ts
+ * const blob = yield* Blob.fromBytes(bytes, { type: 'image/png' });
+ * yield* Database.add(blob);
+ * ```
+ */
+export const fromBytes = (
+  bytes: Uint8Array,
+  options?: FromBytesOptions,
+): Effect.Effect<Blob, Err.BlobTooLargeError | Err.BlobWriteError, Database.Service> =>
+  Database.Service.pipe(
+    Effect.flatMap(({ db }) =>
+      Effect.tryPromise({
+        try: () => db.createBlob(bytes, options),
+        catch: (error) =>
+          error instanceof Err.BlobTooLargeError || error instanceof Err.BlobWriteError
+            ? error
+            : new Err.BlobWriteError({ backend: options?.storage ?? 'unknown' }, { cause: error }),
+      }),
+    ),
+  ).pipe(Effect.withSpan('Blob.fromBytes'));
+
+/**
+ * Loads a blob's bytes. Inline: read directly off the object. External: dispatched to the
+ * registered backend for the URI's scheme.
+ *
+ * @example
+ * ```ts
+ * const bytes = yield* Blob.read(blob);
+ * ```
+ */
+export const read = (blob: Blob): Effect.Effect<Uint8Array, Err.BlobNotAvailableError, Database.Service> =>
+  Database.Service.pipe(
+    Effect.flatMap(({ db }) =>
+      Effect.tryPromise({
+        try: () => db.readBlob(blob),
+        catch: (error) =>
+          error instanceof Err.BlobNotAvailableError
+            ? error
+            : new Err.BlobNotAvailableError(
+                { backend: 'unknown', key: blob.id, reason: 'not-found' },
+                { cause: error },
+              ),
+      }),
+    ),
+  ).pipe(Effect.withSpan('Blob.read'));
+
+/**
+ * Checks whether a blob's bytes are currently available.
+ */
+export const exists = (blob: Blob): Effect.Effect<boolean, never, Database.Service> =>
+  Database.Service.pipe(Effect.flatMap(({ db }) => Effect.promise(() => db.blobExists(blob)))).pipe(
+    Effect.withSpan('Blob.exists'),
+  );
+
+/**
+ * Returns a renderable URL for the blob: inline blobs resolve to a `data:` URL; external blobs
+ * are resolved by the registered backend's `getUrl`, if it implements one.
+ */
+export const url = (blob: Blob): Effect.Effect<Option.Option<string>, never, Database.Service> =>
+  Database.Service.pipe(
+    Effect.flatMap(({ db }) => Effect.promise(() => db.getBlobUrl(blob))),
+    Effect.map(Option.fromNullable),
+  ).pipe(Effect.withSpan('Blob.url'));