@dreamlake/dreamdb 0.5.1 → 0.5.2

package/browser-extras.mjs

@@ -37,6 +37,42 @@ class AuthoringUnavailable {
 
 export { AuthoringUnavailable as Authoring }
 
+/**
+ * `Writer.ingestCmaf` is Node-only, for the same reason `Authoring` is: it is
+ * gated behind the `write-full` feature, and the browser build is compiled
+ * with `write`.
+ *
+ * The shared declarations describe it on both ends (see build.sh, "types"), so
+ * a browser caller type-checks and would otherwise reach
+ * `writer.ingestCmaf is not a function` at runtime — a message that names
+ * neither the cause nor the fix. This replaces it with one that does.
+ *
+ * Applied to the prototype rather than declared as a method because `Writer`
+ * comes from the wasm module; there is no class here to extend.
+ */
+function installIngestCmafStub(Writer) {
+  if (!Writer || typeof Writer !== 'function') return Writer
+  if (typeof Writer.prototype.ingestCmaf === 'function') return Writer // full build
+  Object.defineProperty(Writer.prototype, 'ingestCmaf', {
+    configurable: true,
+    writable: true,
+    value: async function ingestCmaf() {
+      throw new Error(
+        'Writer.ingestCmaf is not available in the browser build of ' +
+          '@dreamlake/dreamdb. CMAF ingest takes the whole clip as one ' +
+          'argument, so peak memory is the entire source — and fragmenting a ' +
+          'multi-GB file inside a tab\'s 32-bit address space is not a ' +
+          'workflow this package pretends to support. Run it under Node (the ' +
+          '"node" export condition resolves to the full build), or fragment ' +
+          'ahead of time and append the fragments.',
+      )
+    },
+  })
+  return Writer
+}
+
+export { installIngestCmafStub }
+
 /**
  * A Backend (contract v2) that gets bytes over plain `fetch` and writes through
  * short-lived presigned URLs minted by *your* server.
@@ -86,7 +122,7 @@ export class PresignedBackend {
    * @param {object} o
    * @param {string} o.readBase                 Base URL for reads.
    * @param {(paths: string[], opts?: {ifMatch?: string}) => Promise<Record<string, {url: string, headers?: Record<string,string>}>>} o.mintPut
-   * @param {(path: string, opts: {ifMatch?: string, ifNoneMatchStar?: boolean}, bytes: Uint8Array) => Promise<{status: string, etag?: string}>} [o.commitRef]
+   * @param {(path: string, opts: {ifMatch?: string, ifNoneMatchStar?: boolean}, bytes: Uint8Array) => Promise<{outcome?: string, status?: string, etag?: string}>} [o.commitRef]
    *        Server-side ref commit. Used when `refStrategy` is `'proxy'`.
    * @param {'proxy'|'presign'} [o.refStrategy='proxy']
    *        How the 33-byte ref object is written. `'proxy'` posts it to your
@@ -119,15 +155,27 @@ export class PresignedBackend {
     return this._readUrl ? this._readUrl(path) : `${this.readBase}/${path}`
   }
 
-  /** v2 get: returns `{bytes, etag}`. Range is inclusive of `end`. */
+  /** v2 get: returns `{bytes, etag, totalLength}`. Range is half-open `[start, end)`. */
   async get(path, range) {
     const headers = {}
-    if (range) headers.range = `bytes=${range.start}-${range.end}`
+    if (range) {
+      if (!Number.isSafeInteger(range.start) || !Number.isSafeInteger(range.end) ||
+          range.start < 0 || range.end <= range.start) {
+        throw new Error(`GET ${path}: invalid half-open range [${range.start}, ${range.end})`)
+      }
+      // HTTP uses an inclusive end; the Backend contract uses an exclusive one.
+      headers.range = `bytes=${range.start}-${range.end - 1}`
+    }
     const r = await fetch(this.#url(path), { headers })
     if (!r.ok) throw new Error(`GET ${path}: ${r.status} ${r.statusText}`)
+    const contentRange = r.headers.get('content-range')
+    const rangeTotal = contentRange && /^bytes \d+-\d+\/(\d+)$/.exec(contentRange)?.[1]
+    const lengthText = range ? rangeTotal : r.headers.get('content-length')
+    const totalLength = lengthText == null ? undefined : Number(lengthText)
     return {
       bytes: new Uint8Array(await r.arrayBuffer()),
       etag: normalizeEtag(r.headers.get('etag')),
+      totalLength: Number.isSafeInteger(totalLength) && totalLength >= 0 ? totalLength : undefined,
     }
   }
 
@@ -148,7 +196,12 @@ export class PresignedBackend {
     // semantics differ per provider — so they get their own path.
     const isRef = path.startsWith('refs/')
     if (isRef && this.refStrategy === 'proxy') {
-      return await this.commitRef(path, opts, bytes)
+      // `commitRef` is the CALLER's callback and its documented contract is the
+      // `{status}` string form, so it is normalised here rather than being
+      // broken. This is the ref path — the one where mistaking a refused CAS
+      // for a successful one loses a write — so it does not rely on the
+      // adapter's compatibility handling.
+      return normalizePutResult(await this.commitRef(path, opts, bytes), path)
     }
 
     const minted = await this.mintPut([path], opts.ifMatch ? { ifMatch: opts.ifMatch } : undefined)
@@ -167,11 +220,11 @@ export class PresignedBackend {
     // someone else moved it, which is a conflict the caller must resolve.
     if (r.status === 412 || r.status === 409) {
       return opts.ifNoneMatchStar
-        ? { status: 'exists' }
-        : { status: 'casFailed', etag: normalizeEtag(r.headers.get('etag')) }
+        ? { outcome: 'alreadyExists' }
+        : { outcome: 'casFailed', etag: normalizeEtag(r.headers.get('etag')) }
     }
     if (!r.ok) throw new Error(`PUT ${path}: ${r.status} ${r.statusText}`)
-    return { status: 'created', etag: normalizeEtag(r.headers.get('etag')) }
+    return { outcome: 'created', etag: normalizeEtag(r.headers.get('etag')) }
   }
 
   // No `delete`. It would need a signed DELETE URL, and the sign endpoint only
@@ -181,6 +234,28 @@ export class PresignedBackend {
   // dropping a ref is an `Authoring` operation, which is Node-only.
 }
 
+/**
+ * Normalise a caller-supplied put result into the tagged form.
+ *
+ * Accepts the documented `{status: 'created'|'exists'|'casFailed'}` and the
+ * tagged `{outcome}`; anything else is refused by name rather than silently
+ * treated as success, because the one thing a ref write must never do is report
+ * a refused CAS as a completed one.
+ */
+function normalizePutResult(r, path) {
+  if (r && typeof r.outcome === 'string') return r
+  const s = r && r.status
+  if (s === 'created') return { outcome: 'created', etag: r.etag }
+  if (s === 'exists' || s === 'alreadyExists') return { outcome: 'alreadyExists', etag: r.etag }
+  if (s === 'casFailed') return { outcome: 'casFailed', etag: r.etag }
+  if (s === 'notFound') return { outcome: 'notFound' }
+  if (typeof s === 'number') return r
+  throw new Error(
+    `commitRef(${path}) resolved an unrecognised result; expected ` +
+      `{outcome} or {status: 'created'|'exists'|'casFailed'}, got ${JSON.stringify(r)}`,
+  )
+}
+
 /**
  * S3 quotes ETags and sometimes suffixes them (`W/"…"`, `"…-3"` for multipart).
  * The conditional-write comparison is byte-exact, so a stray quote turns every

package/browser.mjs

@@ -2,3 +2,10 @@
 // module and instantiated by the bundler; there is nothing to await.
 export * from './browser/dreamdb.js'
 export { ZERO_SPATIAL_KEY, PresignedBackend, Authoring } from './browser-extras.mjs'
+
+// `ingestCmaf` is behind `write-full`, so it is absent here while the shared
+// declarations describe it. Replace the bare `is not a function` with a
+// message that names the reason and the fix.
+import { Writer as _W } from './browser/dreamdb.js'
+import { installIngestCmafStub as _stub } from './browser-extras.mjs'
+_stub(_W)

package/index.d.ts

@@ -23,6 +23,14 @@ export class Authoring {
     private constructor();
     free(): void;
     [Symbol.dispose](): void;
+    /**
+     * Add one fixed-shape typed-array Constant.
+     *
+     * `field` is the same `{name, kind:'array', ...}` descriptor accepted by
+     * `create`, with `trackKind:'constant'` and `objectKind:'constant'`.
+     * `payload` is a raw or NPY payload matching that declaration.
+     */
+    addArrayConstant(field: any, payload: Uint8Array): Promise<void>;
     /**
      * Add an embedding column whose index artifacts already exist.
      *
@@ -127,6 +135,15 @@ export class S3Backend {
      * content is hash-addressed; refs are read fresh each page load anyway).
      */
     get(path: string, _opts: any): Promise<Uint8Array>;
+    /**
+     * Stream `path` without first aggregating the response in wasm memory.
+     *
+     * Returning the length beside the browser stream satisfies the optional
+     * `Backend.getStream` contract. Older injected backends that implement
+     * only [`Self::get`] remain compatible, but cannot claim bounded memory
+     * for historical oversized inline Track reads.
+     */
+    getStream(path: string): Promise<any>;
     /**
      * List object paths under `prefix`. Stub (returns empty); the current
      * consumer read paths don't require listing.
@@ -142,6 +159,32 @@ export class Space {
     private constructor();
     free(): void;
     [Symbol.dispose](): void;
+    /**
+     * The Space's human-readable description, or `undefined` when none is set.
+     *
+     * The read half of `Authoring.setDescription` (#129). It lives here rather
+     * than on `Authoring` because that type is `write-full` and so exists only
+     * in the Node build — a description is metadata a READER wants, and
+     * `Space` is the surface all three build targets share.
+     *
+     * `async` although `Dataset::description` is not: the value comes from the
+     * Dataset this Space opens lazily at the Manifest hash already pinned by
+     * this Space, so the first call may perform that open without re-resolving
+     * a source Ref.
+     *
+     * Deliberately routed through `Dataset::description` rather than read out
+     * of `self.manifest`'s registry, which would be shorter and would also
+     * serve the cached decoded Manifest. `Dataset::recover_meta` is not a prefix
+     * filter: it accepts the legacy `vortex.*` namespace as well as
+     * `dreamdb.*`, canonicalises, and drops structural keys. Matching
+     * `dreamdb.description` here would silently return nothing for a legacy
+     * Space whose description is on disk and readable from Rust — the exact
+     * defect `recover_meta`'s own comment records.
+     *
+     * `Option<String>` marshals to `string | undefined`, NOT to `""`. An unset
+     * description and an empty one stay distinguishable across the boundary.
+     */
+    description(): Promise<string | undefined>;
     /**
      * Open a Space from a `.../refs/<name>` or `.../manifests/<hash>` URI,
      * resolving the ref (if any) and loading the manifest. If `backend` is
@@ -221,18 +264,55 @@ export class Space {
      * two-pass rerank + HotShard merge + tombstone suppression.
      */
     queryVector(field: string, query: Float32Array, opts: any): Promise<Array<any>>;
+    /**
+     * Read an Event/Unbucketed or Continuous/TimeBatch typed-array field as
+     * `Map<anchorString, {values, shape, ...}>`.
+     */
+    readArrayColumn(field: string): Promise<Map<any, any>>;
+    /**
+     * Read a typed-array Constant and decode it to a JS TypedArray plus its
+     * authoritative shape and semantic metadata.
+     */
+    readArrayConstant(field: string): Promise<any>;
     /**
      * Read a scalar track as a `Map<anchorString, value>`. Single-anchor
      * entries resolve from the track index; multi-anchor entries fetch (and
      * de-dupe) their bucket and roaring-decode the anchor set.
      */
     readScalarColumn(track: any, _opts: any): Promise<Map<any, any>>;
+    /**
+     * Fetch verified initialization and complete Fragments overlapping one
+     * item-relative half-open range. This materializes the selected bytes; it
+     * is not a streaming API.
+     */
+    readVideoItemRange(field: string, itemKey: Uint8Array, relativeStartNs: bigint | number, relativeEndNs: bigint | number): Promise<VideoItemRead | undefined>;
     /**
      * Fetch a track object and return its `object_index` (array of CBOR
      * entries). Inline indexes only — paged indexes are not resolved (matching
      * dreamdb-ts, which also doesn't support them on this path).
      */
     resolveObjectIndex(track: any): Promise<Array<any>>;
+    /**
+     * Semantic-cache capacity in bytes (spec/0006 §3.6). Default 1 GiB.
+     */
+    semanticCacheCapacity(): Promise<number>;
+    /**
+     * Semantic-cache occupancy as `{ entries, bytesUsed, maxBytes }`.
+     */
+    semanticCacheStats(): Promise<object>;
+    /**
+     * Set the semantic-cache capacity in bytes, evicting down to it before
+     * returning. Returns the bytes evicted. `0` disables the cache.
+     *
+     * An index whose accounted size exceeds the cap is not cached and is
+     * re-fetched on every query, reported honestly as a miss. The default is
+     * not raised globally to suit one corpus.
+     *
+     * Sizing is workload- and architecture-dependent, and figures measured on
+     * a 64-bit host do NOT carry over here: `usize` is 32 bits on `wasm32`, so
+     * every offset and length inside a decoded index accounts differently.
+     */
+    setSemanticCacheCapacity(n_bytes: number): Promise<number>;
     /**
      * Base32 hash of the manifest's first timeline.
      */
@@ -245,6 +325,16 @@ export class Space {
      * List resolved tracks from this manifest.
      */
     tracks(): Array<any>;
+    /**
+     * Look up the logical VideoItem containing one absolute Timeline anchor
+     * without fetching media. Exact nanosecond values are returned as bigint.
+     */
+    videoItemAt(field: string, anchorNs: bigint | number): Promise<VideoItemInfo | undefined>;
+    /**
+     * Look up one logical VideoItem by its stable opaque key without fetching
+     * initialization or Fragment bytes.
+     */
+    videoItemByKey(field: string, itemKey: Uint8Array): Promise<VideoItemInfo | undefined>;
     /**
      * The base URL objects are fetched relative to (consumers build their own
      * object URLs from this — e.g. `dreamdb-demo`'s track readers).
@@ -289,6 +379,50 @@ export class Writer {
      * Tombstone records by anchor. Returns the new manifest hash.
      */
     deleteRecords(anchors: BigUint64Array, reason?: string | null): Promise<string>;
+    /**
+     * Ingest a pre-fragmented CMAF (fragmented-MP4) video into a Video field
+     * as a `spec/0007` §5 media Track — the thing that makes a video Track
+     * time-seekable and MSE-playable rather than one opaque blob.
+     *
+     * ```js
+     * const out = await writer.ingestCmaf('video', initBytes, [
+     *   [frag0, 0n, 2_000_000_000n],
+     *   [frag1, 2_000_000_000n, 4_000_000_000n],
+     * ])
+     * ```
+     *
+     * - `init` is the initialization segment (`ftyp`+`moov`).
+     * - each fragment is `[Uint8Array, tStartNs, tEndNs]`, half-open, already
+     *   offset to the timeline anchor the caller wants. Pass `bigint`s:
+     *   wall-clock nanosecond anchors are far past `Number.MAX_SAFE_INTEGER`
+     *   and a rounded `tStart` writes the fragment into a 60 s bucket that
+     *   does not match the address it is indexed at.
+     *
+     * Calling it again on the SAME field APPENDS (one episode per call on a
+     * shared timeline) and reuses the track's existing init segment; a clip
+     * whose init differs — different codec, resolution, fps or audio config —
+     * is refused, because a Track carries exactly one init segment and the
+     * mismatch would produce fragments that decode against the wrong one.
+     *
+     * Returns `{field, modality, initHash, fragmentCount, coverage: [bigint,
+     * bigint], trackHash, manifestHash}`. `coverage` is `bigint` for the
+     * reason above; `fragmentCount` is the track's total after the append,
+     * not the number passed in.
+     *
+     * **Node only** (`write-full`). This takes every fragment's bytes as an
+     * argument, so peak memory is the whole clip; the native path form streams
+     * fragment-by-fragment off disk, which wasm32 cannot do because it has no
+     * filesystem. Fragmenting a multi-GB source inside a browser tab's 32-bit
+     * address space is not a workflow worth pretending to support.
+     */
+    /**
+     * **Node only.** Gated behind the `write-full` feature, which the browser
+     * and web builds are not compiled with: CMAF ingest takes the whole clip as
+     * one argument, so peak memory is the entire source. Calling it in a browser
+     * throws with an explanation — see the stub in browser-extras.mjs. Typed on
+     * both ends, like `Authoring`, so shared code type-checks once.
+     */
+    ingestCmaf(field: string, init: Uint8Array, fragments: CmafFragment[]): Promise<CmafIngestResult>;
     /**
      * Open a ref for writing.
      *
@@ -428,6 +562,13 @@ export function timeAnchorHex(value: bigint): string;
  */
 export function timeBucket(t_start: bigint, duration: string): bigint;
 
+/**
+ * Decode one spec/0025 item using the same declaration and payload path as
+ * `Space.readArrayColumn`, without requiring storage IO. This is the SDK's
+ * pure conformance entry point for language-neutral vectors.
+ */
+export function typedArrayDecode(item_type: any, payload: Uint8Array): any;
+
 /**
  * Package version — useful for consumers to confirm which build is loaded.
  */
@@ -444,6 +585,62 @@ export function zeroSpatialKey(): string;
 /** Zero spatial-key path segment for non-spatial (fragment) object addresses. */
 export const ZERO_SPATIAL_KEY: string
 
+/**
+ * One CMAF fragment for {@link Writer.ingestCmaf}: `[bytes, tStartNs, tEndNs)`,
+ * half-open, already offset to the timeline anchor you want.
+ *
+ * Pass `bigint`s. A `number` is accepted only while it is an exact
+ * non-negative integer at or below `Number.MAX_SAFE_INTEGER` and is rejected
+ * above it rather than rounded — wall-clock nanosecond anchors are ~1.7e18, and
+ * a rounded `tStart` files the fragment under a different 60 s bucket than the
+ * address it is indexed at.
+ */
+export type CmafFragment = [bytes: Uint8Array, tStartNs: bigint | number, tEndNs: bigint | number]
+
+/** What {@link Writer.ingestCmaf} resolves to. */
+export interface CmafIngestResult {
+  /** Schema field this Track materializes. */
+  field: string
+  /** Modality string, e.g. `video.h264`. */
+  modality: string
+  /** Base32 multihash of the initialization-segment object. */
+  initHash: string
+  /** Fragments in the Track *after* this call — not the number passed in. */
+  fragmentCount: number
+  /** Half-open `[tMin, tMax)` coverage of the whole Track, in ns. */
+  coverage: [bigint, bigint]
+  /** Base32 multihash of the new Track object. */
+  trackHash: string
+  /** Base32 multihash of the manifest this ingest published. */
+  manifestHash: string
+}
+
+/** Stable metadata for one logical VideoItem. All times are exact bigint ns. */
+export interface VideoItemInfo {
+  modality: string
+  itemKey: Uint8Array
+  tStartNs: bigint
+  durationNs: bigint
+  itemAddress: string
+  initAddress: string
+}
+
+/** One complete Fragment selected by {@link Space.readVideoItemRange}. */
+export interface VideoItemFragmentRead {
+  tStartNs: bigint
+  tEndNs: bigint
+  bytes: Uint8Array
+}
+
+/** Materialized initialization plus complete Fragments for one relative range. */
+export interface VideoItemRead {
+  item: VideoItemInfo
+  rangeStartNs: bigint
+  rangeEndNs: bigint
+  init: Uint8Array
+  fragments: VideoItemFragmentRead[]
+}
+
 /** A resolved manifest track (shape returned by `Space.tracks()`). */
 export interface ResolvedTrack {
   modality: string
@@ -475,11 +672,24 @@ export type PutStatus = 'created' | 'exists' | 'casFailed'
  * CORS rule; without it the browser cannot read the header at all.
  */
 export interface Backend {
-  /** Inclusive byte range. Returning a bare Uint8Array is the accepted v1 shape. */
+  /** Half-open byte range `[start, end)`. Returning a bare Uint8Array is the accepted whole-object v1 shape. */
   get(
     path: string,
     range?: { start: number; end: number },
-  ): Promise<Uint8Array | { bytes: Uint8Array; etag?: string }>
+  ): Promise<Uint8Array | { bytes: Uint8Array; etag?: string; totalLength?: number }>
+
+  /**
+   * Optional bounded full-object stream. Without this method reads remain
+   * functional, but oversized historical inline Tracks use the unbounded v1
+   * compatibility path.
+   */
+  getStream?(
+    path: string,
+  ): Promise<ReadableStream<Uint8Array> | {
+    stream: ReadableStream<Uint8Array>
+    etag?: string
+    totalLength?: number
+  }>
 
   head?(path: string): Promise<{ exists?: boolean; etag?: string; size?: number }>