@dreamlake/dreamdb 0.5.1 → 0.5.3

package/index.d.ts

@@ -23,6 +23,14 @@ export class Authoring {
     private constructor();
     free(): void;
     [Symbol.dispose](): void;
+    /**
+     * Add one fixed-shape typed-array Constant.
+     *
+     * `field` is the same `{name, kind:'array', ...}` descriptor accepted by
+     * `create`, with `trackKind:'constant'` and `objectKind:'constant'`.
+     * `payload` is a raw or NPY payload matching that declaration.
+     */
+    addArrayConstant(field: any, payload: Uint8Array): Promise<void>;
     /**
      * Add an embedding column whose index artifacts already exist.
      *
@@ -38,7 +46,7 @@ export class Authoring {
      * off, reads decode 1-bit reconstructions and relevance degrades in a way
      * no API-level check can see.
      */
-    addEmbeddingLayer(name: string, parent_field: string, dim: number, algorithm: string, spatial_index: string, compressor: string, rerank: boolean, version: number | null | undefined, samples: Array<any>): Promise<void>;
+    addEmbeddingLayer(name: string, parent_field: string, dim: number, algorithm: string, spatial_index: string, compressor: string, rerank: boolean, version: number | null | undefined, samples: Array<any>, bucket_max_bytes?: number | null, embed_spec?: Uint8Array | null): Promise<void>;
     /**
      * Add a field to the schema of an existing dataset.
      */
@@ -56,10 +64,20 @@ export class Authoring {
      * `int|float|bool|string|categorical|timestamp`.
      */
     addScalarLayer(name: string, parent_field: string, value_type: string, samples: Array<any>): Promise<void>;
+    /**
+     * Append `[anchor, vector]` pairs, adopting a raw graph as Fresh Vamana.
+     */
+    appendGraphNodes(field: string, samples: Array<any>): Promise<any>;
     /**
      * Fork the current manifest to a new ref name.
      */
     branch(new_name: string): Promise<Authoring>;
+    /**
+     * Merge one or more branches into this ref. Returns the new manifest hash.
+     * Build a raw Vamana field. Samples are `[anchor, vector]` pairs; anchors
+     * use the same lossless representation as addEmbeddingLayer.
+     */
+    buildGraphIndex(name: string, parent_field: string, dim: number, samples: Array<any>, r: number, seed: Uint8Array, embed_spec?: Uint8Array | null): Promise<void>;
     /**
      * Collapse per-cell fragments. Returns
      * `{cellsExamined, cellsCompacted, fragmentsCollapsed, manifest?}`.
@@ -70,6 +88,15 @@ export class Authoring {
      * nothing needed compacting — a no-op, not a failure.
      */
     compact(modality: string | null | undefined, threshold: number, max_cells: number): Promise<any>;
+    /**
+     * Compress a raw graph with canonical VectorCompressor CBOR. The raw
+     * lineage is retained; rerank selects exact scoring of the candidate pool.
+     */
+    compressGraphIndex(field: string, compressor: Uint8Array, rerank: boolean): Promise<void>;
+    /**
+     * Caller-driven region consolidation; does not erase tombstones or ids.
+     */
+    consolidateGraph(field: string): Promise<any>;
     /**
      * Create a new dataset and publish `refs/<name>`.
      *
@@ -93,9 +120,6 @@ export class Authoring {
      * Ref names under `refs/`. Requires a backend implementing `list`.
      */
     listRefs(): Promise<string[]>;
-    /**
-     * Merge one or more branches into this ref. Returns the new manifest hash.
-     */
     mergeMany(branches: string[]): Promise<string>;
     /**
      * Open an existing ref for authoring.
@@ -115,6 +139,22 @@ export class Authoring {
     readonly refName: string;
 }
 
+export class GeometryReader {
+    private constructor();
+    free(): void;
+    [Symbol.dispose](): void;
+    /**
+     * Returns only the new suffix at this total logical byte budget.
+     */
+    extendCellPrefix(key: any, budget: any): Promise<Uint8Array>;
+    /**
+     * Declaration plus cells/LODs; no payload transfer. u64 values are BigInt.
+     */
+    metadata(): any;
+    readMeshLod(level: any): Promise<Uint8Array>;
+    selectCells(min: any, max: any): Array<any>;
+}
+
 /**
  * A `fetch`-backed backend rooted at a base URL.
  */
@@ -127,6 +167,15 @@ export class S3Backend {
      * content is hash-addressed; refs are read fresh each page load anyway).
      */
     get(path: string, _opts: any): Promise<Uint8Array>;
+    /**
+     * Stream `path` without first aggregating the response in wasm memory.
+     *
+     * Returning the length beside the browser stream satisfies the optional
+     * `Backend.getStream` contract. Older injected backends that implement
+     * only [`Self::get`] remain compatible, but cannot claim bounded memory
+     * for historical oversized inline Track reads.
+     */
+    getStream(path: string): Promise<any>;
     /**
      * List object paths under `prefix`. Stub (returns empty); the current
      * consumer read paths don't require listing.
@@ -142,6 +191,41 @@ export class Space {
     private constructor();
     free(): void;
     [Symbol.dispose](): void;
+    /**
+     * Validate all referenced Items, including tombstoned payloads.
+     */
+    auditArrayField(field: string): Promise<bigint>;
+    /**
+     * Validate all graph pages and, when declared, exact-source correspondence.
+     * The returned count is exactly representable (GraphIndex node_count is u32).
+     */
+    auditGraphIndex(field: string): Promise<number>;
+    /**
+     * The Space's human-readable description, or `undefined` when none is set.
+     *
+     * The read half of `Authoring.setDescription` (#129). It lives here rather
+     * than on `Authoring` because that type is `write-full` and so exists only
+     * in the Node build — a description is metadata a READER wants, and
+     * `Space` is the surface all three build targets share.
+     *
+     * `async` although `Dataset::description` is not: the value comes from the
+     * Dataset this Space opens lazily at the Manifest hash already pinned by
+     * this Space, so the first call may perform that open without re-resolving
+     * a source Ref.
+     *
+     * Deliberately routed through `Dataset::description` rather than read out
+     * of `self.manifest`'s registry, which would be shorter and would also
+     * serve the cached decoded Manifest. `Dataset::recover_meta` is not a prefix
+     * filter: it accepts the legacy `vortex.*` namespace as well as
+     * `dreamdb.*`, canonicalises, and drops structural keys. Matching
+     * `dreamdb.description` here would silently return nothing for a legacy
+     * Space whose description is on disk and readable from Rust — the exact
+     * defect `recover_meta`'s own comment records.
+     *
+     * `Option<String>` marshals to `string | undefined`, NOT to `""`. An unset
+     * description and an empty one stay distinguishable across the boundary.
+     */
+    description(): Promise<string | undefined>;
     /**
      * Open a Space from a `.../refs/<name>` or `.../manifests/<hash>` URI,
      * resolving the ref (if any) and loading the manifest. If `backend` is
@@ -150,11 +234,21 @@ export class Space {
      * no-backend path).
      */
     static fromUri(uri: string, backend: any): Promise<Space>;
+    /**
+     * Read at this pinned Manifest. Returns null for never-created keys;
+     * deleted keys retain metadata with sample=null. Integer keys and anchors
+     * are BigInt; sample values use the Writer's tagged field shapes.
+     */
+    getEntity(key: any): Promise<any>;
     /**
      * Walk the manifest DAG from HEAD, up to `max_depth` entries. Each entry:
      * `{ manifestHash, ts, writer, tracks: [{modality, address}] }`.
      */
     history(max_depth?: number | null): Promise<Array<any>>;
+    /**
+     * Open one pinned geometry Item. Missing/deleted anchors return undefined.
+     */
+    openGeometryItem(field: string, anchor: any): Promise<GeometryReader | undefined>;
     /**
      * Hybrid search fusing a lexical (BM25) sub-query and a dense (vector)
      * sub-query per spec/0015 §5/§6. Returns the fused `[{ anchor, score }]`
@@ -177,10 +271,17 @@ export class Space {
      * ```
      *
      * Delegates to the real (tested) `dreamdb-dataset::Dataset::query_hybrid`.
-     * (v0 does not expose the optional scalar pre/post-filter; it runs with no
-     * scalar gate.)
+     * Optional `scalarField` / `scalarOp` / `scalarValue` properties select
+     * the same scalar pre/post-filter planner used by the native API.
      */
     queryHybrid(vector: Float32Array, opts: any): Promise<Array<any>>;
+    /**
+     * Hybrid query with an observable `{hits, trace}` result. `opts` accepts
+     * `maxLatencyMs`, `costModel`, and the optional scalar triple
+     * `scalarField`/`scalarOp`/`scalarValue` in addition to `queryHybrid`'s
+     * existing properties.
+     */
+    queryHybridPlanned(vector: Float32Array, opts: any): Promise<any>;
     /**
      * Scalar-index lookup: every anchor whose `field` satisfies `op value`
      * (spec/0011). Returns a `BigUint64Array`-style `Array` of anchors,
@@ -221,18 +322,60 @@ export class Space {
      * two-pass rerank + HotShard merge + tombstone suppression.
      */
     queryVector(field: string, query: Float32Array, opts: any): Promise<Array<any>>;
+    /**
+     * Read an Event/Unbucketed or Continuous/TimeBatch typed-array field as
+     * `Map<anchorString, {values, shape, ...}>`.
+     */
+    readArrayColumn(field: string): Promise<Map<any, any>>;
+    /**
+     * Read a typed-array Constant and decode it to a JS TypedArray plus its
+     * authoritative shape and semantic metadata.
+     */
+    readArrayConstant(field: string): Promise<any>;
+    /**
+     * Point read; full-u64 BigInt anchors are supported. Returned components
+     * are owned typed arrays (one Item's bytes), never unaligned aliases.
+     */
+    readArrayItem(field: string, anchor: any): Promise<any>;
     /**
      * Read a scalar track as a `Map<anchorString, value>`. Single-anchor
      * entries resolve from the track index; multi-anchor entries fetch (and
      * de-dupe) their bucket and roaring-decode the anchor set.
      */
     readScalarColumn(track: any, _opts: any): Promise<Map<any, any>>;
+    /**
+     * Fetch verified initialization and complete Fragments overlapping one
+     * item-relative half-open range. This materializes the selected bytes; it
+     * is not a streaming API.
+     */
+    readVideoItemRange(field: string, itemKey: Uint8Array, relativeStartNs: bigint | number, relativeEndNs: bigint | number): Promise<VideoItemRead | undefined>;
     /**
      * Fetch a track object and return its `object_index` (array of CBOR
      * entries). Inline indexes only — paged indexes are not resolved (matching
      * dreamdb-ts, which also doesn't support them on this path).
      */
     resolveObjectIndex(track: any): Promise<Array<any>>;
+    /**
+     * Semantic-cache capacity in bytes (spec/0006 §3.6). Default 1 GiB.
+     */
+    semanticCacheCapacity(): Promise<number>;
+    /**
+     * Semantic-cache occupancy as `{ entries, bytesUsed, maxBytes }`.
+     */
+    semanticCacheStats(): Promise<object>;
+    /**
+     * Set the semantic-cache capacity in bytes, evicting down to it before
+     * returning. Returns the bytes evicted. `0` disables the cache.
+     *
+     * An index whose accounted size exceeds the cap is not cached and is
+     * re-fetched on every query, reported honestly as a miss. The default is
+     * not raised globally to suit one corpus.
+     *
+     * Sizing is workload- and architecture-dependent, and figures measured on
+     * a 64-bit host do NOT carry over here: `usize` is 32 bits on `wasm32`, so
+     * every offset and length inside a decoded index accounts differently.
+     */
+    setSemanticCacheCapacity(n_bytes: number): Promise<number>;
     /**
      * Base32 hash of the manifest's first timeline.
      */
@@ -245,6 +388,16 @@ export class Space {
      * List resolved tracks from this manifest.
      */
     tracks(): Array<any>;
+    /**
+     * Look up the logical VideoItem containing one absolute Timeline anchor
+     * without fetching media. Exact nanosecond values are returned as bigint.
+     */
+    videoItemAt(field: string, anchorNs: bigint | number): Promise<VideoItemInfo | undefined>;
+    /**
+     * Look up one logical VideoItem by its stable opaque key without fetching
+     * initialization or Fragment bytes.
+     */
+    videoItemByKey(field: string, itemKey: Uint8Array): Promise<VideoItemInfo | undefined>;
     /**
      * The base URL objects are fetched relative to (consumers build their own
      * object URLs from this — e.g. `dreamdb-demo`'s track readers).
@@ -263,6 +416,11 @@ export class Writer {
     private constructor();
     free(): void;
     [Symbol.dispose](): void;
+    /**
+     * Atomically hot-append a complete Sample batch. Embeddings retain v1;
+     * scalar/text values use v2. Identified embeddings require embeddingSpecs.
+     */
+    appendHot(samples: Array<any>, embedding_specs?: object | null): Promise<number>;
     /**
      * Append records and commit in one call.
      *
@@ -276,6 +434,11 @@ export class Writer {
      * lifetime is under the caller's control.
      */
     appendMany(samples: Array<any>): Promise<number>;
+    /**
+     * Append records with a field-to-`spec_id` map for identified embeddings.
+     * Values are full canonical base32 multihashes.
+     */
+    appendManyWithSpecs(samples: Array<any>, embedding_specs: object): Promise<number>;
     /**
      * Flush staged entries and publish a new manifest.
      *
@@ -285,10 +448,74 @@ export class Writer {
      * its records on top of the new head is correct.
      */
     commit(): Promise<string>;
+    /**
+     * Producer policy only: TTL is evaluated on append, not by a timer.
+     */
+    configureHotShard(flush_threshold: number, ttl_seconds: number): void;
+    /**
+     * Create a never-used key. Sample uses appendMany's tagged fields, no anchor.
+     */
+    createEntity(key: any, sample: object, embedding_specs?: object | null): Promise<object>;
+    /**
+     * Delete exactly a live revision, retaining its key and stable identity.
+     */
+    deleteEntity(key: any, expected: Uint8Array): Promise<object>;
     /**
      * Tombstone records by anchor. Returns the new manifest hash.
      */
     deleteRecords(anchors: BigUint64Array, reason?: string | null): Promise<string>;
+    /**
+     * Enable one exact typed entity namespace: string, bytes or int.
+     */
+    enableEntityKeys(key_kind: string): Promise<void>;
+    /**
+     * Explicitly fold every pending hot field into cold Tracks in one publish.
+     */
+    flushHot(): Promise<void>;
+    /**
+     * Ingest a pre-fragmented CMAF (fragmented-MP4) video into a Video field
+     * as a `spec/0007` §5 media Track — the thing that makes a video Track
+     * time-seekable and MSE-playable rather than one opaque blob.
+     *
+     * ```js
+     * const out = await writer.ingestCmaf('video', initBytes, [
+     *   [frag0, 0n, 2_000_000_000n],
+     *   [frag1, 2_000_000_000n, 4_000_000_000n],
+     * ])
+     * ```
+     *
+     * - `init` is the initialization segment (`ftyp`+`moov`).
+     * - each fragment is `[Uint8Array, tStartNs, tEndNs]`, half-open, already
+     *   offset to the timeline anchor the caller wants. Pass `bigint`s:
+     *   wall-clock nanosecond anchors are far past `Number.MAX_SAFE_INTEGER`
+     *   and a rounded `tStart` writes the fragment into a 60 s bucket that
+     *   does not match the address it is indexed at.
+     *
+     * Calling it again on the SAME field APPENDS (one episode per call on a
+     * shared timeline) and reuses the track's existing init segment; a clip
+     * whose init differs — different codec, resolution, fps or audio config —
+     * is refused, because a Track carries exactly one init segment and the
+     * mismatch would produce fragments that decode against the wrong one.
+     *
+     * Returns `{field, modality, initHash, fragmentCount, coverage: [bigint,
+     * bigint], trackHash, manifestHash}`. `coverage` is `bigint` for the
+     * reason above; `fragmentCount` is the track's total after the append,
+     * not the number passed in.
+     *
+     * **Node only** (`write-full`). This takes every fragment's bytes as an
+     * argument, so peak memory is the whole clip; the native path form streams
+     * fragment-by-fragment off disk, which wasm32 cannot do because it has no
+     * filesystem. Fragmenting a multi-GB source inside a browser tab's 32-bit
+     * address space is not a workflow worth pretending to support.
+     */
+    /**
+     * **Node only.** Gated behind the `write-full` feature, which the browser
+     * and web builds are not compiled with: CMAF ingest takes the whole clip as
+     * one argument, so peak memory is the entire source. Calling it in a browser
+     * throws with an explanation — see the stub in browser-extras.mjs. Typed on
+     * both ends, like `Authoring`, so shared code type-checks once.
+     */
+    ingestCmaf(field: string, init: Uint8Array, fragments: CmafFragment[]): Promise<CmafIngestResult>;
     /**
      * Open a ref for writing.
      *
@@ -303,10 +530,27 @@ export class Writer {
      * until it calls it.
      */
     static open(uri: string, backend: any): Promise<Writer>;
+    /**
+     * Publish prequantized fixed-stride records. Coordinates is
+     * {origin:[x,y,z],step:[x,y,z],units:string}; depth is 0..16.
+     */
+    publishGeometryCells(field: string, anchor: any, coordinates: any, depth: any, records: Uint8Array): Promise<string>;
+    /**
+     * Each level is {errorGrid,vertices,triangles,bytes:Uint8Array}.
+     */
+    publishGeometryMesh(field: string, anchor: any, coordinates: any, levels: Array<any>): Promise<string>;
     /**
      * Tag the current manifest with an immutable label (`refs/<ref>@<label>`).
      */
     snapshot(label: string): Promise<string>;
+    /**
+     * Replace only a live revision, atomically hiding its old physical row.
+     */
+    supersedeEntity(key: any, sample: object, expected: Uint8Array, embedding_specs?: object | null): Promise<object>;
+    /**
+     * Create with no expected token, or replace/restore exactly that revision.
+     */
+    upsertEntity(key: any, sample: object, expected?: Uint8Array | null, embedding_specs?: object | null): Promise<object>;
     /**
      * Current manifest hash, base32.
      */
@@ -428,6 +672,13 @@ export function timeAnchorHex(value: bigint): string;
  */
 export function timeBucket(t_start: bigint, duration: string): bigint;
 
+/**
+ * Decode one spec/0025 item using the same declaration and payload path as
+ * `Space.readArrayColumn`, without requiring storage IO. This is the SDK's
+ * pure conformance entry point for language-neutral vectors.
+ */
+export function typedArrayDecode(item_type: any, payload: Uint8Array): any;
+
 /**
  * Package version — useful for consumers to confirm which build is loaded.
  */
@@ -444,6 +695,62 @@ export function zeroSpatialKey(): string;
 /** Zero spatial-key path segment for non-spatial (fragment) object addresses. */
 export const ZERO_SPATIAL_KEY: string
 
+/**
+ * One CMAF fragment for {@link Writer.ingestCmaf}: `[bytes, tStartNs, tEndNs)`,
+ * half-open, already offset to the timeline anchor you want.
+ *
+ * Pass `bigint`s. A `number` is accepted only while it is an exact
+ * non-negative integer at or below `Number.MAX_SAFE_INTEGER` and is rejected
+ * above it rather than rounded — wall-clock nanosecond anchors are ~1.7e18, and
+ * a rounded `tStart` files the fragment under a different 60 s bucket than the
+ * address it is indexed at.
+ */
+export type CmafFragment = [bytes: Uint8Array, tStartNs: bigint | number, tEndNs: bigint | number]
+
+/** What {@link Writer.ingestCmaf} resolves to. */
+export interface CmafIngestResult {
+  /** Schema field this Track materializes. */
+  field: string
+  /** Modality string, e.g. `video.h264`. */
+  modality: string
+  /** Base32 multihash of the initialization-segment object. */
+  initHash: string
+  /** Fragments in the Track *after* this call — not the number passed in. */
+  fragmentCount: number
+  /** Half-open `[tMin, tMax)` coverage of the whole Track, in ns. */
+  coverage: [bigint, bigint]
+  /** Base32 multihash of the new Track object. */
+  trackHash: string
+  /** Base32 multihash of the manifest this ingest published. */
+  manifestHash: string
+}
+
+/** Stable metadata for one logical VideoItem. All times are exact bigint ns. */
+export interface VideoItemInfo {
+  modality: string
+  itemKey: Uint8Array
+  tStartNs: bigint
+  durationNs: bigint
+  itemAddress: string
+  initAddress: string
+}
+
+/** One complete Fragment selected by {@link Space.readVideoItemRange}. */
+export interface VideoItemFragmentRead {
+  tStartNs: bigint
+  tEndNs: bigint
+  bytes: Uint8Array
+}
+
+/** Materialized initialization plus complete Fragments for one relative range. */
+export interface VideoItemRead {
+  item: VideoItemInfo
+  rangeStartNs: bigint
+  rangeEndNs: bigint
+  init: Uint8Array
+  fragments: VideoItemFragmentRead[]
+}
+
 /** A resolved manifest track (shape returned by `Space.tracks()`). */
 export interface ResolvedTrack {
   modality: string
@@ -475,11 +782,24 @@ export type PutStatus = 'created' | 'exists' | 'casFailed'
  * CORS rule; without it the browser cannot read the header at all.
  */
 export interface Backend {
-  /** Inclusive byte range. Returning a bare Uint8Array is the accepted v1 shape. */
+  /** Half-open byte range `[start, end)`. Returning a bare Uint8Array is the accepted whole-object v1 shape. */
   get(
     path: string,
     range?: { start: number; end: number },
-  ): Promise<Uint8Array | { bytes: Uint8Array; etag?: string }>
+  ): Promise<Uint8Array | { bytes: Uint8Array; etag?: string; totalLength?: number }>
+
+  /**
+   * Optional bounded full-object stream. Without this method reads remain
+   * functional, but oversized historical inline Tracks use the unbounded v1
+   * compatibility path.
+   */
+  getStream?(
+    path: string,
+  ): Promise<ReadableStream<Uint8Array> | {
+    stream: ReadableStream<Uint8Array>
+    etag?: string
+    totalLength?: number
+  }>
 
   head?(path: string): Promise<{ exists?: boolean; etag?: string; size?: number }>