@dreamlake/dreamdb 0.1.0

package/dist/index.d.cts

@@ -0,0 +1,498 @@
+/** 33-byte multihash, represented as Uint8Array on the wire. */
+type MultihashBytes = Uint8Array;
+/** Base32-encoded string form of a multihash. */
+type Base32Hash = string;
+/** Nanoseconds since UNIX epoch — the join key across all modalities. */
+type TimeAnchor = number;
+type FieldKind = "image" | "audio" | "video" | "embedding" | "scalar";
+type ScalarValueType = "categorical" | "bool" | "int" | "float" | "string" | "timestamp";
+interface ImageFieldDef {
+    kind: "image";
+    name: string;
+    mime: string;
+    required: boolean;
+    chunkSize?: number;
+    packItems?: number;
+}
+interface AudioFieldDef {
+    kind: "audio";
+    name: string;
+    mime: string;
+    required: boolean;
+    chunkSize?: number;
+    packItems?: number;
+}
+interface VideoFieldDef {
+    kind: "video";
+    name: string;
+    mime: string;
+    required: boolean;
+    chunkSize?: number;
+    packItems?: number;
+}
+interface EmbeddingFieldDef {
+    kind: "embedding";
+    name: string;
+    dim: number;
+    algorithm: string;
+    required: boolean;
+    lshBits?: number;
+    compressor?: MultihashBytes;
+    spatialIndex?: MultihashBytes;
+    rerank: boolean;
+}
+interface ScalarFieldDef {
+    kind: "scalar";
+    name: string;
+    valueType: ScalarValueType;
+    required: boolean;
+}
+type FieldDef = ImageFieldDef | AudioFieldDef | VideoFieldDef | EmbeddingFieldDef | ScalarFieldDef;
+interface ManifestTrack {
+    modality: string;
+    kind: string;
+    role: string;
+    address: MultihashBytes;
+    timeline: MultihashBytes;
+    coverage: unknown;
+}
+interface Manifest {
+    timelines: MultihashBytes[];
+    tracks: ManifestTrack[];
+    parents?: MultihashBytes[];
+    ts?: number | bigint;
+    writer?: string;
+    registry?: Record<string, RegistryEntry>;
+}
+interface RegistryEntry {
+    spatial_index?: MultihashBytes[];
+    spatial_index_hashes?: MultihashBytes[];
+    vector_compressor?: MultihashBytes;
+    [key: string]: unknown;
+}
+interface ResolvedTrack {
+    modality: string;
+    field: string | null;
+    key: string;
+    kind: string;
+    role: string;
+    address: Base32Hash;
+    timeline: Base32Hash;
+    coverage: unknown;
+}
+interface VectorQueryOptions {
+    topK?: number;
+    probeCount?: number;
+    maxHamming?: number;
+}
+interface VectorQueryResult {
+    anchor: TimeAnchor;
+    score: number;
+}
+interface QueryStats {
+    bucketsTotal: number;
+    bucketsProbed: number;
+    fellBackToBruteForce: boolean;
+    candidatesScored?: number;
+    topT?: number;
+    rerankFetchCount?: number;
+    rerankSkippedNoHash?: number;
+}
+interface HistoryEntry {
+    manifestHash: Base32Hash;
+    ts: number;
+    writer: string;
+    tracks: Array<{
+        modality: string;
+        address: Base32Hash;
+    }>;
+    parents: number;
+    truncatedAt?: string;
+}
+interface Batch {
+    [fieldName: string]: unknown[];
+    _time_anchors: TimeAnchor[];
+}
+interface BackendGetOptions {
+    noCache?: boolean;
+}
+interface BackendPutOptions {
+    contentType?: string;
+}
+/**
+ * Abstract storage backend. All connector implementations must fulfill
+ * this interface. GET is required; PUT is needed only for write paths.
+ */
+interface Backend {
+    /** Fetch bytes at `path` relative to the backend root. */
+    get(path: string, opts?: BackendGetOptions): Promise<Uint8Array>;
+    /** Store bytes at `path`. Returns the path written. */
+    put(path: string, data: Uint8Array, opts?: BackendPutOptions): Promise<string>;
+    /** List keys under `prefix`. */
+    list(prefix: string): Promise<string[]>;
+    /** Delete a key. */
+    delete(path: string): Promise<void>;
+}
+type SampleValue = Uint8Array | number[] | Float32Array | string | number | boolean | null;
+type Sample = {
+    [fieldName: string]: SampleValue;
+};
+interface DatasetInfo {
+    refName: string;
+    manifest: Base32Hash;
+    timeline: Base32Hash;
+}
+interface SnapshotInfo {
+    label: string;
+    manifest: Base32Hash;
+    timeline: Base32Hash;
+}
+type WhereClause = Record<string, string | number | boolean>;
+interface SpatialDispatcher {
+    algorithm: string;
+    dim: number;
+    bits: number;
+    k?: number;
+    probeKeys(qNorm: Float32Array, probeCount: number, maxHamming: number): string[];
+}
+interface VectorCompressorDecoder {
+    algorithm: string;
+    dim: number;
+    recordCodeBytes: number;
+    decode(codes: Uint8Array): Float32Array;
+    rotateQuery?(qNorm: Float32Array): Float32Array;
+    adcScore?(rotatedQuery: Float32Array, codes: Uint8Array): number;
+}
+interface BucketRecord {
+    anchor: TimeAnchor;
+    ordinal: number;
+    vec?: Float32Array;
+    codes?: Uint8Array;
+}
+type CborValue = number | bigint | string | boolean | null | undefined | Uint8Array | CborValue[] | {
+    [key: string]: CborValue;
+};
+declare const SCALAR_ENCODING_MAP: Record<string, string>;
+
+/**
+ * CBOR decoder — handles the exact DreamDB/Vortex subset:
+ *   Major type 0: unsigned integers (u8/u16/u32/u64)
+ *   Major type 1: negative integers
+ *   Major type 2: byte strings
+ *   Major type 3: text strings
+ *   Major type 4: arrays
+ *   Major type 5: maps
+ *   Major type 7: simple values (false, true, null, undefined) + IEEE-754 floats
+ *
+ * Port of the hand-rolled decoder from demo/dreamdb.js.
+ */
+
+/** Decode a single CBOR value from the front of `bytes`. */
+declare function decodeCbor(bytes: Uint8Array): CborValue;
+/**
+ * Encode a CBOR value. Supports the same subset the decoder handles:
+ * unsigned ints, negative ints, byte strings, text strings, arrays,
+ * maps, booleans, null, undefined, and f64 floats.
+ */
+declare function encodeCbor(value: CborValue): Uint8Array;
+
+/**
+ * DreamDB URI parsing and Base32 encoding/decoding.
+ *
+ * A "DreamDB Space URI" is the absolute HTTP URL of either a `refs/<name>`
+ * or `manifests/<hash>` Object on the connector backend:
+ *
+ *   http://localhost:9000/ucf101-mp4/refs/ucf101-mp4
+ *   http://localhost:9000/ucf101-mp4/manifests/dytslnbcty5b...
+ */
+
+declare const MULTIHASH_BYTES = 33;
+/** Encode raw bytes to lowercase base32 (RFC 4648, no padding). */
+declare function bytesToBase32(bytes: Uint8Array): Base32Hash;
+/** Decode a base32 string back to raw bytes. */
+declare function base32ToBytes(str: string): Uint8Array;
+interface ParsedSpaceUri {
+    connectorBase: string;
+    kind: "ref" | "manifest";
+    id: string;
+}
+/**
+ * Split a Space URI into { connectorBase, kind, id }.
+ *
+ *   "https://host/bucket/refs/foo"        → { connectorBase: "https://host/bucket", kind: "ref", id: "foo" }
+ *   "https://host/bucket/manifests/dXYZ"  → { connectorBase: "https://host/bucket", kind: "manifest", id: "dXYZ" }
+ */
+declare function parseSpaceUri(uri: string): ParsedSpaceUri;
+/** Map a scalar value_type to its modality encoding fragment. */
+declare function scalarModalityFragment(valueType: string): string;
+/**
+ * Given a Schema field definition, compute the expected Track modality string.
+ * Mirrors `scalar_modality_for` in dreamdb-dataset/src/dataset/util.rs.
+ */
+declare function modalityForField(field: {
+    kind: string;
+    mime?: string;
+    dim?: number;
+    name?: string;
+    value_type?: string;
+    valueType?: string;
+}): string | null;
+
+/**
+ * Backend implementations for DreamDB storage.
+ *
+ * - S3Backend: fetch-based HTTP backend for S3-compatible storage (MinIO, R2, etc.)
+ * - MemoryBackend: in-process Map-based backend for tests and ephemeral use
+ */
+
+/**
+ * S3Backend talks to an S3-compatible HTTP endpoint via `fetch`.
+ * Works in both browser and Node 18+.
+ *
+ * The `baseUrl` is the connector base URL, e.g. "http://localhost:9000/my-bucket".
+ * All paths are joined relative to this base.
+ */
+declare class S3Backend implements Backend {
+    readonly baseUrl: string;
+    constructor(baseUrl: string);
+    get(path: string, opts?: BackendGetOptions): Promise<Uint8Array>;
+    put(path: string, data: Uint8Array, opts?: BackendPutOptions): Promise<string>;
+    list(prefix: string): Promise<string[]>;
+    delete(path: string): Promise<void>;
+}
+/**
+ * MemoryBackend stores all Objects in a Map<string, Uint8Array>.
+ * Useful for tests and ephemeral datasets.
+ */
+declare class MemoryBackend implements Backend {
+    private store;
+    get(path: string, _opts?: BackendGetOptions): Promise<Uint8Array>;
+    put(path: string, data: Uint8Array, _opts?: BackendPutOptions): Promise<string>;
+    list(prefix: string): Promise<string[]>;
+    delete(path: string): Promise<void>;
+    /** Check whether a key exists. */
+    has(path: string): boolean;
+    /** Get the number of stored objects. */
+    get size(): number;
+    /** Clear all stored objects. */
+    clear(): void;
+}
+
+/**
+ * Space class — resolve refs, decode manifests, iterate tracks.
+ *
+ * Port of the DreamDBSpace class from demo/dreamdb.js.
+ * Uses the Backend interface for storage access.
+ */
+
+interface SpaceOptions {
+    connectorBase?: string;
+    backend?: Backend;
+}
+declare class Space {
+    readonly connectorBase: string;
+    readonly manifestHash: Base32Hash;
+    readonly manifest: Manifest;
+    readonly refName: string | null;
+    private backend;
+    private _siCache;
+    private _vcCache;
+    private _lastQueryStats;
+    constructor(opts: {
+        connectorBase: string;
+        manifestHash: Base32Hash;
+        manifest: Manifest;
+        refName: string | null;
+        backend?: Backend;
+    });
+    /**
+     * Open a Space from a space URI.
+     * If a Backend is provided, it is used; otherwise fetch-based HTTP access.
+     */
+    static fromUri(uri: string, backend?: Backend): Promise<Space>;
+    /** Get the timeline ID (base32 hash of the first timeline entry). */
+    timelineId(): Base32Hash;
+    /**
+     * Map each Manifest Track to the Schema-declared field name.
+     * Returns a parallel array to manifest.tracks.
+     */
+    private _fieldNamesForTracks;
+    /** List resolved tracks from this Manifest. */
+    tracks(): ResolvedTrack[];
+    /** Find a Track by key (field name or modality). */
+    trackFor(keyOrModality: string): ResolvedTrack | undefined;
+    /** Walk the Manifest DAG and return history entries. */
+    history(maxDepth?: number): Promise<HistoryEntry[]>;
+    /** Get the last query's statistics. */
+    get lastQueryStats(): QueryStats | null;
+    queryVector(field: string, queryVec: number[] | Float32Array, opts?: VectorQueryOptions | number): Promise<VectorQueryResult[]>;
+    private _embeddingTrackFor;
+    private _fetchBytes;
+    private _fetchSpatialDispatcher;
+    private _fetchVectorCompressor;
+}
+
+/**
+ * Vector query, LSH dispatch, IVF dispatch, ChaCha20 hyperplane derivation,
+ * PQ/RaBitQ decode + rerank.
+ *
+ * Port of the spatial-indexing and vector-scoring logic from demo/dreamdb.js.
+ */
+
+interface ChaCha20Stream {
+    fill(out: Uint8Array): void;
+}
+/** Streaming ChaCha20 keystream from a 32-byte seed. */
+declare function chacha20Stream(seedBytes32: Uint8Array): ChaCha20Stream;
+/** f32 left-fold L2-normalize (matches Rust's normalize_l2). */
+declare function normalizeL2F32(v: Float32Array | number[]): Float32Array | null;
+/** f64 L2-normalize (higher precision, for rerank path). */
+declare function normalizeL2(v: Float32Array | number[]): Float32Array;
+/** f32 left-fold dot product (matches Rust's dot_product_left_fold). */
+declare function dotF32(a: Float32Array, b: Float32Array): number;
+/** f64 dot product of L2-normalized vectors. */
+declare function dotL2Normalized(a: Float32Array, b: Float32Array): number;
+/**
+ * Derive `bits` hyperplanes of `dim` f32 values each, from a 32-byte seed.
+ * Mirrors `derive_hyperplanes` in dreamdb-protocol/src/lsh_cosine.rs.
+ */
+declare function deriveHyperplanesLshCosine(seedBytes32: Uint8Array, dim: number, bits: number): Float32Array;
+declare function projectionsLshCosine(qNormalized: Float32Array, hyperplanes: Float32Array, dim: number, bits: number): Float32Array;
+declare function bitsFromProjections(projections: Float32Array): boolean[];
+declare function bitsToBase2(bits: boolean[]): string;
+/**
+ * Enumerate up to `probeCount` keys (including primary) ranked by
+ * smallest total |projection| flip-cost.
+ */
+declare function multiProbeKeys(projections: Float32Array, probeCount: number, maxHamming: number): string[];
+/**
+ * Derive the dim x dim orthogonal rotation matrix from a 32-byte seed.
+ * Uses Modified Gram-Schmidt. Returns Float32Array of length dim*dim, row-major.
+ */
+declare function deriveRotationMatrix(seedBytes32: Uint8Array, dim: number): Float32Array;
+declare function buildPqCosineDecoder(vcCborBytes: Uint8Array): VectorCompressorDecoder;
+declare function buildRabitqCosineDecoder(vcCborBytes: Uint8Array): VectorCompressorDecoder;
+/**
+ * Decode a SpatialBucket binary into BucketRecord[].
+ * Supports v1 (160-byte header, raw f32) and v2 (200-byte header, compressed).
+ */
+declare function decodeBucketRecords(bytes: Uint8Array, expectRecordSize: number, dim: number): BucketRecord[];
+/** Decode only the time anchors from a SpatialBucket binary. */
+declare function decodeBucketAnchorsOnly(bytes: Uint8Array): number[];
+/** Peek the bucket header without decoding records. */
+declare function peekBucketHeader(bytes: Uint8Array): {
+    recordSize: number;
+    recordCount: number;
+    headerSize: number;
+    isCompressed: boolean;
+    vectorCompressorHash: Uint8Array | null;
+};
+
+/**
+ * High-level Dataset API — mirrors the Python SDK's Dataset/Schema surface.
+ *
+ * Provides:
+ *   - Schema (add_image, add_embedding, add_scalar_categorical, etc.)
+ *   - Dataset.create / Dataset.open
+ *   - append_many, iter_vector, iter_stream
+ *   - snapshot, branch, merge, delete, tombstone_set
+ *   - history, list_refs, count
+ */
+
+/**
+ * Builder for a Dataset's schema. Chainable API:
+ *
+ *   const schema = new Schema()
+ *     .addImage("image", { mime: "jpeg" })
+ *     .addEmbedding("embedding", { dim: 512 })
+ *     .addScalarCategorical("label");
+ */
+declare class Schema {
+    readonly fields: FieldDef[];
+    addImage(name: string, opts?: {
+        mime?: string;
+        required?: boolean;
+        chunkSize?: number;
+        packItems?: number;
+    }): this;
+    addAudio(name: string, opts?: {
+        mime?: string;
+        required?: boolean;
+        chunkSize?: number;
+        packItems?: number;
+    }): this;
+    addVideo(name: string, opts?: {
+        mime?: string;
+        required?: boolean;
+        chunkSize?: number;
+        packItems?: number;
+    }): this;
+    addEmbedding(name: string, opts: {
+        dim: number;
+        algorithm?: string;
+        required?: boolean;
+        lshBits?: number;
+        compressor?: Uint8Array;
+        spatialIndex?: Uint8Array;
+        rerank?: boolean;
+    }): this;
+    addScalarCategorical(name: string, opts?: {
+        required?: boolean;
+    }): this;
+    addScalarBool(name: string, opts?: {
+        required?: boolean;
+    }): this;
+    addScalarInt(name: string, opts?: {
+        required?: boolean;
+    }): this;
+    addScalarFloat(name: string, opts?: {
+        required?: boolean;
+    }): this;
+    addScalarString(name: string, opts?: {
+        required?: boolean;
+    }): this;
+    addScalarTimestamp(name: string, opts?: {
+        required?: boolean;
+    }): this;
+    /** Serialize the schema to CBOR-encodable object. */
+    toCbor(): CborValue;
+}
+/**
+ * Versioned multimodal Dataset.
+ *
+ * TypeScript SDK mirror of the Python vortex_dataset.Dataset surface.
+ */
+declare class Dataset {
+    private backend;
+    private _refName;
+    private _manifestHash;
+    private _timelineHash;
+    private _schema;
+    private constructor();
+    static create(name: string, schema: Schema, backendUrl: string): Promise<Dataset>;
+    static open(name: string, schema?: Schema | null, backendUrl?: string): Promise<Dataset>;
+    refName(): string;
+    currentManifest(): Base32Hash;
+    timeline(): Base32Hash;
+    schema(): Schema;
+    history(maxDepth?: number): Promise<Array<{
+        manifest: Base32Hash;
+        ts_ns: number;
+        writer: string;
+        parents_count: number;
+        tracks_count: number;
+    }>>;
+    listRefs(): Promise<string[]>;
+    count(): Promise<number>;
+    appendMany(samples: Sample[]): Promise<number>;
+    iterVector(field: string, query: number[] | Float32Array, topK: number, batchSize?: number, opts?: {
+        whereEq?: WhereClause;
+    }): Promise<Batch[]>;
+    private _iterStreamImpl;
+    iterStream(batchSize?: number): Promise<Batch[]>;
+    snapshot(label: string): Promise<SnapshotInfo>;
+    branch(newName: string): Promise<Dataset>;
+    private _resolveScalarLabels;
+}
+
+export { type AudioFieldDef, type Backend, type BackendGetOptions, type BackendPutOptions, type Base32Hash, type Batch, type BucketRecord, type CborValue, type ChaCha20Stream, Dataset, type DatasetInfo, type EmbeddingFieldDef, type FieldDef, type FieldKind, type HistoryEntry, type ImageFieldDef, MULTIHASH_BYTES, type Manifest, type ManifestTrack, MemoryBackend, type MultihashBytes, type ParsedSpaceUri, type QueryStats, type RegistryEntry, type ResolvedTrack, S3Backend, SCALAR_ENCODING_MAP, type Sample, type SampleValue, type ScalarFieldDef, type ScalarValueType, Schema, type SnapshotInfo, Space, type SpaceOptions, type SpatialDispatcher, type TimeAnchor, type VectorCompressorDecoder, type VectorQueryOptions, type VectorQueryResult, type VideoFieldDef, type WhereClause, base32ToBytes, bitsFromProjections, bitsToBase2, buildPqCosineDecoder, buildRabitqCosineDecoder, bytesToBase32, chacha20Stream, decodeBucketAnchorsOnly, decodeBucketRecords, decodeCbor, deriveHyperplanesLshCosine, deriveRotationMatrix, dotF32, dotL2Normalized, encodeCbor, modalityForField, multiProbeKeys, normalizeL2, normalizeL2F32, parseSpaceUri, peekBucketHeader, projectionsLshCosine, scalarModalityFragment };