@littlebigbrain/client 0.8.1 → 0.9.0

package/README.md

@@ -78,6 +78,20 @@ await graph.facts.import(
 );
 ```
 
+For large or long-running loads, stream records to a durable job instead:
+
+```ts
+const accepted = await lbb.submitImport(records(), {
+  idempotencyKey: "hubspot:portal-42:run-2026-07-29",
+});
+const completed = await lbb.waitForImportJob(accepted.job_id);
+console.log(completed.state, completed.committed_commit_seq);
+```
+
+`records()` may be an iterable or async iterable. Success means every grouped
+commit is durable and final publication was enqueued; it does not mean indexes
+have already reached `committed_commit_seq`.
+
 **Time-travel read.** Pin any search to a past instant — results reflect the graph as it was then:
 
 ```ts

package/dist/client.d.ts

@@ -1,9 +1,9 @@
-import type { ImportLine, LbbClientOptions, ListResponse, RawLbbResponse, ReadConsistencyOptions, RdfImportOptions, Schemas, SearchConsistency, SparqlResults } from "./types.js";
+import type { DurableImportSource, ImportLine, LbbClientOptions, ListResponse, RawLbbResponse, ReadConsistencyOptions, RdfImportOptions, Schemas, SearchConsistency, SparqlResults } from "./types.js";
 import { type CallOptions, type RequestOptions } from "./transport.js";
 import { ContextNamespace, EntityNamespace, GraphNamespace, OntologyNamespace, QueryNamespace, SchemaNamespace, SearchNamespace } from "./namespaces.js";
 export { parseSparqlResults } from "./types.js";
-export type { AttributeFilter, AttributeFilterOp, AttributeFilterValue, EntityAttributeFilterOptions, EntityPropertiesLine, FetchLike, FlatProperties, ImportLine, LbbClientOptions, LbbRequestEvent, LbbResponseEvent, LbbRetryEvent, LbbErrorPayload, ListResponse, RawLbbResponse, ReadConsistencyOptions, RdfImportOptions, Schemas, SearchConsistency, SparqlResults, SparqlResultsJson, SparqlTerm, CommitRequest, CommitResponse, Entity, EntitySelector, GraphMetadata, GraphSummary, SchemaView, SearchRequest, SearchResponse, SearchResult, Snapshot, } from "./types.js";
-export { LbbError } from "./transport.js";
+export type { AttributeFilter, AttributeFilterOp, AttributeFilterValue, EntityAttributeFilterOptions, EntityPropertiesLine, DurableImportLine, DurableImportSource, FetchLike, FlatProperties, ImportLine, LbbClientOptions, LbbRequestEvent, LbbResponseEvent, LbbRetryEvent, LbbErrorPayload, ListResponse, RawLbbResponse, ReadConsistencyOptions, RdfImportOptions, Schemas, SearchConsistency, SparqlResults, SparqlResultsJson, SparqlTerm, CommitRequest, CommitResponse, Entity, EntitySelector, GraphMetadata, GraphSummary, SchemaView, SearchRequest, SearchResponse, SearchResult, Snapshot, } from "./types.js";
+export { LbbCapabilityError, LbbError } from "./transport.js";
 export type { CallOptions, Query, QueryValue, RequestOptions, } from "./transport.js";
 export type { HybridSearchOptions } from "./namespaces.js";
 export { ContextNamespace, EntityNamespace, FactsNamespace, GraphNamespace, OntologyNamespace, QueryNamespace, SchemaNamespace, SearchNamespace, } from "./namespaces.js";
@@ -36,6 +36,7 @@ export declare class LbbClient {
     private readonly onRequest?;
     private readonly onResponse?;
     private readonly onRetry?;
+    private capabilities?;
     /** A5 default read consistency applied when a read omits its own value. */
     readonly defaultConsistency?: SearchConsistency;
     readonly context: ContextNamespace;
@@ -75,6 +76,7 @@ export declare class LbbClient {
     request<T>(method: string, path: string, opts?: RequestOptions): Promise<T>;
     private mutationKey;
     idempotencyKey(prefix?: string): string;
+    private requireCapability;
     /** Commit triplets and optional entity embeddings. Prefer `client.graph("main").facts.create(...)`. */
     commit(body: Schemas["TripletCommitFile"], opts?: {
         idempotencyKey?: string;
@@ -109,6 +111,28 @@ export declare class LbbClient {
     }): Promise<Schemas["GraphImportResponse"] & {
         commitSeq: number | null;
     }>;
+    /**
+     * Stream NDJSON into immutable storage and enqueue a durable import job.
+     *
+     * The idempotency key is mandatory and binds the key to the uploaded content.
+     * Streaming uploads are attempted once: a one-shot async iterator cannot be
+     * replayed safely by an automatic HTTP retry. Call this method again with a
+     * fresh iterable and the same key to perform an explicit idempotent replay.
+     */
+    submitImport(lines: DurableImportSource, opts: {
+        idempotencyKey: string;
+        batch?: number;
+        strict?: boolean;
+        observedAt?: string;
+        signal?: AbortSignal;
+    }): Promise<Schemas["GraphImportJobAccepted"]>;
+    getImportJob(jobId: string): Promise<Schemas["GraphImportJobStatus"]>;
+    cancelImportJob(jobId: string): Promise<Schemas["GraphImportJobCancelResponse"]>;
+    waitForImportJob(jobId: string, opts?: {
+        pollIntervalMs?: number;
+        timeoutMs?: number;
+        signal?: AbortSignal;
+    }): Promise<Schemas["GraphImportJobStatus"]>;
     /**
      * Bulk-ingest N-Triples, Turtle, N-Quads, or TriG without client-side conversion. Resource-object
      * triples become keyed Resource edges; literal-object triples become text

package/dist/client.js

@@ -1,9 +1,74 @@
 import { parseSparqlResults } from "./types.js";
 import { bodyMarksTerminal, errorCodeFromBody, fullJitterBackoffMs, parseLbbError, parseResponseJson, retryAllowed, retryableStatus, retryDelayMs, sleep, } from "./transport.js";
+import { LbbCapabilityError } from "./transport.js";
 import { ContextNamespace, EntityNamespace, GraphNamespace, OntologyNamespace, QueryNamespace, SchemaNamespace, SearchNamespace, } from "./namespaces.js";
 export { parseSparqlResults } from "./types.js";
-export { LbbError } from "./transport.js";
+export { LbbCapabilityError, LbbError } from "./transport.js";
 export { ContextNamespace, EntityNamespace, FactsNamespace, GraphNamespace, OntologyNamespace, QueryNamespace, SchemaNamespace, SearchNamespace, } from "./namespaces.js";
+function durableImportBytes(line) {
+    if (line instanceof Uint8Array)
+        return line;
+    const encoded = typeof line === "string" ? line : JSON.stringify(line);
+    return new TextEncoder().encode(encoded.endsWith("\n") ? encoded : `${encoded}\n`);
+}
+function durableImportIterator(source) {
+    if (typeof source === "string") {
+        return (async function* () {
+            yield source;
+        })();
+    }
+    if (Symbol.asyncIterator in Object(source)) {
+        return source[Symbol.asyncIterator]();
+    }
+    const iterator = source[Symbol.iterator]();
+    return {
+        next: async () => iterator.next(),
+        return: async () => {
+            iterator.return?.();
+            return { done: true, value: undefined };
+        },
+    };
+}
+function durableImportBody(source) {
+    const iterator = durableImportIterator(source);
+    const Stream = globalThis.ReadableStream;
+    if (Stream) {
+        return new Stream({
+            async pull(controller) {
+                try {
+                    const item = await iterator.next();
+                    if (item.done) {
+                        controller.close();
+                    }
+                    else {
+                        controller.enqueue(durableImportBytes(item.value));
+                    }
+                }
+                catch (error) {
+                    controller.error(error);
+                }
+            },
+            async cancel() {
+                await iterator.return?.();
+            },
+        });
+    }
+    return {
+        async *[Symbol.asyncIterator]() {
+            try {
+                for (;;) {
+                    const item = await iterator.next();
+                    if (item.done)
+                        return;
+                    yield durableImportBytes(item.value);
+                }
+            }
+            finally {
+                await iterator.return?.();
+            }
+        },
+    };
+}
 /**
  * A typed HTTP client for a little big brain graph server. One instance is scoped to a
  * single graph/branch; construct another for a different scope. All methods
@@ -24,6 +89,7 @@ export class LbbClient {
     onRequest;
     onResponse;
     onRetry;
+    capabilities;
     /** A5 default read consistency applied when a read omits its own value. */
     defaultConsistency;
     context;
@@ -172,6 +238,7 @@ export class LbbClient {
             method,
             headers,
             body,
+            ...(opts.duplex ? { duplex: opts.duplex } : {}),
         };
         const timeoutMs = opts.timeoutMs ?? this.timeoutMs;
         const maxRetries = opts.maxRetries ?? this.maxRetries;
@@ -207,10 +274,16 @@ export class LbbClient {
                     maxAttempts: maxRetries + 1,
                     idempotencyKey: opts.idempotencyKey,
                 });
-                response = await this.fetchImpl(url, {
+                const requestInit = {
                     ...init,
                     signal: controller?.signal ?? opts.signal,
-                });
+                };
+                // Keep FetchLike's long-standing string-body test-double contract while
+                // allowing this one endpoint to pass a native streaming body to fetch.
+                // Native browser/Node fetch implementations accept the extended shape;
+                // custom transports that need durable imports can inspect it at runtime.
+                const streamingFetch = this.fetchImpl;
+                response = await streamingFetch(url, requestInit);
                 text = await response.text();
             }
             catch (error) {
@@ -321,6 +394,12 @@ export class LbbClient {
     idempotencyKey(prefix = "request") {
         return this.mutationKey(prefix);
     }
+    async requireCapability(capability) {
+        this.capabilities ??= this.request("GET", "/version").then((version) => new Set(version.capabilities));
+        if (!(await this.capabilities).has(capability)) {
+            throw new LbbCapabilityError(capability);
+        }
+    }
     // --- writes ---
     /** Commit triplets and optional entity embeddings. Prefer `client.graph("main").facts.create(...)`. */
     async commit(body, opts = {}) {
@@ -375,6 +454,72 @@ export class LbbClient {
         // write→floor→read loop reads naturally after a bulk load.
         return { ...response, commitSeq: response.committed_commit_seq ?? null };
     }
+    /**
+     * Stream NDJSON into immutable storage and enqueue a durable import job.
+     *
+     * The idempotency key is mandatory and binds the key to the uploaded content.
+     * Streaming uploads are attempted once: a one-shot async iterator cannot be
+     * replayed safely by an automatic HTTP retry. Call this method again with a
+     * fresh iterable and the same key to perform an explicit idempotent replay.
+     */
+    async submitImport(lines, opts) {
+        if (!opts?.idempotencyKey?.trim()) {
+            throw new TypeError("submitImport requires a non-empty idempotencyKey");
+        }
+        await this.requireCapability("durable_import_jobs_v1");
+        return this.request("POST", "/v1/graph/import-jobs", {
+            rawBody: durableImportBody(lines),
+            contentType: "application/x-ndjson",
+            duplex: "half",
+            query: {
+                batch: opts.batch,
+                strict: opts.strict,
+                observed_at: opts.observedAt,
+            },
+            idempotencyKey: opts.idempotencyKey,
+            maxRetries: 0,
+            retry: false,
+            signal: opts.signal,
+        });
+    }
+    async getImportJob(jobId) {
+        await this.requireCapability("durable_import_jobs_v1");
+        return this.request("GET", "/v1/graph/import-jobs", {
+            query: { job_id: jobId },
+        });
+    }
+    async cancelImportJob(jobId) {
+        await this.requireCapability("durable_import_jobs_v1");
+        return this.request("DELETE", "/v1/graph/import-jobs", {
+            query: { job_id: jobId },
+        });
+    }
+    async waitForImportJob(jobId, opts = {}) {
+        const pollIntervalMs = opts.pollIntervalMs ?? 1_000;
+        const timeoutMs = opts.timeoutMs ?? 0;
+        if (!Number.isFinite(pollIntervalMs) || pollIntervalMs < 0) {
+            throw new RangeError("pollIntervalMs must be a non-negative number");
+        }
+        if (!Number.isFinite(timeoutMs) || timeoutMs < 0) {
+            throw new RangeError("timeoutMs must be a non-negative number");
+        }
+        const deadline = timeoutMs > 0 ? Date.now() + timeoutMs : undefined;
+        for (;;) {
+            if (opts.signal?.aborted) {
+                throw opts.signal.reason ?? new Error("request aborted");
+            }
+            const status = await this.getImportJob(jobId);
+            if (status.state === "succeeded" ||
+                status.state === "failed" ||
+                status.state === "cancelled") {
+                return status;
+            }
+            if (deadline !== undefined && Date.now() >= deadline) {
+                throw new Error(`timed out waiting for durable import job ${jobId}`);
+            }
+            await sleep(pollIntervalMs);
+        }
+    }
     /**
      * Bulk-ingest N-Triples, Turtle, N-Quads, or TriG without client-side conversion. Resource-object
      * triples become keyed Resource edges; literal-object triples become text

package/dist/index.d.ts

@@ -1,3 +1,3 @@
-export { LbbClient, LbbError, parseSparqlResults } from "./client.js";
-export type { LbbClientOptions, CallOptions, RequestOptions, HybridSearchOptions, LbbRequestEvent, LbbResponseEvent, FetchLike, ReadConsistencyOptions, SearchConsistency, Schemas, SparqlResults, SparqlResultsJson, SparqlTerm, CommitRequest, CommitResponse, Entity, EntitySelector, GraphMetadata, GraphSummary, SchemaView, SearchRequest, SearchResponse, SearchResult, Snapshot, } from "./client.js";
+export { LbbCapabilityError, LbbClient, LbbError, parseSparqlResults, } from "./client.js";
+export type { DurableImportLine, DurableImportSource, LbbClientOptions, CallOptions, RequestOptions, HybridSearchOptions, LbbRequestEvent, LbbResponseEvent, FetchLike, ReadConsistencyOptions, SearchConsistency, Schemas, SparqlResults, SparqlResultsJson, SparqlTerm, CommitRequest, CommitResponse, Entity, EntitySelector, GraphMetadata, GraphSummary, SchemaView, SearchRequest, SearchResponse, SearchResult, Snapshot, } from "./client.js";
 export type { components, paths, operations } from "./schema.js";

package/dist/index.js

@@ -1 +1 @@
-export { LbbClient, LbbError, parseSparqlResults } from "./client.js";
+export { LbbCapabilityError, LbbClient, LbbError, parseSparqlResults, } from "./client.js";