@babav/knowledge-core-client 0.22.5 → 0.23.0

package/dist/index.d.ts

@@ -214,6 +214,16 @@ export interface UploadUrl {
     gcs_uri: string;
     expires_in: number;
 }
+/** Batch of signed PUT URLs (corpora.uploadMany drives this). `items` is in request order. */
+export interface BatchUploadUrls {
+    items: Array<{
+        filename: string;
+        document_id: UUID;
+        upload_url: string;
+        gcs_uri: string;
+    }>;
+    expires_in: number;
+}
 export interface Conversation {
     id: UUID;
     title: string | null;
@@ -433,11 +443,10 @@ export declare class KnowledgeCoreClient extends HttpBase {
             limit?: number;
             cursor?: string;
         }) => Promise<Page<Document>>;
-        /** Ingest a document ASYNCHRONOUSLY — resolves with the created document at
-         *  status `pending` (HTTP 202); track via documents.get() polling to
-         *  `indexed`/`failed`, or the tenant result webhook. Auto-routes by size: files
-         *  over ~30 MB go via the signed-URL upload path (Cloud Run caps requests at
-         *  ~32 MB), smaller files via one multipart request. One method, any size. */
+        /** Ingest ONE document ASYNCHRONOUSLY — resolves with the created document at status
+         *  `pending` (HTTP 202); track via documents.events()/get() or the tenant webhook.
+         *  ALWAYS uploads the bytes straight to GCS via a signed URL (uploadUrl → PUT → ingestFromUpload),
+         *  regardless of size — so there is NO request-size cap, ever. One method, any size. */
         ingestDocument: (id: UUID, a: {
             file: FileData;
             filename: string;
@@ -472,6 +481,21 @@ export declare class KnowledgeCoreClient extends HttpBase {
             folder_id?: UUID;
             custom_metadata?: Record<string, unknown>;
         }) => Promise<Document>;
+        /** Ingest MANY files in ONE go — any size, no 429, no client backoff. Mints all signed URLs in
+         *  one request, PUTs the bytes straight to GCS (bounded concurrency; never touches the KC), then
+         *  registers the whole batch in one request (N pending docs + N Cloud Tasks). `opts`
+         *  (folder_id / visibility / custom_metadata — the latter carries `owner`) applies to ALL files.
+         *  Resolves with the created `pending` documents. */
+        uploadMany: (id: UUID, files: Array<{
+            file: FileData;
+            filename: string;
+            content_type?: string;
+        }>, opts?: {
+            folder_id?: UUID;
+            visibility?: Visibility;
+            custom_metadata?: Record<string, unknown>;
+            concurrency?: number;
+        }) => Promise<Document[]>;
     };
     parse(a: {
         file: FileData;

package/dist/index.js

@@ -226,24 +226,11 @@ export class KnowledgeCoreClient extends HttpBase {
         countDocuments: (id) => this.request("GET", `/v1/corpora/${id}/documents/count`),
         /** Documents in the corpus whose filename contains `q` (case-insensitive), paginated. */
         searchDocuments: (id, q, opts) => this.request("GET", `/v1/corpora/${id}/documents/search`, { query: { q, ...opts } }),
-        /** Ingest a document ASYNCHRONOUSLY — resolves with the created document at
-         *  status `pending` (HTTP 202); track via documents.get() polling to
-         *  `indexed`/`failed`, or the tenant result webhook. Auto-routes by size: files
-         *  over ~30 MB go via the signed-URL upload path (Cloud Run caps requests at
-         *  ~32 MB), smaller files via one multipart request. One method, any size. */
-        ingestDocument: (id, a) => {
-            if (fileSize(a.file) > MULTIPART_MAX_BYTES)
-                return this.corpora.uploadDocument(id, a);
-            const fd = new FormData();
-            fd.append("file", toBlob(a.file, a.content_type), a.filename);
-            if (a.custom_metadata)
-                fd.append("custom_metadata", JSON.stringify(a.custom_metadata));
-            if (a.visibility)
-                fd.append("visibility", a.visibility);
-            if (a.folder_id)
-                fd.append("folder_id", a.folder_id);
-            return this.request("POST", `/v1/corpora/${id}/documents`, { body: fd });
-        },
+        /** Ingest ONE document ASYNCHRONOUSLY — resolves with the created document at status
+         *  `pending` (HTTP 202); track via documents.events()/get() or the tenant webhook.
+         *  ALWAYS uploads the bytes straight to GCS via a signed URL (uploadUrl → PUT → ingestFromUpload),
+         *  regardless of size — so there is NO request-size cap, ever. One method, any size. */
+        ingestDocument: (id, a) => this.corpora.uploadDocument(id, a),
         /** Mint a signed PUT URL to upload a large file straight to GCS (bypasses the
          *  ~32 MB request limit). PUT the bytes to upload_url, then ingestFromUpload. */
         uploadUrl: (id, a) => this.request("POST", `/v1/corpora/${id}/documents/upload-url`, { json: a }),
@@ -267,6 +254,33 @@ export class KnowledgeCoreClient extends HttpBase {
                 visibility: a.visibility, folder_id: a.folder_id, custom_metadata: a.custom_metadata,
             });
         },
+        /** Ingest MANY files in ONE go — any size, no 429, no client backoff. Mints all signed URLs in
+         *  one request, PUTs the bytes straight to GCS (bounded concurrency; never touches the KC), then
+         *  registers the whole batch in one request (N pending docs + N Cloud Tasks). `opts`
+         *  (folder_id / visibility / custom_metadata — the latter carries `owner`) applies to ALL files.
+         *  Resolves with the created `pending` documents. */
+        uploadMany: async (id, files, opts) => {
+            if (files.length === 0)
+                return [];
+            const urls = await this.request("POST", `/v1/corpora/${id}/documents/upload-urls`, { json: { items: files.map((f) => ({ filename: f.filename, content_type: f.content_type })) } });
+            const items = urls.items; // same order as the request
+            await poolMap(files.length, opts?.concurrency ?? 6, async (i) => {
+                const f = files[i];
+                const it = items[i];
+                const put = await this._fetch(it.upload_url, {
+                    method: "PUT",
+                    body: toBlob(f.file, f.content_type),
+                    headers: f.content_type ? { "content-type": f.content_type } : {},
+                });
+                if (!put.ok)
+                    throw new KnowledgeCoreError(put.status, safeJson(await put.text()), it.upload_url);
+            });
+            const res = await this.request("POST", `/v1/corpora/${id}/documents/batch`, { json: {
+                    folder_id: opts?.folder_id, visibility: opts?.visibility, custom_metadata: opts?.custom_metadata,
+                    items: items.map((it) => ({ document_id: it.document_id, filename: it.filename })),
+                } });
+            return res.documents;
+        },
     };
     // --- parsing (utility: file -> text, stores nothing) ---
     parse(a) {
@@ -376,6 +390,15 @@ export class AdminClient extends HttpBase {
 // ---------------------------------------------------------------------------
 // helpers
 // ---------------------------------------------------------------------------
+/** Run fn(0..n-1) with bounded concurrency (used by uploadMany for the direct-to-GCS PUTs). */
+async function poolMap(n, concurrency, fn) {
+    let next = 0;
+    const workers = Array.from({ length: Math.max(1, Math.min(concurrency, n)) }, async () => {
+        for (let i = next++; i < n; i = next++)
+            await fn(i);
+    });
+    await Promise.all(workers);
+}
 function dispatchSse(frame, h) {
     let event = "message";
     const dataLines = [];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@babav/knowledge-core-client",
-  "version": "0.22.5",
+  "version": "0.23.0",
   "description": "TypeScript client for the Babav Knowledge Core API (Deno + Node 18+, zero deps). Includes the babav.visual grammar TYPES at the ./visual subpath (types only; all visual rendering is server-side).",
   "type": "module",
   "main": "./dist/index.js",

package/src/index.ts

@@ -235,6 +235,11 @@ export interface UploadUrl {
   gcs_uri: string;
   expires_in: number;
 }
+/** Batch of signed PUT URLs (corpora.uploadMany drives this). `items` is in request order. */
+export interface BatchUploadUrls {
+  items: Array<{ filename: string; document_id: UUID; upload_url: string; gcs_uri: string }>;
+  expires_in: number;
+}
 export interface Conversation {
   id: UUID;
   title: string | null;
@@ -596,20 +601,12 @@ export class KnowledgeCoreClient extends HttpBase {
     /** Documents in the corpus whose filename contains `q` (case-insensitive), paginated. */
     searchDocuments: (id: UUID, q: string, opts?: { limit?: number; cursor?: string }) =>
       this.request<Page<Document>>("GET", `/v1/corpora/${id}/documents/search`, { query: { q, ...opts } }),
-    /** Ingest a document ASYNCHRONOUSLY — resolves with the created document at
-     *  status `pending` (HTTP 202); track via documents.get() polling to
-     *  `indexed`/`failed`, or the tenant result webhook. Auto-routes by size: files
-     *  over ~30 MB go via the signed-URL upload path (Cloud Run caps requests at
-     *  ~32 MB), smaller files via one multipart request. One method, any size. */
-    ingestDocument: (id: UUID, a: { file: FileData; filename: string; content_type?: string; visibility?: Visibility; folder_id?: UUID; custom_metadata?: Record<string, unknown> }) => {
-      if (fileSize(a.file) > MULTIPART_MAX_BYTES) return this.corpora.uploadDocument(id, a);
-      const fd = new FormData();
-      fd.append("file", toBlob(a.file, a.content_type), a.filename);
-      if (a.custom_metadata) fd.append("custom_metadata", JSON.stringify(a.custom_metadata));
-      if (a.visibility) fd.append("visibility", a.visibility);
-      if (a.folder_id) fd.append("folder_id", a.folder_id);
-      return this.request<Document>("POST", `/v1/corpora/${id}/documents`, { body: fd });
-    },
+    /** Ingest ONE document ASYNCHRONOUSLY — resolves with the created document at status
+     *  `pending` (HTTP 202); track via documents.events()/get() or the tenant webhook.
+     *  ALWAYS uploads the bytes straight to GCS via a signed URL (uploadUrl → PUT → ingestFromUpload),
+     *  regardless of size — so there is NO request-size cap, ever. One method, any size. */
+    ingestDocument: (id: UUID, a: { file: FileData; filename: string; content_type?: string; visibility?: Visibility; folder_id?: UUID; custom_metadata?: Record<string, unknown> }) =>
+      this.corpora.uploadDocument(id, a),
     /** Mint a signed PUT URL to upload a large file straight to GCS (bypasses the
      *  ~32 MB request limit). PUT the bytes to upload_url, then ingestFromUpload. */
     uploadUrl: (id: UUID, a: { filename: string; content_type?: string }) =>
@@ -635,6 +632,41 @@ export class KnowledgeCoreClient extends HttpBase {
         visibility: a.visibility, folder_id: a.folder_id, custom_metadata: a.custom_metadata,
       });
     },
+    /** Ingest MANY files in ONE go — any size, no 429, no client backoff. Mints all signed URLs in
+     *  one request, PUTs the bytes straight to GCS (bounded concurrency; never touches the KC), then
+     *  registers the whole batch in one request (N pending docs + N Cloud Tasks). `opts`
+     *  (folder_id / visibility / custom_metadata — the latter carries `owner`) applies to ALL files.
+     *  Resolves with the created `pending` documents. */
+    uploadMany: async (
+      id: UUID,
+      files: Array<{ file: FileData; filename: string; content_type?: string }>,
+      opts?: { folder_id?: UUID; visibility?: Visibility; custom_metadata?: Record<string, unknown>; concurrency?: number },
+    ): Promise<Document[]> => {
+      if (files.length === 0) return [];
+      const urls = await this.request<BatchUploadUrls>(
+        "POST", `/v1/corpora/${id}/documents/upload-urls`,
+        { json: { items: files.map((f) => ({ filename: f.filename, content_type: f.content_type })) } },
+      );
+      const items = urls.items; // same order as the request
+      await poolMap(files.length, opts?.concurrency ?? 6, async (i) => {
+        const f = files[i]!;
+        const it = items[i]!;
+        const put = await this._fetch(it.upload_url, {
+          method: "PUT",
+          body: toBlob(f.file, f.content_type),
+          headers: f.content_type ? { "content-type": f.content_type } : {},
+        });
+        if (!put.ok) throw new KnowledgeCoreError(put.status, safeJson(await put.text()), it.upload_url);
+      });
+      const res = await this.request<{ documents: Document[] }>(
+        "POST", `/v1/corpora/${id}/documents/batch`,
+        { json: {
+            folder_id: opts?.folder_id, visibility: opts?.visibility, custom_metadata: opts?.custom_metadata,
+            items: items.map((it) => ({ document_id: it.document_id, filename: it.filename })),
+        } },
+      );
+      return res.documents;
+    },
   };
 
   // --- parsing (utility: file -> text, stores nothing) ---
@@ -772,6 +804,15 @@ export class AdminClient extends HttpBase {
 // ---------------------------------------------------------------------------
 // helpers
 // ---------------------------------------------------------------------------
+/** Run fn(0..n-1) with bounded concurrency (used by uploadMany for the direct-to-GCS PUTs). */
+async function poolMap(n: number, concurrency: number, fn: (i: number) => Promise<void>): Promise<void> {
+  let next = 0;
+  const workers = Array.from({ length: Math.max(1, Math.min(concurrency, n)) }, async () => {
+    for (let i = next++; i < n; i = next++) await fn(i);
+  });
+  await Promise.all(workers);
+}
+
 function dispatchSse(frame: string, h: StreamHandlers): void {
   let event = "message";
   const dataLines: string[] = [];