@probeo/anymodel 0.5.0 → 0.5.1

package/dist/index.d.cts

@@ -456,6 +456,7 @@ declare class BatchManager {
     private syncNativeBatchStatus;
     /**
      * Process batch requests concurrently (fallback path).
+     * Streams requests from disk to avoid holding them all in memory.
      */
     private processConcurrentBatch;
 }
@@ -575,6 +576,10 @@ declare class BatchStore {
      * List all batch IDs.
      */
     listBatches(): Promise<string[]>;
+    /**
+     * Stream requests from JSONL one line at a time (memory-efficient).
+     */
+    streamRequests(id: string): AsyncGenerator<unknown>;
     /**
      * Check if a batch exists.
      */

package/dist/index.d.ts

@@ -456,6 +456,7 @@ declare class BatchManager {
     private syncNativeBatchStatus;
     /**
      * Process batch requests concurrently (fallback path).
+     * Streams requests from disk to avoid holding them all in memory.
      */
     private processConcurrentBatch;
 }
@@ -575,6 +576,10 @@ declare class BatchStore {
      * List all batch IDs.
      */
     listBatches(): Promise<string[]>;
+    /**
+     * Stream requests from JSONL one line at a time (memory-efficient).
+     */
+    streamRequests(id: string): AsyncGenerator<unknown>;
     /**
      * Check if a batch exists.
      */

package/dist/index.js

@@ -1953,6 +1953,17 @@ var BatchStore = class {
     const entries = await readDirQueued(this.dir);
     return entries.filter((d) => d.isDirectory()).map((d) => d.name).sort();
   }
+  /**
+   * Stream requests from JSONL one line at a time (memory-efficient).
+   */
+  async *streamRequests(id) {
+    const p = joinPath(this.batchDir(id), "requests.jsonl");
+    if (!await fileExistsQueued(p)) return;
+    const raw = await readFileQueued(p, "utf8");
+    for (const line of raw.split("\n")) {
+      if (line.trim()) yield JSON.parse(line);
+    }
+  }
   /**
    * Check if a batch exists.
    */
@@ -2017,7 +2028,7 @@ var BatchManager = class {
       this.processNativeBatch(id, request, native.adapter).catch(() => {
       });
     } else {
-      this.processConcurrentBatch(id, request).catch(() => {
+      this.processConcurrentBatch(id, request.model, request.options).catch(() => {
       });
     }
     return batch;
@@ -2197,28 +2208,28 @@ var BatchManager = class {
   }
   /**
    * Process batch requests concurrently (fallback path).
+   * Streams requests from disk to avoid holding them all in memory.
    */
-  async processConcurrentBatch(batchId, request) {
+  async processConcurrentBatch(batchId, model, options) {
     const batch = await this.store.getMeta(batchId);
     if (!batch) return;
     batch.status = "processing";
     await this.store.updateMeta(batch);
-    const items = request.requests;
     const active = /* @__PURE__ */ new Set();
     const processItem = async (item) => {
       const current = await this.store.getMeta(batchId);
       if (current?.status === "cancelled") return;
       const chatRequest = {
-        model: request.model,
+        model,
         messages: item.messages,
-        max_tokens: item.max_tokens ?? request.options?.max_tokens,
-        temperature: item.temperature ?? request.options?.temperature,
-        top_p: item.top_p ?? request.options?.top_p,
-        top_k: item.top_k ?? request.options?.top_k,
-        stop: item.stop ?? request.options?.stop,
-        response_format: item.response_format ?? request.options?.response_format,
-        tools: item.tools ?? request.options?.tools,
-        tool_choice: item.tool_choice ?? request.options?.tool_choice
+        max_tokens: item.max_tokens ?? options?.max_tokens,
+        temperature: item.temperature ?? options?.temperature,
+        top_p: item.top_p ?? options?.top_p,
+        top_k: item.top_k ?? options?.top_k,
+        stop: item.stop ?? options?.stop,
+        response_format: item.response_format ?? options?.response_format,
+        tools: item.tools ?? options?.tools,
+        tool_choice: item.tool_choice ?? options?.tool_choice
       };
       let result;
       try {
@@ -2249,7 +2260,7 @@ var BatchManager = class {
         await this.store.updateMeta(meta);
       }
     };
-    for (const item of items) {
+    for await (const item of this.store.streamRequests(batchId)) {
       const current = await this.store.getMeta(batchId);
       if (current?.status === "cancelled") break;
       if (active.size >= this.concurrencyLimit) {