@mindstudio-ai/agent 0.1.140 → 0.1.141

package/dist/index.js

@@ -273,7 +273,6 @@ function getOrOpen(url, token) {
       pending: /* @__PURE__ */ new Map(),
       chunked: false,
       inbound: /* @__PURE__ */ new Map(),
-      acksSeen: false,
       retired: false
     };
     const c = self;
@@ -297,7 +296,6 @@ function getOrOpen(url, token) {
         return;
       }
       if (msg.t === "db-ack") {
-        c.acksSeen = true;
         const p = c.pending.get(msg.id);
         if (p && !p.acked) {
           p.acked = true;
@@ -373,15 +371,11 @@ async function executeDbBatchOverWs(url, token, databaseId, queries) {
   }
   return await new Promise((resolve, reject) => {
     const pending = { resolve, reject, timer: null, acked: false };
-    if (self.acksSeen) {
-      pending.timer = setTimeout(() => {
-        forget(self, id);
-        retire(self, "a request went unacknowledged");
-        reject(new DbWsTransportError("ws ack timed out", true));
-      }, WS_ACK_TIMEOUT_MS);
-    } else {
-      armIdleTimer(self, id, pending);
-    }
+    pending.timer = setTimeout(() => {
+      forget(self, id);
+      retire(self, "a request went unacknowledged");
+      reject(new DbWsTransportError("ws ack timed out", true));
+    }, WS_ACK_TIMEOUT_MS);
     self.pending.set(id, pending);
     let started = false;
     let complete = false;
@@ -3410,7 +3404,15 @@ var DataSource = class {
    * as the new one is registered, before its build lands; a search in that
    * window sees neither. **Throws `mapper_skipped`** (HTTP 422) when the
    * mapper skipped the object — a specific thing it refused, which the caller
-   * should see — and `mapper_error` when the mapper threw.
+   * should see — and `mapper_error` when the mapper threw, or when none of
+   * its documents passed validation. A document that fails is left out and
+   * reported in {@link AddResult.invalid}; the rest are registered.
+   *
+   * An object that maps to more than a thousand documents (a large export,
+   * streamed out of the mapper as it parses) registers its first thousand
+   * here and returns {@link AddResult.job}, a job that registers and builds
+   * the rest; `documentCount` says how many there are in all. Another job on
+   * the source refuses it with `data_source_busy`.
    */
   async add(content, options) {
     const bytes = typeof content === "string" ? Buffer.from(content) : Buffer.from(content);
@@ -3420,6 +3422,7 @@ var DataSource = class {
       ...options.contentType ? { contentType: options.contentType } : {},
       ...options.metadata !== void 0 ? { metadata: options.metadata } : {},
       ...options.entities !== void 0 ? { entities: options.entities } : {},
+      ...options.budgetDollars !== void 0 ? { budgetDollars: options.budgetDollars } : {},
       body: bytes.toString("base64")
     });
   }
@@ -3631,6 +3634,15 @@ var DataSource = class {
       ...name ? { name } : {}
     });
   }
+  /**
+   * @internal A mapper frame's grant to store the next part of its spill: how
+   * `defineMapper`'s executor hands the platform an object's documents a part
+   * at a time once they outgrow the frame's record. Only a mapper frame's
+   * token is granted one.
+   */
+  async _mapSpillGrant(spillId) {
+    return this._call("map/spill", { slug: this._slug, spillId });
+  }
   /**
    * @internal Content hash of some bytes, matching what the server computes.
    * Exposed for callers that want to check whether they already added a file.
@@ -3781,7 +3793,7 @@ function createEvents(call) {
   };
 }
 
-// src/task/index.ts
+// src/task/local.ts
 import { randomUUID as randomUUID2 } from "crypto";
 
 // src/generated/metadata.ts
@@ -4926,6 +4938,49 @@ var stepMetadata = {
   }
 };
 
+// src/task/index.ts
+function resolveStepType(name) {
+  const meta = stepMetadata[name];
+  return meta ? meta.stepType : name;
+}
+function mapTools(tools) {
+  return tools.map((t) => {
+    if (typeof t === "object" && "execute" in t) {
+      return {
+        name: t.name,
+        description: t.description,
+        inputSchema: t.inputSchema ?? { type: "object", properties: {} }
+      };
+    }
+    if (typeof t === "object" && "appMethod" in t) {
+      return {
+        appMethod: t.appMethod,
+        ...t.description ? { description: t.description } : {},
+        ...t.defaults ? { defaults: t.defaults } : {}
+      };
+    }
+    const method = typeof t === "string" ? t : t.method;
+    const stepType = resolveStepType(method);
+    const defaults = typeof t === "object" ? t.defaults : void 0;
+    return defaults ? { stepType, defaults } : { stepType };
+  });
+}
+function sleep2(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+function isDevMode() {
+  return !!(process.env.CALLBACK_TOKEN || getRequestContext()?.callbackToken);
+}
+function logTaskResult(result) {
+  if (!isDevMode()) {
+    return;
+  }
+  const toolSummary = result.toolCalls.map((tc) => `${tc.name} (${tc.durationMs}ms) ${tc.success ? "\u2713" : "\u2717"}`).join(", ");
+  console.log(
+    `[task] ${result.turns} turn${result.turns === 1 ? "" : "s"}` + (toolSummary ? `: ${toolSummary}` : "") + ` | ${result.parsedSuccessfully ? "output OK" : "\u26A0 output not valid JSON"} | cost: ${result.usage.totalBillingCost} | trace: ${result.traceId}`
+  );
+}
+
 // src/task/schema.ts
 function describe(value) {
   if (value === void 0) {
@@ -5066,38 +5121,6 @@ function assertSupportedNode(schema, path) {
     assertSupportedNode(schema.items, `${path}[]`);
   }
 }
-function buildExampleFromSchema(schema) {
-  if (schema.const !== void 0) {
-    return schema.const;
-  }
-  if (schema.enum !== void 0 && schema.enum.length > 0) {
-    return schema.enum[0];
-  }
-  const names = Array.isArray(schema.type) ? schema.type : schema.type ? [schema.type] : [];
-  const name = names.find((n) => n !== "null") ?? names[0];
-  switch (name) {
-    case "object": {
-      const example = {};
-      for (const [key, propSchema] of Object.entries(schema.properties ?? {})) {
-        example[key] = buildExampleFromSchema(propSchema);
-      }
-      return example;
-    }
-    case "array":
-      return schema.items ? [buildExampleFromSchema(schema.items)] : [];
-    case "string":
-      return "...";
-    case "number":
-    case "integer":
-      return 0;
-    case "boolean":
-      return true;
-    case "null":
-      return null;
-    default:
-      return null;
-  }
-}
 function stripCodeFences(text) {
   const match = /^```[a-zA-Z]*[ \t]*\r?\n([\s\S]*?)\r?\n?```$/.exec(
     text.trim()
@@ -5105,262 +5128,7 @@ function stripCodeFences(text) {
   return match ? match[1] : text;
 }
 
-// src/task/index.ts
-function resolveStepType(name) {
-  const meta = stepMetadata[name];
-  return meta ? meta.stepType : name;
-}
-function mapTools(tools) {
-  return tools.map((t) => {
-    if (typeof t === "object" && "execute" in t) {
-      return {
-        name: t.name,
-        description: t.description,
-        inputSchema: t.inputSchema ?? { type: "object", properties: {} }
-      };
-    }
-    if (typeof t === "object" && "appMethod" in t) {
-      return {
-        appMethod: t.appMethod,
-        ...t.description ? { description: t.description } : {},
-        ...t.defaults ? { defaults: t.defaults } : {}
-      };
-    }
-    const method = typeof t === "string" ? t : t.method;
-    const stepType = resolveStepType(method);
-    const defaults = typeof t === "object" ? t.defaults : void 0;
-    return defaults ? { stepType, defaults } : { stepType };
-  });
-}
-function buildTaskRequestBody(options) {
-  return {
-    prompt: options.prompt,
-    input: options.input,
-    tools: mapTools(options.tools),
-    // The legacy whole-task route requires structuredOutputExample and
-    // composes its own prompt server-side, so in schema mode we synthesize a
-    // skeleton example from the schema. Validation still happens client-side
-    // after the result comes back (see _runTaskInner).
-    structuredOutputExample: options.outputSchema ? JSON.stringify(buildExampleFromSchema(options.outputSchema)) : typeof options.structuredOutputExample === "string" ? options.structuredOutputExample : JSON.stringify(options.structuredOutputExample),
-    model: options.model,
-    ...options.maxTurns != null && { maxTurns: options.maxTurns },
-    ...options.appId != null && { appId: options.appId },
-    ...options.threadId != null && { threadId: options.threadId }
-  };
-}
-function sleep2(ms) {
-  return new Promise((resolve) => setTimeout(resolve, ms));
-}
-function isDevMode() {
-  return !!(process.env.CALLBACK_TOKEN || getRequestContext()?.callbackToken);
-}
-function logTaskResult(result) {
-  if (!isDevMode()) {
-    return;
-  }
-  const toolSummary = result.toolCalls.map((tc) => `${tc.name} (${tc.durationMs}ms) ${tc.success ? "\u2713" : "\u2717"}`).join(", ");
-  console.log(
-    `[task] ${result.turns} turn${result.turns === 1 ? "" : "s"}` + (toolSummary ? `: ${toolSummary}` : "") + ` | ${result.parsedSuccessfully ? "output OK" : "\u26A0 output not valid JSON"} | cost: ${result.usage.totalBillingCost} | trace: ${result.traceId}`
-  );
-}
-async function runTaskPoll(httpConfig, body) {
-  const traceId = randomUUID2();
-  const { data } = await request(
-    httpConfig,
-    "POST",
-    "/task",
-    body
-  );
-  const pollUrl = `${httpConfig.baseUrl}/developer/v2/task/poll/${data.taskToken}`;
-  let pollDelay = 300;
-  while (true) {
-    await sleep2(pollDelay);
-    pollDelay = Math.min(pollDelay * 1.5, 3e3);
-    const res = await fetch(pollUrl, {
-      headers: { "User-Agent": "@mindstudio-ai/agent" }
-    });
-    if (res.status === 502 || res.status === 503 || res.status === 504) {
-      continue;
-    }
-    if (res.status === 404) {
-      throw new MindStudioError(
-        "Task poll token not found or expired.",
-        "poll_token_expired",
-        404
-      );
-    }
-    if (!res.ok) {
-      const errorBody = await res.json().catch(() => ({}));
-      throw new MindStudioError(
-        errorBody.message ?? errorBody.error ?? `Task poll failed: ${res.status} ${res.statusText}`,
-        errorCodeFrom(errorBody, "poll_error"),
-        res.status,
-        errorBody
-      );
-    }
-    const poll = await res.json();
-    if (poll.status === "pending") {
-      if (isDevMode() && poll.currentTurn != null) {
-        console.log(
-          `[task] running... turn ${poll.currentTurn}/${poll.maxTurns ?? "?"}`
-        );
-      }
-      continue;
-    }
-    if (poll.status === "error") {
-      throw new MindStudioError(
-        poll.error ?? "Task execution failed.",
-        "task_execution_error",
-        500
-      );
-    }
-    const result = {
-      output: poll.output,
-      outputRaw: poll.outputRaw ?? "",
-      parsedSuccessfully: poll.parsedSuccessfully ?? true,
-      turns: poll.turns ?? 0,
-      usage: poll.usage ?? {
-        inputTokens: 0,
-        outputTokens: 0,
-        totalBillingCost: 0
-      },
-      toolCalls: poll.toolCalls ?? [],
-      traceId
-    };
-    logTaskResult(result);
-    return result;
-  }
-}
-async function runTaskStream(httpConfig, body, onEvent) {
-  const traceId = randomUUID2();
-  const url = `${httpConfig.baseUrl}/developer/v2/task`;
-  const res = await fetch(url, {
-    method: "POST",
-    headers: {
-      Authorization: `Bearer ${httpConfig.token}`,
-      "Content-Type": "application/json",
-      "User-Agent": "@mindstudio-ai/agent",
-      Accept: "text/event-stream"
-    },
-    body: JSON.stringify(body)
-  });
-  if (!res.ok) {
-    let message = `${res.status} ${res.statusText}`;
-    let code = "api_error";
-    let details;
-    try {
-      const text = await res.text();
-      try {
-        const errBody = JSON.parse(text);
-        details = errBody;
-        const errMsg = (typeof errBody.error === "string" ? errBody.error : void 0) ?? (typeof errBody.message === "string" ? errBody.message : void 0);
-        if (errMsg) {
-          message = errMsg;
-        }
-        code = errorCodeFrom(errBody, code);
-      } catch {
-        const stripped = text.replace(/<[^>]*>/g, " ").replace(/\s+/g, " ").trim();
-        if (stripped) {
-          message = stripped.slice(0, 200);
-        }
-      }
-    } catch {
-    }
-    throw new MindStudioError(`[task] ${message}`, code, res.status, details);
-  }
-  const reader = res.body.getReader();
-  const decoder = new TextDecoder();
-  let buffer = "";
-  let result = null;
-  while (true) {
-    const { done, value } = await reader.read();
-    if (done) {
-      break;
-    }
-    buffer += decoder.decode(value, { stream: true });
-    const lines = buffer.split("\n");
-    buffer = lines.pop() ?? "";
-    for (const line of lines) {
-      if (!line.startsWith("data: ")) {
-        continue;
-      }
-      try {
-        const event = JSON.parse(line.slice(6));
-        onEvent(event);
-        if (event.type === "error") {
-          throw new MindStudioError(
-            event.error ?? "Task execution failed.",
-            "task_execution_error",
-            500
-          );
-        }
-        if (event.type === "done") {
-          result = {
-            output: event.output,
-            outputRaw: event.outputRaw ?? "",
-            parsedSuccessfully: event.parsedSuccessfully ?? true,
-            turns: event.turns ?? 0,
-            usage: event.usage ?? {
-              inputTokens: 0,
-              outputTokens: 0,
-              totalBillingCost: 0
-            },
-            toolCalls: event.toolCalls ?? [],
-            traceId
-          };
-        }
-      } catch (err) {
-        if (err instanceof MindStudioError) {
-          throw err;
-        }
-      }
-    }
-  }
-  if (buffer.startsWith("data: ")) {
-    try {
-      const event = JSON.parse(buffer.slice(6));
-      onEvent(event);
-      if (event.type === "error") {
-        throw new MindStudioError(
-          event.error ?? "Task execution failed.",
-          "task_execution_error",
-          500
-        );
-      }
-      if (event.type === "done") {
-        result = {
-          output: event.output,
-          outputRaw: event.outputRaw ?? "",
-          parsedSuccessfully: event.parsedSuccessfully ?? true,
-          turns: event.turns ?? 0,
-          usage: event.usage ?? {
-            inputTokens: 0,
-            outputTokens: 0,
-            totalBillingCost: 0
-          },
-          toolCalls: event.toolCalls ?? [],
-          traceId
-        };
-      }
-    } catch (err) {
-      if (err instanceof MindStudioError) {
-        throw err;
-      }
-    }
-  }
-  if (!result) {
-    throw new MindStudioError(
-      "[task] Stream ended without a done event. The task execution may have been interrupted.",
-      "stream_error",
-      500
-    );
-  }
-  logTaskResult(result);
-  return result;
-}
-
 // src/task/local.ts
-import { randomUUID as randomUUID3 } from "crypto";
 var DEFAULT_MAX_TURNS = 20;
 var MAX_TURNS_LIMIT = 100;
 var MAX_SCHEMA_REPAIR_ATTEMPTS = 3;
@@ -5368,7 +5136,6 @@ var MAX_TOOL_OUTPUT_CHARS = 5e4;
 var MAX_TURN_ATTEMPTS = 5;
 var INITIAL_BACKOFF_MS = 1e3;
 var STALL_TIMEOUT_MS = 3e5;
-var TURN_UNAVAILABLE_CODE = "task_turn_unavailable";
 var UNSAFE_KEYS = /* @__PURE__ */ new Set(["__proto__", "constructor", "prototype"]);
 function isPlainObject(value) {
   if (typeof value !== "object" || value === null || Array.isArray(value)) {
@@ -5614,7 +5381,7 @@ ${structuredOutputExample}
     Math.max(options.maxTurns || DEFAULT_MAX_TURNS, 1),
     MAX_TURNS_LIMIT
   );
-  const traceId = randomUUID3();
+  const traceId = randomUUID2();
   const messages = [
     { role: "user", content: JSON.stringify(options.input) }
   ];
@@ -5714,13 +5481,6 @@ ${structuredOutputExample}
         traceId
       });
     } catch (err) {
-      if (loopCount === 1 && err instanceof TurnError && err.status === 404 && err.errorCode === "not_found") {
-        throw new MindStudioError(
-          "Task turn endpoint unavailable.",
-          TURN_UNAVAILABLE_CODE,
-          404
-        );
-      }
       return turnFailure(err);
     }
     accumulate(turn);
@@ -6945,108 +6705,75 @@ var MindStudioAgent = class {
   }
   async _runTaskInner(options) {
     const httpConfig = this._currentHttpConfig;
-    try {
-      return await runTaskLocal(
-        {
-          httpConfig,
-          executeStepTool: async (stepType, input) => {
-            try {
-              const result = await this.executeStep(stepType, input, {
-                requestSource: "v2-task"
-              });
-              const output = {};
-              let billingCost = 0;
-              for (const [key, value] of Object.entries(
-                result
-              )) {
-                if (key === "$billingCost" && typeof value === "number") {
-                  billingCost = value;
-                }
-                if (!key.startsWith("$")) {
-                  output[key] = value;
-                }
+    return runTaskLocal(
+      {
+        httpConfig,
+        executeStepTool: async (stepType, input) => {
+          try {
+            const result = await this.executeStep(stepType, input, {
+              requestSource: "v2-task"
+            });
+            const output = {};
+            let billingCost = 0;
+            for (const [key, value] of Object.entries(
+              result
+            )) {
+              if (key === "$billingCost" && typeof value === "number") {
+                billingCost = value;
               }
-              return { output, billingCost, isError: false };
-            } catch (err) {
-              return {
-                output: {
-                  error: err instanceof Error ? err.message : "Step execution failed"
-                },
-                billingCost: 0,
-                isError: true
-              };
-            }
-          },
-          executeMethodTool: async (methodId, input, call) => {
-            try {
-              const { data } = await request(
-                { ...httpConfig, maxRetries: 0 },
-                "POST",
-                "/task/invoke-method",
-                {
-                  methodId,
-                  input,
-                  traceId: call.traceId,
-                  toolCallId: call.toolCallId
-                }
-              );
-              if (data.error) {
-                return {
-                  output: { error: data.error },
-                  billingCost: 0,
-                  isError: true
-                };
+              if (!key.startsWith("$")) {
+                output[key] = value;
               }
-              return {
-                output: data.output ?? null,
-                billingCost: 0,
-                isError: false
-              };
-            } catch (err) {
-              return {
-                output: {
-                  error: err instanceof Error ? err.message : "Method execution failed"
-                },
-                billingCost: 0,
-                isError: true
-              };
             }
+            return { output, billingCost, isError: false };
+          } catch (err) {
+            return {
+              output: {
+                error: err instanceof Error ? err.message : "Step execution failed"
+              },
+              billingCost: 0,
+              isError: true
+            };
           }
         },
-        options
-      );
-    } catch (err) {
-      if (err instanceof MindStudioError && err.code === TURN_UNAVAILABLE_CODE) {
-        if (options.tools.some((t) => typeof t === "object" && "execute" in t)) {
-          throw new MindStudioError(
-            "[task] This server does not support the per-turn task endpoint required for function tools.",
-            "task_function_tools_unsupported",
-            404
-          );
-        }
-        const body = buildTaskRequestBody(options);
-        const result = options.onEvent ? await runTaskStream(httpConfig, body, options.onEvent) : await runTaskPoll(httpConfig, body);
-        if (options.outputSchema) {
-          const errors = result.parsedSuccessfully ? validateAgainstSchema(result.output, options.outputSchema) : [{ path: "$", message: "output was not valid JSON" }];
-          if (errors.length > 0) {
-            throw new MindStudioError(
-              "[task] Output did not conform to outputSchema (legacy task route, no repair turns).",
-              "task_output_schema_mismatch",
-              422,
+        executeMethodTool: async (methodId, input, call) => {
+          try {
+            const { data } = await request(
+              { ...httpConfig, maxRetries: 0 },
+              "POST",
+              "/task/invoke-method",
               {
-                outputRaw: result.outputRaw,
-                errors,
-                turns: result.turns,
-                usage: result.usage,
-                toolCalls: result.toolCalls
+                methodId,
+                input,
+                traceId: call.traceId,
+                toolCallId: call.toolCallId
               }
             );
+            if (data.error) {
+              return {
+                output: { error: data.error },
+                billingCost: 0,
+                isError: true
+              };
+            }
+            return {
+              output: data.output ?? null,
+              billingCost: 0,
+              isError: false
+            };
+          } catch (err) {
+            return {
+              output: {
+                error: err instanceof Error ? err.message : "Method execution failed"
+              },
+              billingCost: 0,
+              isError: true
+            };
           }
         }
-        return result;
-      }
-      throw err;
-    }
+      },
+      options
+    );
   }
   /**
    * Get the authenticated user's identity and organization info.
@@ -8419,8 +8146,18 @@ function defineJewel(method, config) {
 }
 
 // src/mapper/index.ts
+import { gzipSync } from "zlib";
 function documents(list) {
-  return { kind: "documents", documents: Array.isArray(list) ? list : [list] };
+  return {
+    kind: "documents",
+    documents: isIterable(list) || isAsyncIterable(list) ? list : [list]
+  };
+}
+function isIterable(value) {
+  return value !== null && typeof value === "object" && typeof value[Symbol.iterator] === "function";
+}
+function isAsyncIterable(value) {
+  return value !== null && typeof value === "object" && typeof value[Symbol.asyncIterator] === "function";
 }
 function passthrough(options) {
   return {
@@ -8440,6 +8177,8 @@ function deletes(externalIds) {
 }
 var DEFAULT_TIMEOUT_MS = 3e4;
 var DEFAULT_CONCURRENCY2 = 4;
+var PART_DOCUMENTS = 1e3;
+var PART_BYTES = 8 * 1024 * 1024;
 function errorInfo2(e) {
   if (e instanceof Error) {
     return { message: e.message, stack: e.stack };
@@ -8471,7 +8210,7 @@ function objectFor(input) {
     etag: input.etag,
     lastModified: input.lastModified,
     metadata: input.metadata,
-    entities: input.entities ?? null,
+    entities: input.entities,
     bytes,
     text: async () => new TextDecoder("utf-8").decode(await bytes()),
     json: async () => JSON.parse(new TextDecoder("utf-8").decode(await bytes()))
@@ -8480,13 +8219,17 @@ function objectFor(input) {
 var OUTCOME_KINDS = /* @__PURE__ */ new Set(["documents", "passthrough", "skip", "deletes"]);
 function normalizeOutcome(value) {
   const outcome = value;
+  const invalid = () => ({
+    kind: "error",
+    error: {
+      message: `map returned an invalid outcome: ${JSON.stringify(value)?.slice(0, 200)}. Return documents(), passthrough(), skip() or deletes().`
+    }
+  });
   if (!outcome || typeof outcome !== "object" || !OUTCOME_KINDS.has(outcome.kind)) {
-    return {
-      kind: "error",
-      error: {
-        message: `map returned an invalid outcome: ${JSON.stringify(value)?.slice(0, 200)}. Return documents(), passthrough(), skip() or deletes().`
-      }
-    };
+    return invalid();
+  }
+  if (outcome.kind === "documents" && !isIterable(outcome.documents) && !isAsyncIterable(outcome.documents)) {
+    return invalid();
   }
   return outcome;
 }
@@ -8508,13 +8251,107 @@ async function withTimeout(work, timeoutMs, key) {
     }
   }
 }
+var PartWriter = class {
+  constructor(source, spillId) {
+    this.source = source;
+    this.spillId = spillId;
+  }
+  source;
+  spillId;
+  parts = [];
+  /** Objects whose documents did not all store, with why. */
+  failed = /* @__PURE__ */ new Map();
+  lines = [];
+  objects = /* @__PURE__ */ new Set();
+  bytes = 0;
+  uploading = Promise.resolve();
+  async write(o, i, d) {
+    const line = JSON.stringify({ o, i, d });
+    this.lines.push(line);
+    this.objects.add(o);
+    this.bytes += line.length;
+    if (this.lines.length >= PART_DOCUMENTS || this.bytes >= PART_BYTES) {
+      await this.flush();
+    }
+  }
+  /** Store what the buffer holds; resolves once every upload so far is done. */
+  async flush() {
+    if (this.lines.length > 0) {
+      const lines = this.lines;
+      const objects = this.objects;
+      this.lines = [];
+      this.objects = /* @__PURE__ */ new Set();
+      this.bytes = 0;
+      this.uploading = this.uploading.then(() => this.store(lines, objects));
+    }
+    await this.uploading;
+  }
+  async store(lines, objects) {
+    try {
+      const grant = await this.source._mapSpillGrant(this.spillId);
+      const form = new FormData();
+      for (const [k, v] of Object.entries(grant.uploadFields)) {
+        form.append(k, v);
+      }
+      form.append(
+        "file",
+        new Blob(
+          [new Uint8Array(gzipSync(Buffer.from(lines.join("\n") + "\n")))],
+          { type: "application/gzip" }
+        ),
+        `${grant.part}.ndjson.gz`
+      );
+      const response = await fetch(grant.uploadUrl, {
+        method: "POST",
+        body: form
+      });
+      if (!response.ok) {
+        throw new Error(
+          `Could not store part ${grant.part} of the documents: HTTP ${response.status}`
+        );
+      }
+      this.parts.push({ part: grant.part, count: lines.length });
+    } catch (e) {
+      const err = e instanceof Error ? e : new Error(String(e));
+      for (const o of objects) {
+        if (!this.failed.has(o)) {
+          this.failed.set(o, err);
+        }
+      }
+    }
+  }
+};
+async function writeDocuments(list, ctx) {
+  const iterator = isAsyncIterable(list) ? list[Symbol.asyncIterator]() : list[Symbol.iterator]();
+  let count = 0;
+  try {
+    while (true) {
+      const step = await withTimeout(
+        Promise.resolve(iterator.next()),
+        ctx.timeoutMs,
+        ctx.key
+      );
+      if (step.done) {
+        return count;
+      }
+      await ctx.writer.write(ctx.o, count, step.value);
+      count += 1;
+    }
+  } catch (e) {
+    await Promise.resolve(iterator.return?.()).catch(() => {
+    });
+    throw e;
+  }
+}
 function defineMapper(source, config) {
+  const handle = source;
   const run = async (params) => {
     const startedAt = Date.now();
     const timeoutMs = params.timeoutMs ?? DEFAULT_TIMEOUT_MS;
     const concurrency = Math.max(1, params.concurrency ?? DEFAULT_CONCURRENCY2);
     const objects = Array.isArray(params.objects) ? params.objects : [];
     const outcomes = new Array(objects.length);
+    const writer = params.spill?.id && typeof handle._mapSpillGrant === "function" ? new PartWriter(handle, params.spill.id) : null;
     let next = 0;
     const worker = async () => {
       while (true) {
@@ -8526,13 +8363,30 @@ function defineMapper(source, config) {
         const t = Date.now();
         let outcome;
         try {
-          outcome = normalizeOutcome(
+          const mapped = normalizeOutcome(
             await withTimeout(
               Promise.resolve().then(() => config.map(objectFor(input))),
               timeoutMs,
               input.key
             )
           );
+          if (mapped.kind !== "documents") {
+            outcome = mapped;
+          } else if (!writer) {
+            throw new Error(
+              "documents() needs the platform to store what it produces, and this frame was given nowhere to store it."
+            );
+          } else {
+            outcome = {
+              kind: "documents",
+              count: await writeDocuments(mapped.documents, {
+                o: index,
+                key: input.key,
+                timeoutMs,
+                writer
+              })
+            };
+          }
         } catch (e) {
           outcome = { kind: "error", error: errorInfo2(e) };
         }
@@ -8546,10 +8400,17 @@ function defineMapper(source, config) {
     await Promise.all(
       Array.from({ length: Math.min(concurrency, objects.length) }, worker)
     );
+    await writer?.flush();
+    for (const [o, err] of writer?.failed ?? []) {
+      if (outcomes[o]?.outcome.kind === "documents") {
+        outcomes[o].outcome = { kind: "error", error: errorInfo2(err) };
+      }
+    }
     return {
       v: 1,
       source: source.name,
       outcomes,
+      parts: writer?.parts ?? [],
       startedAt,
       durationMs: Date.now() - startedAt
     };