@mindstudio-ai/agent 0.1.140 → 0.1.141

package/dist/index.d.ts

@@ -2543,6 +2543,14 @@ interface AddOptions {
      * supplying them replaces the whole set, and `[]` clears it.
      */
     entities?: string[];
+    /**
+     * On a mapped source, the budget in dollars of the job that takes an
+     * object's documents past the first thousand ({@link AddResult.job}): it
+     * pauses there, and a plan projected over it waits for an approval rather
+     * than starting. Without one the job runs within its plan's projection,
+     * with headroom.
+     */
+    budgetDollars?: number;
 }
 /** What {@link DataSource.add} returns. */
 interface AddResult {
@@ -2551,8 +2559,14 @@ interface AddResult {
      * mapper answered `deletes` — nothing was added.
      */
     document: DataSourceDocument | null;
-    /** Every document the bytes became. One entry on an unmapped source. */
+    /**
+     * Every document the bytes became, or the first thousand of an object that
+     * maps to more ({@link job} takes the rest). One entry on an unmapped
+     * source.
+     */
     documents: DataSourceDocument[];
+    /** How many documents the bytes map to in all: `documents.length` unless {@link job} is set. */
+    documentCount: number;
     /** Whether any of them was queued for a build (the rest were already current). */
     queued: boolean;
     /**
@@ -2562,6 +2576,28 @@ interface AddResult {
     outcome: 'unmapped' | 'documents' | 'passthrough' | 'deletes';
     /** Documents removed or superseded as a result — a `deletes` outcome, a `replaces`. */
     removed: number;
+    /**
+     * The documents the mapper produced that failed validation and were not
+     * registered, by their index in its output (`document: 1234` is the
+     * 1,235th it returned or yielded), with why; null when every one passed.
+     * The rest are registered. `add()` throws `mapper_error` only when none
+     * passed.
+     */
+    invalid: {
+        count: number;
+        examples: {
+            document: number;
+            reason: string;
+        }[];
+    } | null;
+    /**
+     * The job registering and building the rest of an object that maps to more
+     * than a thousand documents; null otherwise. It plans and starts on its own
+     * (within {@link AddOptions.budgetDollars}); follow it with
+     * {@link DataSource.job}. What the object supersedes or retires goes when
+     * the job finishes, once every one of its documents is searchable.
+     */
+    job: DataSourceJob | null;
 }
 interface DataSourceDocument {
     id: string;
@@ -2720,8 +2756,9 @@ interface DataSourceJobPlan {
 }
 /**
  * A bulk ingestion job — a sync, a corpus load started from the CLI, a remap
- * of a mapped source's raw copies, a replay of an earlier job's quarantine, or
- * a reread of the source's documents by its reader.
+ * of a mapped source's raw copies, a replay of an earlier job's quarantine, a
+ * reread of the source's documents by its reader, or the rest of one large
+ * object an {@link DataSource.add} took.
  */
 interface DataSourceJob {
     id: string;
@@ -2750,6 +2787,9 @@ interface DataSourceJob {
     } | {
         type: 'reread';
         unread: boolean;
+    } | {
+        type: 'object';
+        ref: string;
     };
     /**
      * Objects the mapper skipped or failed on, and documents the reader failed
@@ -3070,7 +3110,15 @@ declare class DataSource {
      * as the new one is registered, before its build lands; a search in that
      * window sees neither. **Throws `mapper_skipped`** (HTTP 422) when the
      * mapper skipped the object — a specific thing it refused, which the caller
-     * should see — and `mapper_error` when the mapper threw.
+     * should see — and `mapper_error` when the mapper threw, or when none of
+     * its documents passed validation. A document that fails is left out and
+     * reported in {@link AddResult.invalid}; the rest are registered.
+     *
+     * An object that maps to more than a thousand documents (a large export,
+     * streamed out of the mapper as it parses) registers its first thousand
+     * here and returns {@link AddResult.job}, a job that registers and builds
+     * the rest; `documentCount` says how many there are in all. Another job on
+     * the source refuses it with `data_source_busy`.
      */
     add(content: Buffer | Uint8Array | string, options: AddOptions): Promise<AddResult>;
     /**
@@ -3221,6 +3269,17 @@ declare class DataSource {
      * document has been added.
      */
     ensure(name?: string): Promise<void>;
+    /**
+     * @internal A mapper frame's grant to store the next part of its spill: how
+     * `defineMapper`'s executor hands the platform an object's documents a part
+     * at a time once they outgrow the frame's record. Only a mapper frame's
+     * token is granted one.
+     */
+    _mapSpillGrant(spillId: string): Promise<{
+        part: number;
+        uploadUrl: string;
+        uploadFields: Record<string, string>;
+    }>;
     /**
      * @internal Content hash of some bytes, matching what the server computes.
      * Exposed for callers that want to check whether they already added a file.
@@ -11659,7 +11718,9 @@ declare function runWithContext<T>(ctx: RequestContext, fn: () => T | Promise<T>
  * - `documents([...])` — the object becomes these documents (an array is
  *   one object → many). Each is `{ externalId, title, markdown, metadata?,
  *   entities?, replaces? }`; `externalId` is the identity the platform
- *   replaces by.
+ *   replaces by. Any number: pass a generator (sync or async) to stream an
+ *   export of a million rows without holding it. A document that fails
+ *   validation is left out and named; the rest of the object goes in.
  * - `passthrough({ metadata?, entities? })` — ingest the raw object itself through
  *   built-in extraction, as an unmapped source would. What lets a mapped
  *   source still accept a PDF.
@@ -11672,10 +11733,14 @@ declare function runWithContext<T>(ctx: RequestContext, fn: () => T | Promise<T>
  * `defineMapper` returns a CALLABLE — the executor — with the config attached
  * (the same pattern as `defineJewel`). The platform runs it as an ordinary
  * execution frame: one call maps a slice of objects and returns their
- * outcomes. Everything a method can do, a mapper can do — call models via
+ * outcomes, and stores the documents its objects produce as it goes, a part
+ * of up to a thousand at a time, so a frame's memory holds one part
+ * whatever its objects become. Everything a method can do, a mapper can do — call models via
  * `runTask`, `fetch` an external API, read the app's tables — and every call
  * is spend per object. The mapper's `map` runs under a per-object timeout
- * (declared in `mindstudio.json`, default 30 s); the executor never throws.
+ * (declared in `mindstudio.json`, default 30 s), which also bounds the wait
+ * for each next document a generator yields; a whole object has the frame's
+ * twenty minutes. The executor never throws.
  *
  * On a job, mapping is its own stage: the mapper turns each object into
  * documents, and the platform ingests those documents in parallel batches of
@@ -11730,9 +11795,11 @@ interface MappedDocument {
     /** External ids this document replaces (a writethru's original id). */
     replaces?: string | string[];
 }
+/** What `documents()` takes: one document, an array, or any iterable or async iterable. */
+type MappedDocuments = MappedDocument | Iterable<MappedDocument> | AsyncIterable<MappedDocument>;
 type MapOutcome = {
     kind: 'documents';
-    documents: MappedDocument[];
+    documents: Iterable<MappedDocument> | AsyncIterable<MappedDocument>;
 } | {
     kind: 'passthrough';
     metadata?: DocumentMetadata;
@@ -11757,15 +11824,13 @@ interface MapRunObject {
     etag: string | null;
     lastModified: string | null;
     metadata: DocumentMetadata | null;
-    /** Absent from a platform that predates entities. */
-    entities?: string[] | null;
+    entities: string[] | null;
     /** Short-lived read URL for the object's bytes. */
     url: string;
     /**
      * The object's bytes, base64, when the platform already held them (it
      * hashes and lands every object before mapping, and small objects ride
-     * along). Present, it is the bytes; `url` is the fallback for large objects
-     * and for frames from a platform that predates it.
+     * along). Present, it is the bytes; `url` is the fallback for large objects.
      */
     content?: string;
 }
@@ -11776,8 +11841,26 @@ interface MapRunParams {
     timeoutMs?: number;
     /** Objects mapped at once inside the frame. */
     concurrency?: number;
+    /** Where the frame's documents are stored, a part at a time. */
+    spill?: {
+        id: string;
+    };
 }
-type MapRunOutcome = MapOutcome | {
+/**
+ * One stored part of a frame's documents: gzipped NDJSON, a line `{ o, i, d }`
+ * per document (the object's index in the frame, the document's index in
+ * that object's output, the document).
+ */
+interface MapRunPart {
+    part: number;
+    count: number;
+}
+type MapRunOutcome = {
+    kind: 'documents';
+    count: number;
+} | Exclude<MapOutcome, {
+    kind: 'documents';
+}> | {
     kind: 'error';
     error: {
         message: string;
@@ -11794,6 +11877,8 @@ interface MapRunRecord {
     v: 1;
     source: string;
     outcomes: MapRunResult[];
+    /** Every document the frame's objects produced, in the order stored. */
+    parts: MapRunPart[];
     startedAt: number;
     durationMs: number;
 }
@@ -11805,8 +11890,11 @@ interface Mapper {
     readonly source: string;
     readonly config: MapperConfig;
 }
-/** The object becomes these documents. One document may be passed bare. */
-declare function documents(list: MappedDocument | MappedDocument[]): MapOutcome;
+/**
+ * The object becomes these documents: one passed bare, an array, or a
+ * generator that yields them, which streams an export of any size.
+ */
+declare function documents(list: MappedDocuments): MapOutcome;
 /**
  * Ingest the raw object itself through built-in extraction. `metadata` and
  * `entities`, when given, replace what the origin supplied.