@mindstudio-ai/agent 0.1.139 → 0.1.141

package/dist/index.d.ts

@@ -2332,6 +2332,21 @@ interface SearchExplain {
     lexical: BranchPosition | null;
     matchedVia: 'dense' | 'lexical' | 'both';
 }
+/** One passage of a listing ({@link DataSource.passages}): a search hit without the ranking. */
+interface DataSourcePassage {
+    /** The chunk, prefixed with its heading path for context. */
+    text: string;
+    /** The ids the passage carries: its document's, and those its text names from the dictionary. */
+    entities: string[];
+    citation: Citation;
+}
+/** A page of {@link DataSource.passages}. */
+interface DataSourcePassagePage {
+    passages: DataSourcePassage[];
+    /** Pass back for the next page; null after the last. */
+    nextCursor: string | null;
+    latencyMs: number;
+}
 interface SearchHit {
     /**
      * Relevance of this hit. Comparable within a response, not across them —
@@ -2528,6 +2543,14 @@ interface AddOptions {
      * supplying them replaces the whole set, and `[]` clears it.
      */
     entities?: string[];
+    /**
+     * On a mapped source, the budget in dollars of the job that takes an
+     * object's documents past the first thousand ({@link AddResult.job}): it
+     * pauses there, and a plan projected over it waits for an approval rather
+     * than starting. Without one the job runs within its plan's projection,
+     * with headroom.
+     */
+    budgetDollars?: number;
 }
 /** What {@link DataSource.add} returns. */
 interface AddResult {
@@ -2536,8 +2559,14 @@ interface AddResult {
      * mapper answered `deletes` — nothing was added.
      */
     document: DataSourceDocument | null;
-    /** Every document the bytes became. One entry on an unmapped source. */
+    /**
+     * Every document the bytes became, or the first thousand of an object that
+     * maps to more ({@link job} takes the rest). One entry on an unmapped
+     * source.
+     */
     documents: DataSourceDocument[];
+    /** How many documents the bytes map to in all: `documents.length` unless {@link job} is set. */
+    documentCount: number;
     /** Whether any of them was queued for a build (the rest were already current). */
     queued: boolean;
     /**
@@ -2547,6 +2576,28 @@ interface AddResult {
     outcome: 'unmapped' | 'documents' | 'passthrough' | 'deletes';
     /** Documents removed or superseded as a result — a `deletes` outcome, a `replaces`. */
     removed: number;
+    /**
+     * The documents the mapper produced that failed validation and were not
+     * registered, by their index in its output (`document: 1234` is the
+     * 1,235th it returned or yielded), with why; null when every one passed.
+     * The rest are registered. `add()` throws `mapper_error` only when none
+     * passed.
+     */
+    invalid: {
+        count: number;
+        examples: {
+            document: number;
+            reason: string;
+        }[];
+    } | null;
+    /**
+     * The job registering and building the rest of an object that maps to more
+     * than a thousand documents; null otherwise. It plans and starts on its own
+     * (within {@link AddOptions.budgetDollars}); follow it with
+     * {@link DataSource.job}. What the object supersedes or retires goes when
+     * the job finishes, once every one of its documents is searchable.
+     */
+    job: DataSourceJob | null;
 }
 interface DataSourceDocument {
     id: string;
@@ -2705,8 +2756,9 @@ interface DataSourceJobPlan {
 }
 /**
  * A bulk ingestion job — a sync, a corpus load started from the CLI, a remap
- * of a mapped source's raw copies, a replay of an earlier job's quarantine, or
- * a reread of the source's documents by its reader.
+ * of a mapped source's raw copies, a replay of an earlier job's quarantine, a
+ * reread of the source's documents by its reader, or the rest of one large
+ * object an {@link DataSource.add} took.
  */
 interface DataSourceJob {
     id: string;
@@ -2735,6 +2787,9 @@ interface DataSourceJob {
     } | {
         type: 'reread';
         unread: boolean;
+    } | {
+        type: 'object';
+        ref: string;
     };
     /**
      * Objects the mapper skipped or failed on, and documents the reader failed
@@ -2818,9 +2873,14 @@ interface DictionaryReport {
     names: number;
     /** Names more than one id claims: a passage naming one is tagged with every claimant. */
     ambiguousNames: number;
-    /** The names (as matched: folded, separators dropped) the most ids claim. */
+    /**
+     * The names the most ids claim: `name` as you pushed it (one of them, when
+     * several fold alike), `key` as the linker matches it (folded, separators
+     * dropped), and how many ids claim it.
+     */
     mostClaimed: {
         key: string;
+        name?: string;
         ids: number;
     }[];
     /**
@@ -2959,7 +3019,8 @@ declare class DataSource {
      * it is who or what was written about then. No filter covers the corpus.
      * `chunks` is how many matching passages carry each id, not documents: a
      * long document's passages each count. Approximate counts, ordered most
-     * first; `limit` defaults to 20, at most 100.
+     * first; `limit` defaults to 20, at most 1,000. One document's entities:
+     * `{ filter: { documentIds: [id] } }`.
      *
      * Shares the search rate limit (300/minute per app), and shows the same
      * `index_warming` / `index_building` as search while a corpus loads.
@@ -3049,7 +3110,15 @@ declare class DataSource {
      * as the new one is registered, before its build lands; a search in that
      * window sees neither. **Throws `mapper_skipped`** (HTTP 422) when the
      * mapper skipped the object — a specific thing it refused, which the caller
-     * should see — and `mapper_error` when the mapper threw.
+     * should see — and `mapper_error` when the mapper threw, or when none of
+     * its documents passed validation. A document that fails is left out and
+     * reported in {@link AddResult.invalid}; the rest are registered.
+     *
+     * An object that maps to more than a thousand documents (a large export,
+     * streamed out of the mapper as it parses) registers its first thousand
+     * here and returns {@link AddResult.job}, a job that registers and builds
+     * the rest; `documentCount` says how many there are in all. Another job on
+     * the source refuses it with `data_source_busy`.
      */
     add(content: Buffer | Uint8Array | string, options: AddOptions): Promise<AddResult>;
     /**
@@ -3075,6 +3144,38 @@ declare class DataSource {
         limit?: number;
         order?: 'oldest' | 'newest';
     }): Promise<DataSourceDocumentPage>;
+    /**
+     * One page of the passages matching a filter, unranked: every page that
+     * mentions an asset (`{ entities: ['P-1004A'] }`), every passage holding a
+     * phrase, every chunk of some documents. Where {@link search} ranks and
+     * stops at its top results and {@link count} gives a number, this lists
+     * them all. Passages come document by document, each document's in order;
+     * pass `nextCursor` back for the next page, which is null after the last.
+     * `limit` defaults to 50, at most 200.
+     *
+     * Shares the search rate limit (300/minute per app), and shows the same
+     * `index_warming` / `index_building` as search while a corpus loads.
+     */
+    passages(options?: {
+        filter?: SearchFilter;
+        cursor?: string | null;
+        limit?: number;
+    }): Promise<DataSourcePassagePage>;
+    /**
+     * Every passage matching a filter, a page at a time behind the scenes: the
+     * walk over {@link passages}. Each page counts against the search rate
+     * limit, so a large walk belongs in a background task.
+     *
+     * ```ts
+     * for await (const passage of Manuals.allPassages({ entities: ['P-1004A'] })) {
+     *   mentions.push({ asset: 'P-1004A', ...passage.citation });
+     * }
+     * ```
+     */
+    allPassages(filter?: SearchFilter, options?: {
+        /** Passages per page, at most 200 (the default). */
+        pageSize?: number;
+    }): AsyncGenerator<DataSourcePassage, void, undefined>;
     /**
      * Every document in the corpus, oldest first, a page at a time behind the
      * scenes. This is how an app builds its own view of a large corpus (a
@@ -3168,6 +3269,17 @@ declare class DataSource {
      * document has been added.
      */
     ensure(name?: string): Promise<void>;
+    /**
+     * @internal A mapper frame's grant to store the next part of its spill: how
+     * `defineMapper`'s executor hands the platform an object's documents a part
+     * at a time once they outgrow the frame's record. Only a mapper frame's
+     * token is granted one.
+     */
+    _mapSpillGrant(spillId: string): Promise<{
+        part: number;
+        uploadUrl: string;
+        uploadFields: Record<string, string>;
+    }>;
     /**
      * @internal Content hash of some bytes, matching what the server computes.
      * Exposed for callers that want to check whether they already added a file.
@@ -11606,7 +11718,9 @@ declare function runWithContext<T>(ctx: RequestContext, fn: () => T | Promise<T>
  * - `documents([...])` — the object becomes these documents (an array is
  *   one object → many). Each is `{ externalId, title, markdown, metadata?,
  *   entities?, replaces? }`; `externalId` is the identity the platform
- *   replaces by.
+ *   replaces by. Any number: pass a generator (sync or async) to stream an
+ *   export of a million rows without holding it. A document that fails
+ *   validation is left out and named; the rest of the object goes in.
  * - `passthrough({ metadata?, entities? })` — ingest the raw object itself through
  *   built-in extraction, as an unmapped source would. What lets a mapped
  *   source still accept a PDF.
@@ -11619,10 +11733,14 @@ declare function runWithContext<T>(ctx: RequestContext, fn: () => T | Promise<T>
  * `defineMapper` returns a CALLABLE — the executor — with the config attached
  * (the same pattern as `defineJewel`). The platform runs it as an ordinary
  * execution frame: one call maps a slice of objects and returns their
- * outcomes. Everything a method can do, a mapper can do — call models via
+ * outcomes, and stores the documents its objects produce as it goes, a part
+ * of up to a thousand at a time, so a frame's memory holds one part
+ * whatever its objects become. Everything a method can do, a mapper can do — call models via
  * `runTask`, `fetch` an external API, read the app's tables — and every call
  * is spend per object. The mapper's `map` runs under a per-object timeout
- * (declared in `mindstudio.json`, default 30 s); the executor never throws.
+ * (declared in `mindstudio.json`, default 30 s), which also bounds the wait
+ * for each next document a generator yields; a whole object has the frame's
+ * twenty minutes. The executor never throws.
  *
  * On a job, mapping is its own stage: the mapper turns each object into
  * documents, and the platform ingests those documents in parallel batches of
@@ -11677,9 +11795,11 @@ interface MappedDocument {
     /** External ids this document replaces (a writethru's original id). */
     replaces?: string | string[];
 }
+/** What `documents()` takes: one document, an array, or any iterable or async iterable. */
+type MappedDocuments = MappedDocument | Iterable<MappedDocument> | AsyncIterable<MappedDocument>;
 type MapOutcome = {
     kind: 'documents';
-    documents: MappedDocument[];
+    documents: Iterable<MappedDocument> | AsyncIterable<MappedDocument>;
 } | {
     kind: 'passthrough';
     metadata?: DocumentMetadata;
@@ -11704,15 +11824,13 @@ interface MapRunObject {
     etag: string | null;
     lastModified: string | null;
     metadata: DocumentMetadata | null;
-    /** Absent from a platform that predates entities. */
-    entities?: string[] | null;
+    entities: string[] | null;
     /** Short-lived read URL for the object's bytes. */
     url: string;
     /**
      * The object's bytes, base64, when the platform already held them (it
      * hashes and lands every object before mapping, and small objects ride
-     * along). Present, it is the bytes; `url` is the fallback for large objects
-     * and for frames from a platform that predates it.
+     * along). Present, it is the bytes; `url` is the fallback for large objects.
      */
     content?: string;
 }
@@ -11723,8 +11841,26 @@ interface MapRunParams {
     timeoutMs?: number;
     /** Objects mapped at once inside the frame. */
     concurrency?: number;
+    /** Where the frame's documents are stored, a part at a time. */
+    spill?: {
+        id: string;
+    };
+}
+/**
+ * One stored part of a frame's documents: gzipped NDJSON, a line `{ o, i, d }`
+ * per document (the object's index in the frame, the document's index in
+ * that object's output, the document).
+ */
+interface MapRunPart {
+    part: number;
+    count: number;
 }
-type MapRunOutcome = MapOutcome | {
+type MapRunOutcome = {
+    kind: 'documents';
+    count: number;
+} | Exclude<MapOutcome, {
+    kind: 'documents';
+}> | {
     kind: 'error';
     error: {
         message: string;
@@ -11741,6 +11877,8 @@ interface MapRunRecord {
     v: 1;
     source: string;
     outcomes: MapRunResult[];
+    /** Every document the frame's objects produced, in the order stored. */
+    parts: MapRunPart[];
     startedAt: number;
     durationMs: number;
 }
@@ -11752,8 +11890,11 @@ interface Mapper {
     readonly source: string;
     readonly config: MapperConfig;
 }
-/** The object becomes these documents. One document may be passed bare. */
-declare function documents(list: MappedDocument | MappedDocument[]): MapOutcome;
+/**
+ * The object becomes these documents: one passed bare, an array, or a
+ * generator that yields them, which streams an export of any size.
+ */
+declare function documents(list: MappedDocuments): MapOutcome;
 /**
  * Ingest the raw object itself through built-in extraction. `metadata` and
  * `entities`, when given, replace what the origin supplied.
@@ -11816,7 +11957,12 @@ declare function defineMapper(source: {
  *   what named the document.
  * - `cleared(sourceId)` — the whole source was deleted, once, so the app can
  *   drop that source's rows in one statement. Optional.
- * A throw fails that delivery; the platform retries it.
+ * - `relinked(document)` — a document already read was re-tagged by a new
+ *   dictionary: its passages carry different entities, its text is the same.
+ *   For rows derived from the tags. Optional.
+ * A throw fails that delivery; the platform retries it. `document.bytes()`
+ * reads the file the document came from, for a reader that needs the drawing
+ * itself rather than its text.
  *
  * A reader runs against the app's LIVE release (or the dev session under
  * `reader test --dev`), never a pinned one: app databases are per release, so
@@ -11844,26 +11990,47 @@ interface ReaderDocument {
     pageCount: number | null;
     chunkCount: number | null;
     createdAt: string;
+    /**
+     * A short-lived link to the file the document came from: the uploaded
+     * file, or on a mapped source the raw object the mapper read. Valid for the
+     * delivery; null when the file is no longer kept.
+     */
+    url: string | null;
     /**
      * The document's passages as indexed: text, page, heading path, and each
      * passage's entities (its document's asserted ones and those its text
      * names from the source's dictionary). Read once, then held.
      */
     chunks(): Promise<DataSourceChunk[]>;
+    /** The bytes behind {@link url}: a drawing's image, a PDF's pages. Read once, then held. */
+    bytes(): Promise<Uint8Array>;
 }
 type MaybePromise<T> = T | Promise<T>;
 interface ReaderConfig {
     read: (document: ReaderDocument) => MaybePromise<void>;
     removed: (documentId: string) => MaybePromise<void>;
     cleared?: (sourceId: string) => MaybePromise<void>;
+    /**
+     * A document already read was re-tagged: the source's dictionary changed,
+     * and its passages' entities with it. The document arrives as `read` gets
+     * it, chunks and all, carrying the new entities, so rows derived from the
+     * tags (links from assets to documents) can follow. Nothing about the text
+     * changed, so this is no reason to extract again. Optional: without it a
+     * relink delivers nothing.
+     */
+    relinked?: (document: ReaderDocument) => MaybePromise<void>;
 }
-/** One delivery as the platform hands it to the executor. */
-type ReadDelivery = {
-    kind: 'read';
-    document: Omit<ReaderDocument, 'chunks'>;
+type DocumentDelivery = {
+    document: Omit<ReaderDocument, 'chunks' | 'bytes'>;
     /** The passages, when the platform carried them inline. */
     chunks?: DataSourceChunk[];
-} | {
+};
+/** One delivery as the platform hands it to the executor. */
+type ReadDelivery = ({
+    kind: 'read';
+} & DocumentDelivery) | ({
+    kind: 'relinked';
+} & DocumentDelivery) | {
     kind: 'removed';
     documentId: string;
 } | {
@@ -11894,6 +12061,10 @@ interface ReadRunRecord {
     v: 1;
     source: string;
     outcomes: ReadRunResult[];
+    /** The optional hooks this reader defines, so the platform sends only what one handles. */
+    hooks: {
+        relinked: boolean;
+    };
     startedAt: number;
     durationMs: number;
 }