@mindstudio-ai/agent 0.1.86 → 0.1.88

package/dist/index.d.ts

@@ -1508,6 +1508,16 @@ interface Citation {
     filename: string | null;
     /** 1-based. Null for formats with no pagination (plain text, html). */
     pageNumber: number | null;
+    /**
+     * Position within the document, 0-based.
+     *
+     * `(documentId, chunkIndex)` is the stable identity of a chunk — use it as
+     * the key for an eval set or a regression check rather than matching on
+     * `text`. Stable for as long as the corpus keeps its current configuration:
+     * re-chunking a source (a different chunk size, say) moves the boundaries and
+     * therefore renumbers, which is inherent rather than a wobble.
+     */
+    chunkIndex: number | null;
     /** Enclosing headings, outermost first. */
     headingPath: string[];
     /**
@@ -1527,12 +1537,49 @@ interface Citation {
      */
     url: string;
 }
+/** Where one retrieval branch put a hit, and what that branch scored it. */
+interface BranchPosition {
+    /** 0-based position within that branch's own results. */
+    rank: number;
+    score: number;
+}
+/**
+ * Which half of hybrid retrieval found a hit. Only present when `explain` was
+ * requested — see {@link SearchOptions.explain}.
+ */
+interface SearchExplain {
+    /** Semantic (embedding) retrieval. Null if this branch didn't find the hit. */
+    dense: BranchPosition | null;
+    /** Keyword/IDF retrieval. Null when `hybrid` is off, or if it didn't find it. */
+    lexical: BranchPosition | null;
+    matchedVia: 'dense' | 'lexical' | 'both';
+}
 interface SearchHit {
     /** Provider relevance score. Comparable within a response, not across them. */
     score: number;
     /** The matched chunk, prefixed with its heading path for context. */
     text: string;
     citation: Citation;
+    /**
+     * Where retrieval put this hit BEFORE reranking, and what it scored.
+     *
+     * With reranking on, `score` is the reranker's relevance score and this is
+     * the retriever's — different quantities, so they're kept apart rather than
+     * blended. Comparing `retrievalRank` with the hit's final position is how you
+     * see what reranking actually did ("retrieved 7th, reranked to 1st").
+     *
+     * Named for the stage rather than the method: it's a fused hybrid score when
+     * `hybrid` is on and a cosine similarity when it's off.
+     */
+    retrievalRank?: number;
+    retrievalScore?: number;
+    /** Only when `explain` was requested. */
+    explain?: SearchExplain;
+    /** Only when `expand` was requested. Outermost first, so `[...before, text, ...after]` reads in order. */
+    neighbors?: {
+        before: string[];
+        after: string[];
+    };
 }
 /**
  * Per-query overrides.
@@ -1566,6 +1613,23 @@ interface SearchOptions {
      * that an embedding model never learned. Rarely worth disabling.
      */
     hybrid?: boolean;
+    /**
+     * Report which branch found each hit, and where each ranked it.
+     *
+     * A debugging aid, off by default because it costs two extra round trips: a
+     * fused result carries one blended score, so the branches have to be asked
+     * separately. Results and their order are identical either way — this only
+     * adds {@link SearchHit.explain}.
+     */
+    explain?: boolean;
+    /**
+     * Also return this many chunks either side of each hit, in
+     * {@link SearchHit.neighbors} — for showing a passage in context. 0-2.
+     *
+     * `text` is untouched, so citations and highlighting still point at the
+     * chunk that actually matched.
+     */
+    expand?: number;
 }
 interface AddOptions {
     /**
@@ -1585,6 +1649,68 @@ interface DataSourceDocument {
     createdAt: string;
     ingestedAt: string | null;
 }
+/** One chunk exactly as it was indexed. See {@link DataSource.chunks}. */
+interface DataSourceChunk {
+    index: number;
+    text: string;
+    pageNumber: number;
+    headingPath: string[];
+    /**
+     * Offsets into the page's extracted markdown. Null for PDFs, which carry a
+     * `boundingBox` instead — there is no character stream to point into.
+     */
+    charStart: number | null;
+    charEnd: number | null;
+    boundingBox?: {
+        topLeftX: number;
+        topLeftY: number;
+        bottomRightX: number;
+        bottomRightY: number;
+    };
+    /** base64 of a Float32Array. Only when `vectors: true` was passed. */
+    vector?: string;
+}
+/** How a corpus was built and what is in it. See {@link DataSource.stats}. */
+interface DataSourceStats {
+    /** False for a source nothing has created yet — everything else reads zero. */
+    exists: boolean;
+    documentCount: number;
+    counts: {
+        total: number;
+        done: number;
+        processing: number;
+        error: number;
+    };
+    chunkCount: number;
+    /** Original document bytes, not index size. */
+    storageBytes: number;
+    lastIngestedAt: string | null;
+    /**
+     * The configuration these documents were actually built with — not the
+     * platform default, and not necessarily the newest. Changing it is an
+     * explicit, owner-triggered migration.
+     */
+    pipeline: {
+        version: number;
+        embeddingModelId: string;
+        dimensions: number;
+        chunking: {
+            strategy: string;
+            version: number;
+            maxChars: number;
+            minChars: number;
+            dropBlockTypes: string[];
+        };
+        contextual: {
+            enabled: boolean;
+            modelId: string | null;
+        };
+        images: {
+            describe: boolean;
+            modelId: string | null;
+        };
+    } | null;
+}
 /**
  * A typed handle to one data source. Lazy — nothing executes until a method is
  * awaited, so it's safe to `defineDataSource()` at module scope and import the
@@ -1602,10 +1728,39 @@ declare class DataSource {
      * Returns chunks ranked by relevance, each with a citation. Searching a
      * source that doesn't exist yet returns no results rather than throwing —
      * code may name a corpus the build hasn't populated.
+     *
+     * **Deterministic** for a fixed corpus and configuration: the same query
+     * returns the same hits in the same order, so it's safe to build an eval set
+     * or a regression check on top of it. There is no seed to set. Two things do
+     * legitimately move the results: adding or removing documents, and changing
+     * the corpus configuration — both of which you control.
      */
     search(query: string, options?: SearchOptions): Promise<{
         results: SearchHit[];
+        latencyMs: number;
     }>;
+    /**
+     * What is in the corpus, and how it was built.
+     *
+     * Document and chunk counts, storage, and the embedding model and chunking
+     * settings actually in effect — which is not the same as the platform
+     * default, since a corpus keeps the configuration it was built with until
+     * someone migrates it.
+     */
+    stats(): Promise<DataSourceStats>;
+    /**
+     * Every chunk of one document, exactly as it was indexed.
+     *
+     * The direct answer to "why isn't this document coming back?" — search only
+     * shows you the chunks that surface, which is no help when none do. Reading
+     * how a document was actually split usually is.
+     *
+     * Pass `{ vectors: true }` to include each chunk's embedding. Large: roughly
+     * 8KB per chunk, so a 500-chunk document is several megabytes.
+     */
+    chunks(documentId: string, options?: {
+        vectors?: boolean;
+    }): Promise<DataSourceChunk[]>;
     /**
      * Add a document to the corpus.
      *
@@ -2300,6 +2455,12 @@ declare class MindStudioAgent$1 {
      *
      * Gets a presigned upload request from the API, POSTs the file as
      * multipart/form-data, and returns the permanent public URL.
+     *
+     * @deprecated For app file storage use the `files` store
+     * (`files.defineStore(...).put(...)`) — private by default, app-scoped, and
+     * served on the app's own domain. `uploadFile` uploads to the shared account
+     * media CDN; it remains only for account-level assets (e.g. an agent avatar
+     * passed to `changeProfilePicture`).
      */
     uploadFile(content: Buffer | Uint8Array, options: {
         extension: string;
@@ -6708,7 +6869,7 @@ interface StepMethods {
      */
     convertPdfToImages(step: ConvertPdfToImagesStepInput, options?: StepExecutionOptions): Promise<StepExecutionResult<ConvertPdfToImagesStepOutput>>;
     /**
-     * Create a new empty vector data source for the current app.
+     * @deprecated Legacy v1 data sources. v2 apps should use `dataSources.defineDataSource()` from @mindstudio-ai/agent instead, which owns parsing, chunking, embedding and citations. Create a new empty vector data source for the current app.
      *
      * @remarks
      * - Creates a new data source (vector database) associated with the current app version.
@@ -6790,7 +6951,7 @@ interface StepMethods {
      */
     createGoogleSheet(step: CreateGoogleSheetStepInput, options?: StepExecutionOptions): Promise<StepExecutionResult<CreateGoogleSheetStepOutput>>;
     /**
-     * Delete a vector data source from the current app.
+     * @deprecated Legacy v1 data sources. v2 apps should use `dataSources.defineDataSource()` from @mindstudio-ai/agent instead, which owns parsing, chunking, embedding and citations. Delete a vector data source from the current app.
      *
      * @remarks
      * - Soft-deletes a data source (vector database) by marking it as deleted.
@@ -6806,7 +6967,7 @@ interface StepMethods {
      */
     deleteDataSource(step: DeleteDataSourceStepInput, options?: StepExecutionOptions): Promise<StepExecutionResult<DeleteDataSourceStepOutput>>;
     /**
-     * Delete a single document from a data source.
+     * @deprecated Legacy v1 data sources. v2 apps should use `dataSources.defineDataSource()` from @mindstudio-ai/agent instead, which owns parsing, chunking, embedding and citations. Delete a single document from a data source.
      *
      * @remarks
      * - Soft-deletes a document by marking it as deleted.
@@ -7065,7 +7226,7 @@ interface StepMethods {
      */
     extractText(step: ExtractTextStepInput, options?: StepExecutionOptions): Promise<StepExecutionResult<ExtractTextStepOutput>>;
     /**
-     * Fetch the full extracted text contents of a document in a data source.
+     * @deprecated Legacy v1 data sources. v2 apps should use `dataSources.defineDataSource()` from @mindstudio-ai/agent instead, which owns parsing, chunking, embedding and citations. Fetch the full extracted text contents of a document in a data source.
      *
      * @remarks
      * - Loads a document by ID and returns its full extracted text content.
@@ -7639,7 +7800,7 @@ interface StepMethods {
      */
     insertVideoClips(step: InsertVideoClipsStepInput, options?: StepExecutionOptions): Promise<StepExecutionResult<InsertVideoClipsStepOutput>>;
     /**
-     * List all data sources for the current app.
+     * @deprecated Legacy v1 data sources. v2 apps should use `dataSources.defineDataSource()` from @mindstudio-ai/agent instead, which owns parsing, chunking, embedding and citations. List all data sources for the current app.
      *
      * @remarks
      * - Returns metadata for every data source associated with the current app version.
@@ -8182,7 +8343,7 @@ interface StepMethods {
      */
     queryAppDatabase(step: QueryAppDatabaseStepInput, options?: StepExecutionOptions): Promise<StepExecutionResult<QueryAppDatabaseStepOutput>>;
     /**
-     * Search a vector data source (RAG) and return relevant document chunks.
+     * @deprecated Legacy v1 data sources. v2 apps should use `dataSources.defineDataSource()` from @mindstudio-ai/agent instead, which owns parsing, chunking, embedding and citations. Search a vector data source (RAG) and return relevant document chunks.
      *
      * @remarks
      * - Queries a vectorized data source and returns the most relevant chunks.
@@ -9012,7 +9173,7 @@ interface StepMethods {
      */
     updateGoogleSheet(step: UpdateGoogleSheetStepInput, options?: StepExecutionOptions): Promise<StepExecutionResult<UpdateGoogleSheetStepOutput>>;
     /**
-     * Upload a file into an existing data source from a URL or raw text content.
+     * @deprecated Legacy v1 data sources. v2 apps should use `dataSources.defineDataSource()` from @mindstudio-ai/agent instead, which owns parsing, chunking, embedding and citations. Upload a file into an existing data source from a URL or raw text content.
      *
      * @remarks
      * - If "file" is a single URL, the file is downloaded from that URL and uploaded.

package/dist/index.js

@@ -2128,17 +2128,54 @@ var DataSource = class {
    * Returns chunks ranked by relevance, each with a citation. Searching a
    * source that doesn't exist yet returns no results rather than throwing —
    * code may name a corpus the build hasn't populated.
+   *
+   * **Deterministic** for a fixed corpus and configuration: the same query
+   * returns the same hits in the same order, so it's safe to build an eval set
+   * or a regression check on top of it. There is no seed to set. Two things do
+   * legitimately move the results: adding or removing documents, and changing
+   * the corpus configuration — both of which you control.
    */
   async search(query, options) {
-    const { results } = await this._call("search", {
+    const { results, latencyMs } = await this._call("search", {
       slug: this._slug,
       query,
       ...options?.topK !== void 0 ? { topK: options.topK } : {},
       ...options?.scoreThreshold !== void 0 ? { scoreThreshold: options.scoreThreshold } : {},
       ...options?.rerank !== void 0 ? { rerank: options.rerank } : {},
-      ...options?.hybrid !== void 0 ? { hybrid: options.hybrid } : {}
+      ...options?.hybrid !== void 0 ? { hybrid: options.hybrid } : {},
+      ...options?.explain !== void 0 ? { explain: options.explain } : {},
+      ...options?.expand !== void 0 ? { expand: options.expand } : {}
     });
-    return { results: results ?? [] };
+    return { results: results ?? [], latencyMs: latencyMs ?? 0 };
+  }
+  /**
+   * What is in the corpus, and how it was built.
+   *
+   * Document and chunk counts, storage, and the embedding model and chunking
+   * settings actually in effect — which is not the same as the platform
+   * default, since a corpus keeps the configuration it was built with until
+   * someone migrates it.
+   */
+  async stats() {
+    return this._call("stats", { slug: this._slug });
+  }
+  /**
+   * Every chunk of one document, exactly as it was indexed.
+   *
+   * The direct answer to "why isn't this document coming back?" — search only
+   * shows you the chunks that surface, which is no help when none do. Reading
+   * how a document was actually split usually is.
+   *
+   * Pass `{ vectors: true }` to include each chunk's embedding. Large: roughly
+   * 8KB per chunk, so a 500-chunk document is several megabytes.
+   */
+  async chunks(documentId, options) {
+    const { chunks } = await this._call("chunks", {
+      slug: this._slug,
+      documentId,
+      ...options?.vectors ? { vectors: true } : {}
+    });
+    return chunks ?? [];
   }
   /**
    * Add a document to the corpus.
@@ -2325,7 +2362,7 @@ var stepMetadata = {
   },
   "createDataSource": {
     stepType: "createDataSource",
-    description: "Create a new empty vector data source for the current app.",
+    description: "Legacy v1 data sources. v2 apps should use `dataSources.defineDataSource()` from @mindstudio-ai/agent instead, which owns parsing, chunking, embedding and citations. Create a new empty vector data source for the current app.",
     usageNotes: '- Creates a new data source (vector database) associated with the current app version.\n- The data source is created empty \u2014 use the "Upload Data Source Document" block to add documents.\n- Returns the new data source ID which can be used in subsequent blocks.',
     inputSchema: { "type": "object", "properties": { "name": { "type": "string", "description": "Name for the new data source (supports variable interpolation)" } }, "required": ["name"] },
     outputSchema: { "description": "This step does not produce output data." }
@@ -2362,14 +2399,14 @@ var stepMetadata = {
   },
   "deleteDataSource": {
     stepType: "deleteDataSource",
-    description: "Delete a vector data source from the current app.",
+    description: "Legacy v1 data sources. v2 apps should use `dataSources.defineDataSource()` from @mindstudio-ai/agent instead, which owns parsing, chunking, embedding and citations. Delete a vector data source from the current app.",
     usageNotes: "- Soft-deletes a data source (vector database) by marking it as deleted.\n- The Milvus partition is cleaned up asynchronously by a background cron job.\n- The data source must belong to the current app version.",
     inputSchema: { "type": "object", "properties": { "dataSourceId": { "type": "string", "description": "ID of the data source to delete (supports variable interpolation)" } }, "required": ["dataSourceId"] },
     outputSchema: { "description": "This step does not produce output data." }
   },
   "deleteDataSourceDocument": {
     stepType: "deleteDataSourceDocument",
-    description: "Delete a single document from a data source.",
+    description: "Legacy v1 data sources. v2 apps should use `dataSources.defineDataSource()` from @mindstudio-ai/agent instead, which owns parsing, chunking, embedding and citations. Delete a single document from a data source.",
     usageNotes: "- Soft-deletes a document by marking it as deleted.\n- Requires both the data source ID and document ID.\n- After deletion, reloads vectors into Milvus so the data source reflects the change immediately.",
     inputSchema: { "type": "object", "properties": { "dataSourceId": { "type": "string", "description": "ID of the data source containing the document (supports variable interpolation)" }, "documentId": { "type": "string", "description": "ID of the document to delete (supports variable interpolation)" } }, "required": ["dataSourceId", "documentId"] },
     outputSchema: { "description": "This step does not produce output data." }
@@ -2474,7 +2511,7 @@ var stepMetadata = {
   },
   "fetchDataSourceDocument": {
     stepType: "fetchDataSourceDocument",
-    description: "Fetch the full extracted text contents of a document in a data source.",
+    description: "Legacy v1 data sources. v2 apps should use `dataSources.defineDataSource()` from @mindstudio-ai/agent instead, which owns parsing, chunking, embedding and citations. Fetch the full extracted text contents of a document in a data source.",
     usageNotes: '- Loads a document by ID and returns its full extracted text content.\n- The document must have been successfully processed (status "done").\n- Also returns document metadata (name, summary, word count).',
     inputSchema: { "type": "object", "properties": { "dataSourceId": { "type": "string", "description": "ID of the data source containing the document (supports variable interpolation)" }, "documentId": { "type": "string", "description": "ID of the document to fetch (supports variable interpolation)" } }, "required": ["dataSourceId", "documentId"] },
     outputSchema: { "description": "This step does not produce output data." }
@@ -2751,7 +2788,7 @@ var stepMetadata = {
   },
   "listDataSources": {
     stepType: "listDataSources",
-    description: "List all data sources for the current app.",
+    description: "Legacy v1 data sources. v2 apps should use `dataSources.defineDataSource()` from @mindstudio-ai/agent instead, which owns parsing, chunking, embedding and citations. List all data sources for the current app.",
     usageNotes: "- Returns metadata for every data source associated with the current app version.\n- Each entry includes the data source ID, name, description, status, and document list.",
     inputSchema: { "type": "object", "properties": {}, "required": [] },
     outputSchema: { "description": "This step does not produce output data." }
@@ -2986,7 +3023,7 @@ var stepMetadata = {
   },
   "queryDataSource": {
     stepType: "queryDataSource",
-    description: "Search a vector data source (RAG) and return relevant document chunks.",
+    description: "Legacy v1 data sources. v2 apps should use `dataSources.defineDataSource()` from @mindstudio-ai/agent instead, which owns parsing, chunking, embedding and citations. Search a vector data source (RAG) and return relevant document chunks.",
     usageNotes: "- Queries a vectorized data source and returns the most relevant chunks.\n- Useful for retrieval-augmented generation (RAG) workflows.",
     inputSchema: { "type": "object", "properties": { "dataSourceId": { "type": "string", "description": "ID of the vector data source to query" }, "query": { "type": "string", "description": "The search query to run against the data source" }, "maxResults": { "type": "number", "description": "Maximum number of chunks to return (recommended 1-3)" } }, "required": ["dataSourceId", "query", "maxResults"] },
     outputSchema: { "type": "object", "properties": { "text": { "type": "string", "description": "All matching chunks joined with newlines" }, "chunks": { "type": "array", "items": { "type": "string" }, "description": "Individual matching text chunks from the data source" }, "query": { "type": "string", "description": "The resolved search query that was executed" }, "citations": { "type": "array", "items": {}, "description": "Source citations for the matched chunks" }, "latencyMs": { "type": "number", "description": "Query execution time in milliseconds" } }, "required": ["text", "chunks", "query", "citations", "latencyMs"] }
@@ -3343,7 +3380,7 @@ var stepMetadata = {
   },
   "uploadDataSourceDocument": {
     stepType: "uploadDataSourceDocument",
-    description: "Upload a file into an existing data source from a URL or raw text content.",
+    description: "Legacy v1 data sources. v2 apps should use `dataSources.defineDataSource()` from @mindstudio-ai/agent instead, which owns parsing, chunking, embedding and citations. Upload a file into an existing data source from a URL or raw text content.",
     usageNotes: '- If "file" is a single URL, the file is downloaded from that URL and uploaded.\n- If "file" is any other string, a .txt document is created from that content and uploaded.\n- The block waits (polls) for processing to complete before transitioning, up to 5 minutes.\n- Once processing finishes, vectors are loaded into Milvus so the data source is immediately queryable.\n- Supported file types (when using a URL) are the same as the data source upload UI (PDF, DOCX, TXT, etc.).',
     inputSchema: { "type": "object", "properties": { "dataSourceId": { "type": "string", "description": "ID of the target data source (supports variable interpolation)" }, "file": { "type": "string", "description": "A URL to download, or raw text content to create a .txt document from (supports variable interpolation)" }, "fileName": { "type": "string", "description": "Display name for the document (supports variable interpolation)" } }, "required": ["dataSourceId", "file", "fileName"] },
     outputSchema: { "description": "This step does not produce output data." }
@@ -5572,6 +5609,12 @@ var MindStudioAgent = class {
    *
    * Gets a presigned upload request from the API, POSTs the file as
    * multipart/form-data, and returns the permanent public URL.
+   *
+   * @deprecated For app file storage use the `files` store
+   * (`files.defineStore(...).put(...)`) — private by default, app-scoped, and
+   * served on the app's own domain. `uploadFile` uploads to the shared account
+   * media CDN; it remains only for account-level assets (e.g. an agent avatar
+   * passed to `changeProfilePicture`).
    */
   async uploadFile(content, options) {
     const filename = options.filename ?? `upload.${options.extension}`;