@mindstudio-ai/agent 0.1.138 → 0.1.140

package/dist/index.js

@@ -3131,6 +3131,8 @@ function createFiles(call) {
 
 // src/datasources/source.ts
 import { createHash as createHash2 } from "crypto";
+var DICTIONARY_PAGE_ENTRIES = 1e4;
+var DICTIONARY_PAGE_BYTES = 4e6;
 var REMOVE_WHERE_MAX_PAGES = 100;
 var DataSource = class {
   constructor(_slug, _call) {
@@ -3251,6 +3253,121 @@ var DataSource = class {
     });
     return { chunks: chunks ?? 0, latencyMs: latencyMs ?? 0 };
   }
+  /**
+   * Which entities the passages matching a filter are about, most first.
+   *
+   * Filter on one entity (`{ entities: ['P-101A'] }`) and the answer is what
+   * appears beside it: a co-mention graph for free. Filter on a date range and
+   * it is who or what was written about then. No filter covers the corpus.
+   * `chunks` is how many matching passages carry each id, not documents: a
+   * long document's passages each count. Approximate counts, ordered most
+   * first; `limit` defaults to 20, at most 1,000. One document's entities:
+   * `{ filter: { documentIds: [id] } }`.
+   *
+   * Shares the search rate limit (300/minute per app), and shows the same
+   * `index_warming` / `index_building` as search while a corpus loads.
+   */
+  async entities(options) {
+    const { entities, latencyMs } = await this._call("entities", {
+      slug: this._slug,
+      ...options?.filter !== void 0 ? { filter: options.filter } : {},
+      ...options?.limit !== void 0 ? { limit: options.limit } : {}
+    });
+    return { entities: entities ?? [], latencyMs: latencyMs ?? 0 };
+  }
+  /**
+   * Push this source's dictionary: every entity id you have and the names it
+   * goes by, replacing the dictionary it had. From then on every passage is
+   * tagged with the ids whose names its text mentions, beside the ones its
+   * document asserts, so `filter.entities` and {@link entities} find a tag
+   * wherever it's written. Passages indexed before are re-tagged in the
+   * background (a relink; see {@link dictionary}), and nothing is
+   * re-embedded.
+   *
+   * Push it whole after each load of the tables it comes from: an unchanged
+   * dictionary commits nothing (`unchanged: true`). Takes an array or any
+   * (async) iterable, and sends it in pages, so a register of a quarter of a
+   * million entries is fine. A repeated id merges its names.
+   */
+  async pushDictionary(entries, options) {
+    const { stageId, pageMaxEntries } = await this._call("dictionary/stage", {
+      slug: this._slug,
+      ...options?.linkerVersion !== void 0 ? { linkerVersion: options.linkerVersion } : {}
+    });
+    const perPage = Math.min(
+      DICTIONARY_PAGE_ENTRIES,
+      Number(pageMaxEntries) || DICTIONARY_PAGE_ENTRIES
+    );
+    let pages = 0;
+    let page = [];
+    let bytes = 0;
+    const send = async () => {
+      if (page.length === 0) {
+        return;
+      }
+      await this._call("dictionary/page", {
+        slug: this._slug,
+        stageId,
+        page: pages,
+        entries: page
+      });
+      pages += 1;
+      page = [];
+      bytes = 0;
+    };
+    for await (const entry of entries) {
+      const size = JSON.stringify(entry).length;
+      if (page.length >= perPage || bytes + size > DICTIONARY_PAGE_BYTES) {
+        await send();
+      }
+      page.push(entry);
+      bytes += size;
+    }
+    await send();
+    if (pages === 0) {
+      throw new Error(
+        "pushDictionary needs at least one entry. To clear what a dictionary tags, push its entries with the names you mean."
+      );
+    }
+    return this._call("dictionary/commit", {
+      slug: this._slug,
+      stageId,
+      pages
+    });
+  }
+  /**
+   * The source's current dictionary, or null when none was pushed, and the
+   * relink re-tagging its passages while one runs.
+   */
+  async dictionary() {
+    const { dictionary, relink } = await this._call("dictionary", {
+      slug: this._slug
+    });
+    return {
+      dictionary: dictionary ?? null,
+      relink: relink ? {
+        version: relink.version,
+        lap: relink.lap,
+        walked: relink.walked,
+        total: relink.total,
+        waiting: relink.waiting ?? null,
+        error: relink.error ?? null
+      } : null
+    };
+  }
+  /**
+   * What the current dictionary finds in a piece of text: each mention, in
+   * order, with the ids it tags. How to check a dictionary before relying on
+   * it. `version` is null when the source has none. Shares the search rate
+   * limit.
+   */
+  async link(text) {
+    const { version, mentions } = await this._call("link", {
+      slug: this._slug,
+      text
+    });
+    return { version: version ?? null, mentions: mentions ?? [] };
+  }
   /**
    * Every chunk of one document, exactly as it was indexed.
    *
@@ -3302,6 +3419,7 @@ var DataSource = class {
       filename: options.filename,
       ...options.contentType ? { contentType: options.contentType } : {},
       ...options.metadata !== void 0 ? { metadata: options.metadata } : {},
+      ...options.entities !== void 0 ? { entities: options.entities } : {},
       body: bytes.toString("base64")
     });
   }
@@ -3335,6 +3453,59 @@ var DataSource = class {
     });
     return { documents: documents2 ?? [], nextCursor: nextCursor ?? null };
   }
+  /**
+   * One page of the passages matching a filter, unranked: every page that
+   * mentions an asset (`{ entities: ['P-1004A'] }`), every passage holding a
+   * phrase, every chunk of some documents. Where {@link search} ranks and
+   * stops at its top results and {@link count} gives a number, this lists
+   * them all. Passages come document by document, each document's in order;
+   * pass `nextCursor` back for the next page, which is null after the last.
+   * `limit` defaults to 50, at most 200.
+   *
+   * Shares the search rate limit (300/minute per app), and shows the same
+   * `index_warming` / `index_building` as search while a corpus loads.
+   */
+  async passages(options) {
+    const { passages, nextCursor, latencyMs } = await this._call("passages", {
+      slug: this._slug,
+      ...options?.filter !== void 0 ? { filter: options.filter } : {},
+      ...options?.cursor ? { cursor: options.cursor } : {},
+      ...options?.limit !== void 0 ? { limit: options.limit } : {}
+    });
+    return {
+      passages: passages ?? [],
+      nextCursor: nextCursor ?? null,
+      latencyMs: latencyMs ?? 0
+    };
+  }
+  /**
+   * Every passage matching a filter, a page at a time behind the scenes: the
+   * walk over {@link passages}. Each page counts against the search rate
+   * limit, so a large walk belongs in a background task.
+   *
+   * ```ts
+   * for await (const passage of Manuals.allPassages({ entities: ['P-1004A'] })) {
+   *   mentions.push({ asset: 'P-1004A', ...passage.citation });
+   * }
+   * ```
+   */
+  async *allPassages(filter, options) {
+    let cursor = null;
+    while (true) {
+      const page = await this.passages({
+        ...filter !== void 0 ? { filter } : {},
+        cursor,
+        limit: options?.pageSize ?? 200
+      });
+      for (const passage of page.passages) {
+        yield passage;
+      }
+      if (!page.nextCursor || page.passages.length === 0) {
+        return;
+      }
+      cursor = page.nextCursor;
+    }
+  }
   /**
    * Every document in the corpus, oldest first, a page at a time behind the
    * scenes. This is how an app builds its own view of a large corpus (a
@@ -8252,7 +8423,11 @@ function documents(list) {
   return { kind: "documents", documents: Array.isArray(list) ? list : [list] };
 }
 function passthrough(options) {
-  return options?.metadata !== void 0 ? { kind: "passthrough", metadata: options.metadata } : { kind: "passthrough" };
+  return {
+    kind: "passthrough",
+    ...options?.metadata !== void 0 ? { metadata: options.metadata } : {},
+    ...options?.entities !== void 0 ? { entities: options.entities } : {}
+  };
 }
 function skip(reason) {
   return { kind: "skip", reason };
@@ -8296,6 +8471,7 @@ function objectFor(input) {
     etag: input.etag,
     lastModified: input.lastModified,
     metadata: input.metadata,
+    entities: input.entities ?? null,
     bytes,
     text: async () => new TextDecoder("utf-8").decode(await bytes()),
     json: async () => JSON.parse(new TextDecoder("utf-8").decode(await bytes()))
@@ -8385,6 +8561,154 @@ function defineMapper(source, config) {
   });
 }
 
+// src/reader/index.ts
+var DEFAULT_TIMEOUT_MS2 = 6e4;
+var DEFAULT_CONCURRENCY3 = 4;
+function errorInfo3(e) {
+  if (e instanceof Error) {
+    return { message: e.message, stack: e.stack };
+  }
+  return { message: String(e) };
+}
+function refOf(delivery) {
+  switch (delivery.kind) {
+    case "read":
+    case "relinked":
+      return delivery.document.id;
+    case "removed":
+      return delivery.documentId;
+    case "cleared":
+      return delivery.sourceId;
+  }
+}
+async function withTimeout2(work, timeoutMs, ref) {
+  let timer;
+  try {
+    return await Promise.race([
+      work,
+      new Promise((_, reject) => {
+        timer = setTimeout(
+          () => reject(
+            new Error(`reader timed out after ${timeoutMs}ms on ${ref}`)
+          ),
+          timeoutMs
+        );
+      })
+    ]);
+  } finally {
+    if (timer) {
+      clearTimeout(timer);
+    }
+  }
+}
+function defineReader(source, config) {
+  const documentFor = (delivery) => {
+    let cached = delivery.chunks ? Promise.resolve(delivery.chunks) : null;
+    let bytes = null;
+    const { url } = delivery.document;
+    return {
+      ...delivery.document,
+      chunks: () => {
+        if (!cached) {
+          cached = source.chunks(delivery.document.id);
+        }
+        return cached;
+      },
+      bytes: () => {
+        if (!bytes) {
+          bytes = (async () => {
+            if (!url) {
+              throw new Error(
+                `The file behind document ${delivery.document.id} is no longer kept.`
+              );
+            }
+            const response = await fetch(url);
+            if (!response.ok) {
+              throw new Error(
+                `Could not read the file behind document ${delivery.document.id}: HTTP ${response.status}`
+              );
+            }
+            return new Uint8Array(await response.arrayBuffer());
+          })();
+        }
+        return bytes;
+      }
+    };
+  };
+  const deliver2 = async (delivery) => {
+    switch (delivery.kind) {
+      case "read":
+        await config.read(documentFor(delivery));
+        return;
+      case "relinked":
+        await config.relinked?.(documentFor(delivery));
+        return;
+      case "removed":
+        await config.removed(delivery.documentId);
+        return;
+      case "cleared":
+        await config.cleared?.(delivery.sourceId);
+        return;
+    }
+  };
+  const run = async (params) => {
+    const startedAt = Date.now();
+    const timeoutMs = params.timeoutMs ?? DEFAULT_TIMEOUT_MS2;
+    const concurrency = Math.max(1, params.concurrency ?? DEFAULT_CONCURRENCY3);
+    const deliveries = Array.isArray(params.deliveries) ? params.deliveries : [];
+    const outcomes = new Array(deliveries.length);
+    let next = 0;
+    const worker = async () => {
+      while (true) {
+        const index = next++;
+        if (index >= deliveries.length) {
+          return;
+        }
+        const delivery = deliveries[index];
+        const ref = refOf(delivery);
+        const t = Date.now();
+        try {
+          await withTimeout2(
+            Promise.resolve().then(() => deliver2(delivery)),
+            timeoutMs,
+            ref
+          );
+          outcomes[index] = {
+            ref,
+            kind: delivery.kind,
+            ok: true,
+            durationMs: Date.now() - t
+          };
+        } catch (e) {
+          outcomes[index] = {
+            ref,
+            kind: delivery.kind,
+            ok: false,
+            error: errorInfo3(e),
+            durationMs: Date.now() - t
+          };
+        }
+      }
+    };
+    await Promise.all(
+      Array.from({ length: Math.min(concurrency, deliveries.length) }, worker)
+    );
+    return {
+      v: 1,
+      source: source.name,
+      outcomes,
+      hooks: { relinked: typeof config.relinked === "function" },
+      startedAt,
+      durationMs: Date.now() - startedAt
+    };
+  };
+  return Object.assign(run, {
+    kind: "reader",
+    source: source.name,
+    config
+  });
+}
+
 // src/generated/snippets.ts
 var monacoSnippets = {
   "activeCampaignAddNote": { fields: [["contactId", "string"], ["note", "string"]], outputKeys: [] },
@@ -8645,6 +8969,7 @@ export {
   index_default as default,
   defineJewel,
   defineMapper,
+  defineReader,
   deletes,
   documents,
   email,