@mindstudio-ai/agent 0.1.117 → 0.1.119

package/dist/index.js

@@ -2840,6 +2840,7 @@ function createFiles(call) {
 
 // src/datasources/source.ts
 import { createHash as createHash2 } from "crypto";
+var REMOVE_WHERE_MAX_PAGES = 100;
 var DataSource = class {
   constructor(_slug, _call) {
     this._slug = _slug;
@@ -2864,6 +2865,15 @@ var DataSource = class {
    * Alongside `results` and `latencyMs` comes {@link SearchRan} — what the
    * search actually did, which is the first thing to check when results don't
    * look like the options you passed.
+   *
+   * **Throws `index_warming`** (HTTP 503) when a large corpus's index is being
+   * reloaded after an eviction: the platform keeps a working set of indexes
+   * resident and reloads the rest on first use, which for a big source takes a
+   * minute or two in the background. Treat it as "loading", never as "empty" —
+   * tell the user the knowledge base is warming up and retry shortly. Small
+   * corpora reload inside the search and never raise it. Sources on dedicated
+   * capacity are never evicted, so they never raise it either; they raise
+   * `capacity_hibernated` and friends instead when their capacity is parked.
    */
   async search(query, options) {
     const { results, mode, latencyMs } = await this._call("search", {
@@ -2878,10 +2888,29 @@ var DataSource = class {
       ...options?.rerank !== void 0 ? { rerank: options.rerank } : {},
       ...options?.hybrid !== void 0 ? { hybrid: options.hybrid } : {},
       ...options?.explain !== void 0 ? { explain: options.explain } : {},
-      ...options?.expand !== void 0 ? { expand: options.expand } : {}
+      ...options?.expand !== void 0 ? { expand: options.expand } : {},
+      ...options?.shareCitations !== void 0 ? { shareCitations: options.shareCitations } : {}
     });
     return { results: results ?? [], mode, latencyMs: latencyMs ?? 0 };
   }
+  /**
+   * Mint an ABSOLUTE, signed link to one document that works with **no**
+   * session — the same verb as `store.shareUrl()` for a file. Takes a
+   * document id or anything carrying one (`hit.citation`). Expires (default
+   * 24h; clamped to 60s–90d); each click still redirects to a fresh
+   * short-lived storage URL, so the expiry you pass is the only one that
+   * matters. For every citation of a search at once, pass
+   * {@link SearchOptions.shareCitations} instead.
+   */
+  async shareUrl(document, options) {
+    const documentId = typeof document === "string" ? document : document.documentId;
+    const res = await this._call("share", {
+      slug: this._slug,
+      documentId,
+      ...options?.expiresIn ? { expiresIn: options.expiresIn } : {}
+    });
+    return res.url;
+  }
   /**
    * What is in the corpus, and how it was built.
    *
@@ -2923,6 +2952,19 @@ var DataSource = class {
    * bytes under its current configuration. Reconfiguring the source is what
    * makes a re-add do work again — and that is an explicit, owner-triggered
    * migration, never something a deploy causes.
+   *
+   * **On a source with a mapper** (`dataSources[].mapper` in mindstudio.json,
+   * see `defineMapper`) the bytes are a raw object and the mapper decides
+   * what they become: `documents` holds everything it produced (one object
+   * can become many documents), `document` is the first of them for
+   * convenience, and `outcome` says which way the mapper went. A
+   * `passthrough` is one document from the bytes themselves; a `deletes`
+   * outcome removes the named documents and returns none. A document whose
+   * `externalId` (or `replaces`) names an existing one supersedes it as soon
+   * as the new one is registered, before its build lands; a search in that
+   * window sees neither. **Throws `mapper_skipped`** (HTTP 422) when the
+   * mapper skipped the object — a specific thing it refused, which the caller
+   * should see — and `mapper_error` when the mapper threw.
    */
   async add(content, options) {
     const bytes = typeof content === "string" ? Buffer.from(content) : Buffer.from(content);
@@ -2934,15 +2976,104 @@ var DataSource = class {
       body: bytes.toString("base64")
     });
   }
-  /** Every document in the corpus, with ingest status. */
-  async documents() {
-    const { documents } = await this._call("documents", { slug: this._slug });
-    return documents ?? [];
+  /**
+   * Documents in the corpus, with ingest status.
+   *
+   * Without options this is the first thousand, newest first: enough to show
+   * a small corpus, not a way to walk a large one (a corpus of any size is
+   * loaded and inspected from the CLI). To follow what {@link add} just
+   * queued, pass the ids it returned: `documents({ ids })` answers only those,
+   * at most two hundred per call.
+   */
+  async documents(options) {
+    const { documents: documents2 } = await this._call("documents", {
+      slug: this._slug,
+      ...options?.ids?.length ? { ids: options.ids } : {}
+    });
+    return documents2 ?? [];
   }
   /** Remove a document and its vectors. */
   async remove(documentId) {
     await this._call("remove", { slug: this._slug, documentId });
   }
+  /**
+   * Remove every document matching a selector — by metadata tags, filename,
+   * ids, or the external id prefix a bulk load or connector recorded.
+   *
+   * Works in pages of a thousand and keeps going until nothing matches (or a
+   * very large removal has run for a hundred pages, in which case `remaining`
+   * says how many are left and a second call continues). Vectors, the catalog
+   * row and the stored bytes all go; the removal is immediate and shared
+   * across dev and prod like every data-source write.
+   *
+   * ```ts
+   * await Policies.removeWhere({ metadata: { department: 'legal', year: 2019 } });
+   * await Policies.removeWhere({ externalIdPrefix: 'archive/2019/' });
+   * ```
+   */
+  async removeWhere(selector) {
+    let deleted = 0;
+    let remaining = 0;
+    for (let page = 0; page < REMOVE_WHERE_MAX_PAGES; page++) {
+      const result = await this._call("remove-many", {
+        slug: this._slug,
+        filter: selector
+      });
+      deleted += result?.deleted ?? 0;
+      remaining = result?.remaining ?? 0;
+      if (remaining === 0 || !(result?.deleted > 0)) {
+        break;
+      }
+    }
+    return { deleted, remaining };
+  }
+  /**
+   * Bring the source up to date with the S3 bucket it is connected to.
+   *
+   * Lists the bucket, diffs it against what was ingested before (by ETag), and
+   * runs an ingestion job over what is new or changed; keys that disappeared
+   * upstream are removed at the end when the connector's deletion policy says
+   * so. Returns as soon as the job is started — it plans, then runs in the
+   * background. Poll {@link job} for progress, or read the latest sync from
+   * {@link jobs}.
+   *
+   * The job auto-approves under the connector's per-sync budget. A plan over
+   * that budget, or over the capacity of where the source lives, waits in
+   * `planned` for a human (`remy-admin datasources jobs approve <id>`) — so the
+   * first backfill of a large bucket stops for approval by itself, and the
+   * steady-state deltas run unattended. Nothing is spent when nothing changed.
+   *
+   * The connection itself — which bucket, which app secrets hold the keys — is
+   * created by the app's owner with `remy-admin datasources connect`, never
+   * from code. A nightly sync is an ordinary cron interface job whose method
+   * calls this:
+   *
+   * ```ts
+   * // methods/sync-archive.ts, scheduled "0 3 * * *" in the cron interface
+   * export default async function () {
+   *   await Archive.sync();
+   * }
+   * ```
+   *
+   * @throws `connector_not_found` when the source is not connected;
+   *   `data_source_busy` while another job or a move is in flight.
+   */
+  async sync(options) {
+    return this._call("sync", {
+      slug: this._slug,
+      ...options?.budgetDollars !== void 0 ? { budgetDollars: options.budgetDollars } : {}
+    });
+  }
+  /** Recent ingestion jobs on this source (syncs and CLI loads), newest first. */
+  async jobs() {
+    const { jobs } = await this._call("jobs", { slug: this._slug });
+    return jobs ?? [];
+  }
+  /** One job by id — progress, plan, failures, and why it paused if it did. */
+  async job(id) {
+    const { job } = await this._call("job", { slug: this._slug, id });
+    return job;
+  }
   /**
    * Create the data source if it doesn't exist yet.
    *
@@ -7757,6 +7888,136 @@ function defineJewel(method, config) {
   });
 }
 
+// src/mapper/index.ts
+function documents(list) {
+  return { kind: "documents", documents: Array.isArray(list) ? list : [list] };
+}
+function passthrough(options) {
+  return options?.metadata !== void 0 ? { kind: "passthrough", metadata: options.metadata } : { kind: "passthrough" };
+}
+function skip(reason) {
+  return { kind: "skip", reason };
+}
+function deletes(externalIds) {
+  return {
+    kind: "deletes",
+    externalIds: Array.isArray(externalIds) ? externalIds : [externalIds]
+  };
+}
+var DEFAULT_TIMEOUT_MS = 3e4;
+var DEFAULT_CONCURRENCY = 4;
+function errorInfo2(e) {
+  if (e instanceof Error) return { message: e.message, stack: e.stack };
+  return { message: String(e) };
+}
+function objectFor(input) {
+  let cached = null;
+  const bytes = () => {
+    if (!cached) {
+      cached = (async () => {
+        const response = await fetch(input.url);
+        if (!response.ok) {
+          throw new Error(
+            `Could not read ${input.key}: HTTP ${response.status}`
+          );
+        }
+        return new Uint8Array(await response.arrayBuffer());
+      })();
+    }
+    return cached;
+  };
+  return {
+    key: input.key,
+    size: input.size,
+    contentType: input.contentType,
+    etag: input.etag,
+    lastModified: input.lastModified,
+    metadata: input.metadata,
+    bytes,
+    text: async () => new TextDecoder("utf-8").decode(await bytes()),
+    json: async () => JSON.parse(new TextDecoder("utf-8").decode(await bytes()))
+  };
+}
+var OUTCOME_KINDS = /* @__PURE__ */ new Set(["documents", "passthrough", "skip", "deletes"]);
+function normalizeOutcome(value) {
+  const outcome = value;
+  if (!outcome || typeof outcome !== "object" || !OUTCOME_KINDS.has(outcome.kind)) {
+    return {
+      kind: "error",
+      error: {
+        message: `map returned an invalid outcome: ${JSON.stringify(value)?.slice(0, 200)}. Return documents(), passthrough(), skip() or deletes().`
+      }
+    };
+  }
+  return outcome;
+}
+async function withTimeout(work, timeoutMs, key) {
+  let timer;
+  try {
+    return await Promise.race([
+      work,
+      new Promise((_, reject) => {
+        timer = setTimeout(
+          () => reject(new Error(`map timed out after ${timeoutMs}ms on ${key}`)),
+          timeoutMs
+        );
+      })
+    ]);
+  } finally {
+    if (timer) clearTimeout(timer);
+  }
+}
+function defineMapper(source, config) {
+  const run = async (params) => {
+    const startedAt = Date.now();
+    const timeoutMs = params.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+    const concurrency = Math.max(1, params.concurrency ?? DEFAULT_CONCURRENCY);
+    const objects = Array.isArray(params.objects) ? params.objects : [];
+    const outcomes = new Array(objects.length);
+    let next = 0;
+    const worker = async () => {
+      while (true) {
+        const index = next++;
+        if (index >= objects.length) return;
+        const input = objects[index];
+        const t = Date.now();
+        let outcome;
+        try {
+          outcome = normalizeOutcome(
+            await withTimeout(
+              Promise.resolve().then(() => config.map(objectFor(input))),
+              timeoutMs,
+              input.key
+            )
+          );
+        } catch (e) {
+          outcome = { kind: "error", error: errorInfo2(e) };
+        }
+        outcomes[index] = {
+          key: input.key,
+          outcome,
+          durationMs: Date.now() - t
+        };
+      }
+    };
+    await Promise.all(
+      Array.from({ length: Math.min(concurrency, objects.length) }, worker)
+    );
+    return {
+      v: 1,
+      source: source.name,
+      outcomes,
+      startedAt,
+      durationMs: Date.now() - startedAt
+    };
+  };
+  return Object.assign(run, {
+    kind: "mapper",
+    source: source.name,
+    config
+  });
+}
+
 // src/generated/snippets.ts
 var monacoSnippets = {
   "activeCampaignAddNote": { fields: [["contactId", "string"], ["note", "string"]], outputKeys: [] },
@@ -8031,17 +8292,22 @@ export {
   db,
   index_default as default,
   defineJewel,
+  defineMapper,
+  deletes,
+  documents,
   email,
   events,
   files,
   getRequestContext,
   mindstudio,
   monacoSnippets,
+  passthrough,
   prerender,
   reportIssue,
   resolveUser,
   runWithContext,
   session,
+  skip,
   stepMetadata,
   stream,
   voice,