@mindstudio-ai/remy 0.1.320 → 0.1.321

package/dist/headless.js

@@ -1603,7 +1603,7 @@ var promptUserTool = {
               type: {
                 type: "string",
                 enum: ["select", "checklist", "text", "file"],
-                description: 'select: pick one from a list. checklist: pick one or more from a list. The user can always provide a custom "Other" answer for select and checklist questions, so there is no need to include an "Other" option. text: free-form input. file: file/image upload. The answer is the local path (string) under src/.user-uploads/ \u2014 the file is already downloaded to disk. Documents may have an extracted-text sidecar at <path>.txt. Reference the path directly; pass image paths straight to analyzeImage / screenshot tools.'
+                description: 'select: pick one from a list. checklist: pick one or more from a list. The user can always provide a custom "Other" answer for select and checklist questions, so there is no need to include an "Other" option. text: free-form input. file: the user hands over files, a folder, or a paste (set `multiple: true` to take several). Each answer arrives in one of two shapes. A local path (string) under src/.user-uploads/: the file is already on disk, documents may have an extracted-text sidecar at <path>.txt, and a folder arrives as src/.user-uploads/<folder>.zip for you to unpack with bash. Or, when the user brought a dataset (a folder over about 100 MB or 500 files, or any single file over 100 MB), a store landing object { kind: "store", store, prefix, files, bytes, failed } \u2014 the files were streamed into the app\'s private file store `uploads` under that prefix and are NOT on disk; read them in place with `remy-admin datasources inspect --store <store> --prefix <prefix>` and `remy-admin files get`, never download the whole set. Pass image paths straight to analyzeImage / screenshot tools.'
               },
               helpText: {
                 type: "string",
@@ -1638,7 +1638,7 @@ var promptUserTool = {
               },
               multiple: {
                 type: "boolean",
-                description: "For file type: allow multiple uploads (the answer is an array of local paths). Defaults to false."
+                description: 'For file type: allow several files or folders (the answer is an array of local paths and/or store landings). Defaults to false. Set it whenever you ask for "a few examples" or "a sample of your data".'
               },
               format: {
                 type: "string",
@@ -2629,7 +2629,7 @@ async function readAndSort(dirPath) {
 async function collapsePath(basePath, name) {
   let display = name;
   let current = path8.join(basePath, name);
-  for (; ; ) {
+  while (true) {
     let children;
     try {
       children = await readAndSort(current);
@@ -10852,12 +10852,15 @@ var HeadlessSession = class {
     const questions = Array.isArray(input?.questions) ? input.questions : [];
     const batch = [];
     const slots = [];
+    let landings = 0;
     for (const q of questions) {
       if (q?.type !== "file" || typeof q.id !== "string") {
         continue;
       }
       const val = answers[q.id];
-      const items = (Array.isArray(val) ? val : [val]).filter(isDescriptor);
+      const all = Array.isArray(val) ? val : [val];
+      landings += all.filter((v) => v && v.kind === "store").length;
+      const items = all.filter(isDescriptor);
       if (items.length === 0) {
         continue;
       }
@@ -10876,6 +10879,9 @@ var HeadlessSession = class {
       }
     }
     if (batch.length === 0) {
+      if (landings > 0) {
+        log17.info("promptUser store landings passed through", { landings });
+      }
       return raw;
     }
     let results;
@@ -11219,7 +11225,7 @@ var HeadlessSession = class {
    * there would strand the chain steps and background results behind it.
    */
   async drainQueueLoop() {
-    for (; ; ) {
+    while (true) {
       const at = this.queue.firstDeliverableIndex();
       if (at === -1) {
         return;

package/dist/index.js

@@ -1067,7 +1067,7 @@ var init_promptUser = __esm({
                   type: {
                     type: "string",
                     enum: ["select", "checklist", "text", "file"],
-                    description: 'select: pick one from a list. checklist: pick one or more from a list. The user can always provide a custom "Other" answer for select and checklist questions, so there is no need to include an "Other" option. text: free-form input. file: file/image upload. The answer is the local path (string) under src/.user-uploads/ \u2014 the file is already downloaded to disk. Documents may have an extracted-text sidecar at <path>.txt. Reference the path directly; pass image paths straight to analyzeImage / screenshot tools.'
+                    description: 'select: pick one from a list. checklist: pick one or more from a list. The user can always provide a custom "Other" answer for select and checklist questions, so there is no need to include an "Other" option. text: free-form input. file: the user hands over files, a folder, or a paste (set `multiple: true` to take several). Each answer arrives in one of two shapes. A local path (string) under src/.user-uploads/: the file is already on disk, documents may have an extracted-text sidecar at <path>.txt, and a folder arrives as src/.user-uploads/<folder>.zip for you to unpack with bash. Or, when the user brought a dataset (a folder over about 100 MB or 500 files, or any single file over 100 MB), a store landing object { kind: "store", store, prefix, files, bytes, failed } \u2014 the files were streamed into the app\'s private file store `uploads` under that prefix and are NOT on disk; read them in place with `remy-admin datasources inspect --store <store> --prefix <prefix>` and `remy-admin files get`, never download the whole set. Pass image paths straight to analyzeImage / screenshot tools.'
                   },
                   helpText: {
                     type: "string",
@@ -1102,7 +1102,7 @@ var init_promptUser = __esm({
                   },
                   multiple: {
                     type: "boolean",
-                    description: "For file type: allow multiple uploads (the answer is an array of local paths). Defaults to false."
+                    description: 'For file type: allow several files or folders (the answer is an array of local paths and/or store landings). Defaults to false. Set it whenever you ask for "a few examples" or "a sample of your data".'
                   },
                   format: {
                     type: "string",
@@ -3826,7 +3826,7 @@ async function readAndSort(dirPath) {
 async function collapsePath(basePath, name) {
   let display = name;
   let current = path8.join(basePath, name);
-  for (; ; ) {
+  while (true) {
     let children;
     try {
       children = await readAndSort(current);
@@ -11896,12 +11896,15 @@ var init_headless = __esm({
         const questions = Array.isArray(input?.questions) ? input.questions : [];
         const batch = [];
         const slots = [];
+        let landings = 0;
         for (const q of questions) {
           if (q?.type !== "file" || typeof q.id !== "string") {
             continue;
           }
           const val = answers[q.id];
-          const items = (Array.isArray(val) ? val : [val]).filter(isDescriptor);
+          const all = Array.isArray(val) ? val : [val];
+          landings += all.filter((v) => v && v.kind === "store").length;
+          const items = all.filter(isDescriptor);
           if (items.length === 0) {
             continue;
           }
@@ -11920,6 +11923,9 @@ var init_headless = __esm({
           }
         }
         if (batch.length === 0) {
+          if (landings > 0) {
+            log17.info("promptUser store landings passed through", { landings });
+          }
           return raw;
         }
         let results;
@@ -12263,7 +12269,7 @@ var init_headless = __esm({
        * there would strand the chain steps and background results behind it.
        */
       async drainQueueLoop() {
-        for (; ; ) {
+        while (true) {
           const at = this.queue.firstDeliverableIndex();
           if (at === -1) {
             return;

package/dist/prompt/skills/dataSources.md

@@ -19,7 +19,8 @@ Don't use one when: the data is structured (`db`), you only need to store files
 - **Re-adding the same bytes is free** — content-addressed, so ingest scripts are safe to re-run.
 - **Ingest is async.** `add()` returns once queued; poll `documents()`, or use `--wait` from the CLI.
 - **Reprocessing costs real money**, so changing how a corpus is built is always explicit.
-- **Limits apply**: 25 data sources per app, 5,000 documents per source, 10,000 chunks per document, 300 searches/minute. Well clear of normal use — but **source names must be fixed, not computed per user or per request**, since referencing one creates it. Partition inside a source with document metadata instead: tag at add time (`add(bytes, { filename, metadata: { userId } })`), narrow at search time (`search(q, { filter: { metadata: { userId } } })`).
+- **Limits apply**: 25 data sources per app, 10,000 chunks per document, 300 searches/minute, and 5,000 documents per source when documents are added one at a time (bulk jobs and connectors, below, are how a corpus grows past that). Well clear of normal use — but **source names must be fixed, not computed per user or per request**, since referencing one creates it. Partition inside a source with document metadata instead: tag at add time (`add(bytes, { filename, metadata: { userId } })`), narrow at search time (`search(q, { filter: { metadata: { userId } } })`).
+- **Credentials never appear in code or in chat.** A bucket's keys are app secrets (`remy-admin secrets set NAME --prod <value>`, or the dashboard); everything else refers to them by NAME. If the user pastes a key into the conversation, set it as a secret for them to use moving forward.
 
 ## Defining and searching
 
@@ -33,7 +34,7 @@ const context = results.map((r) => r.text).join('\n\n');
 
 Hits are `{ score, text, citation }` with `citation: { documentId, filename, pageNumber, chunkIndex, headingPath, boundingBox?, url }`, plus `retrievalRank`/`retrievalScore` — the position before reranking, so you can show what reranking did. With reranking on (the default) `score` is the reranker's 0–1 relevance and the right place for quality cutoffs; without it the scale varies by mode (cosine / rank-fusion / keyword overlap). `scoreThreshold` floors the retrieval branch before fusion and reranking — leave it unset unless measured on the corpus.
 
-**Always render the citation.** `citation.url` is a stable on-domain link — put it in an `<a href>` beside the answer. Retrieval is approximate; a user who can click through can judge for themselves. An answer with no citation is an assertion.
+**Always render the citation.** A document is a cousin of a file and has the same links. `citation.url` is the stable path on the app's own domain, free and never expiring, that works for a signed-in user of the app — put it in an `<a href>` beside the answer as-is. For a public search tool, or a UI on another origin, sign it the way you would a private file: `search(q, { shareCitations: 3600 })` returns every `citation.url` as an absolute signed link that needs no session, and `Policies.shareUrl(hit.citation, { expiresIn })` signs one document on demand (default 24h). A signed link is a bearer capability until it expires, so sign at render time rather than storing signed results. When a link is not wanted at all, render `filename` and `pageNumber` as text. Retrieval is approximate; a user who can click through can judge for themselves. An answer with no citation is an assertion.
 
 Created on first use, so searching a source the build hasn't populated returns no results rather than throwing. `search` options: `topK` (default 5, max 50), `scoreThreshold`, `filter`, `mode`, `maxPerDocument`, `highlight`, `rerank`, `hybrid`.
 
@@ -45,8 +46,40 @@ Search is deterministic for a fixed corpus and configuration, so eval sets and r
 
 **Debugging retrieval.** Two opt-in options, neither of which changes the results or their order: `explain: true` adds `explain.{dense, lexical, matchedVia}` (which half of hybrid found each hit; costs two extra round trips), and `expand: 1` adds `neighbors.{before, after}` for surrounding context. When a document never comes back at all, `Policies.stats()` reports the config actually in effect and `Policies.chunks(documentId)` shows exactly how it was split.
 
+**A cold index (shared capacity).** By default a data source lives on shared retrieval capacity: its vectors sit in their own isolated partition of a pool many apps share, and the pool keeps only a working set resident. A source nobody has searched for a while is unloaded to make room and reloaded from durable storage on the next search. A small corpus reloads inside that search and nobody notices; a large one (hundreds of thousands of chunks) reloads in the background for a minute or two, and `search()` throws `index_warming` (HTTP 503) until it lands. That means *loading*, never *empty*: catch it, tell the user the knowledge base is warming up, and retry shortly. Before a demo, `remy-admin datasources hydrate --source <slug> --wait` reloads it ahead of time. The way out of the cycle is dedicated capacity (below): a source on its own provisioned retrieval is never unloaded and never warms.
+
 **Configuration is not declared in code** — chunking and embedding settings live on the corpus and are set with the CLI, so code and reality can't drift.
 
+## Working with someone's data
+
+People are bad at describing their own data, and the more of it they have the worse the description gets. "We have SO much data and a really complex system" is often a few thousand pages of PDFs; "just some documents" is occasionally a bucket of ten million records. Treat the description as a mood, not a measurement. This section is how you find out what is really there and set expectations to match, before any tooling comes up.
+
+**1. Get real data in hand first.** Before proposing anything, ask for a sample of the actual thing: a handful of representative files, a link to where they live, or wherever the data is. Ask in the user's terms — "can you give me a few examples of what we're working with?" — as a `promptUser` form with a `file` question (`multiple: true`) and a text field for a link or a location. Let them answer however they can: if the data lives in an S3 bucket, for example, they will say so, and the bucket becomes the sample once its keys are set as app secrets (`datasources inspect --connector`). Do not ask them to size or classify it; that is your job once you can see it. What arrives depends on how much they brought: files land under `src/.user-uploads/` (documents with an extracted-text sidecar at `<path>.txt`; a folder as `<folder>.zip`, unpack it), and a folder too large to hand over that way is streamed into the app's private `uploads` file store instead, and the answer is a landing `{ store, prefix, files, bytes }` you read in place.
+
+**2. Look, then say what it is.** Poke around the sample yourself: the files on disk, or for a landing `datasources inspect --store uploads --prefix <folder>/` plus `remy-admin files get` on a few keys (a landing's file count and bytes already tell you how big the corpus, or its known fraction, is). From it you can tell whether the file is the document (PDFs, Word, slides, pages, plain text: built-in extraction handles it) or a container of records (JSON, JSONL, CSV exports, a database dump: the file is not the document and the source needs a mapper); how big each one is; and whether there is one shape or several. The one thing a sample cannot tell you is how much more there is, so ask exactly that, in whatever terms the user has (a folder, a year of exports, a bucket). Then say what you found back in plain words.
+
+**3. Pick the rung from the count, and say it before any tooling.** The count decides the path, never the adjectives.
+- Up to a few hundred files: `datasources add`, a few dollars, the shared pool. Nothing else in this skill needs to come up; jobs, connectors and capacity stay out of the conversation.
+- Thousands to a few hundred thousand documents: a job with a plan before anything is spent, still the shared pool (up to about a million chunks), a mapper only if the file is not the document. Tens to a few hundred dollars.
+- Millions of records, or tens of gigabytes: the full path described below. Embedding runs over time based on rate limits, storage and compute are real line items, and the plan will likely answer `plan_requires_dedicated`, which is the moment dedicated capacity is discussed, with its price, and not before.
+
+**4. The two ways this goes wrong.** 
+- Over-building: a user who sounds big doesn't need connectors and dedicated retrieval for a folder of PDFs; the rung, said out loud before any tooling, is the guard. 
+- Under-explaining: when the data really is big, the user has usually been handed keys and told "go do RAG" and has never been told what that entails. Walk them through it in plain numbers: it costs money in three places (embedding, storage, and possibly dedicated compute); it takes time because embedding is rate-limited (a truly massive corpus might take days, even! but the system is designed for exactly this. 99% of users will never need this, though); there are decisions only they can make and you will stop for each; the first search on a cold index is slow; and here is roughly what the bill looks like before anyone spends. Self-described "RAG experts" get the same walkthrough, phrased as what this platform will do rather than what RAG is: respect the expertise, verify the specifics anyway.
+
+**5. The order of operations for a real load.** Once the sample says this is a load rather than an `add`, this is the shape; each step is detailed in the sections below.
+
+1. **Look before you promise.** `datasources inspect` over the store or the bucket: key shapes, counts and bytes by extension, size buckets, sampled heads with their JSON keys. Report it in the user's vocabulary: how many objects, how many formats, what is not documents, which parts nobody mentioned.
+2. **Ask the questions only the user can answer**, as a form, and stop for them: what is in scope and what is not, what one document is (a file, a record, the latest version of a record), what counts as a duplicate, what to do with the low-value kinds, who holds the rights to anything third-party. Do not guess these; they decide what the mapper does.
+3. **Shape it, if the file is not the document.** Write the mapper and `map test --dev` it over twenty real objects; read the outcomes back to the user as documents, metadata and skips with reasons. Adjust and re-run; deploy only when the outcomes read right.
+4. **Sample before the whole thing.** `jobs start --limit 500` loads a slice cheaply; search it; fix the mapper; `remap`. A mistake on five hundred documents costs cents.
+5. **Show the plan and get an explicit yes.** The plan is the bill: documents, chunks, cost per stage, storage, duration, and whether the corpus fits where the source lives. Present it, then `jobs approve`. If it answers `plan_requires_dedicated`, the size decision comes first: show the offering's price, get a yes, `infra provision`, `datasources move`, then approve.
+6. **Run it, and read what came out.** `jobs status` for progress and the last failures; `jobs quarantine` for what the mapper skipped or failed on, by reason; fix, deploy, `jobs replay`. Search works on the partial corpus throughout.
+7. **Keep it current.** For a bucket, a cron method calling `Source.sync()`; for anything else, the same job re-run (unchanged objects cost nothing).
+8. **Measure before you change anything.** A sample source and an eval set; compare versions and models with numbers, and ask before promoting.
+
+Three rules hold throughout: credentials are app secrets referred to by NAME and never appear in chat or code; nothing that spends is approved or provisioned without the user's explicit yes on the numbers; and dedicated capacity is proposed when a plan asks for it, not before.
+
 ## Loading documents — normally at build time, from the CLI
 
 ```bash
@@ -57,7 +90,7 @@ remy-admin datasources search --source policies --filter department=legal --mode
 remy-admin datasources delete --source policies   # whole source; --source is required, never defaulted
 ```
 
-`--wait` blocks until processing finishes and exits non-zero on failure. Also `datasources list`, `status` (per-document state + ingest errors), `rm --document <id>`. `--help` for flags.
+`--wait` blocks until processing finishes and exits non-zero on failure. Also `datasources list`, `status` (per-document state + ingest errors), `rm --document <id>` or `rm --filter <k=v,...>`. `--help` for flags. For more than a few dozen files, use a job (below) rather than `add`.
 
 **Seeding a test corpus:** scenarios don't touch data sources, so load fixtures with the same command in a setup script — `datasources add --source <slug> --wait fixtures/*.pdf`. Re-running is free, so it needs no guard.
 
@@ -69,11 +102,108 @@ await Policies.add(buffer, {
   contentType: 'application/pdf',
   metadata: { department: 'legal' },   // filterable at search time
 });
-const docs = await Policies.documents();   // 'processing' | 'done' | 'error'
+const docs = await Policies.documents({ ids: [document!.id] });   // 'processing' | 'done' | 'error'; plain documents() is the first thousand
 await Policies.remove(documentId);
 ```
 
-Formats: pdf, docx, pptx, xlsx, odt, rtf, epub, images, txt, md, json, csv, tsv, log, html.
+Formats: pdf, docx, pptx, xlsx, odt, rtf, epub, images, txt, md, json, csv, tsv, log, html. When the file is not the document (a JSON record, a JSONL bundle of articles, a kill notice), the source needs a mapper — see below.
+
+Removing many documents at once: `Policies.removeWhere({ metadata: { year: 2019 } })` or `{ externalIdPrefix: 'archive/2019/' }` (the key a job or connector recorded) removes every match, vectors and bytes included, in pages of a thousand. From the CLI, `datasources rm --source policies --filter year=2019`. An empty filter is refused; deleting a whole source is `datasources delete`, never something app code does.
+
+## Loading a corpus of any size (jobs)
+
+`datasources add` is for a handful of files. A corpus of thousands to millions of documents is loaded by a **job**, which reads either every object under a prefix of one of the app's file stores or a JSONL manifest of URLs, and shows a **plan before anything is spent**: documents, chunks, cost per stage at today's rates, storage, whether it fits where the source lives, and a duration.
+
+```bash
+remy-admin datasources jobs start --source archive --store raw --prefix 2024/ --wait     # plan, then stop
+remy-admin datasources jobs status <id>                                                # read the plan
+remy-admin datasources jobs approve <id> --wait                                        # the yes to the spend
+remy-admin datasources jobs start --source archive --manifest urls.jsonl --limit 200 --approve --wait   # a cheap sample first
+```
+
+Two gates decide whether a plan can run: the corpus has to fit the source's placement (a shared-pool source over the per-source cap answers `plan_requires_dedicated`; see Dedicated capacity below), and the workspace has to be able to cover the projection (`insufficient_credits`). **Show the user the plan and get an explicit yes before approving** — the plan is the whole point. `--budget <dollars>` pauses the job at a ceiling; `--limit <n>` loads a sample of the corpus to check quality before committing to all of it. Unchanged documents are skipped by content hash, so re-running a job is free. `jobs pause|resume|cancel` are the controls; search works on the partial corpus throughout. One bulk operation per source at a time (`data_source_busy`).
+
+## Keeping a corpus in sync with an S3 bucket (connectors)
+
+When the documents live in a bucket the user owns, connect it once and sync from then on. The keys are the NAMES of two app secrets, set first:
+
+```bash
+remy-admin secrets set ARCHIVE_S3_KEY --prod <value>        # the user sets these, or does it in the dashboard
+remy-admin secrets set ARCHIVE_S3_SECRET --prod <value>
+remy-admin datasources connect --source archive --bucket acme-docs --region us-east-1 --prefix contracts/ --access-key-secret ARCHIVE_S3_KEY --secret-key-secret ARCHIVE_S3_SECRET --budget-per-sync 5
+remy-admin datasources sync --source archive --wait     # first sync: plans the whole bucket, stops for approval if over the budget
+remy-admin datasources connector --source archive       # what it follows, last sync, object counts
+```
+
+`connect` checks that both secrets exist and that the keys can list the prefix before recording anything. `sync` lists the bucket, compares every object's ETag with what was ingested before, and runs a job over what is new or changed — auto-approved under the connector's per-sync budget, so **the first backfill of a big bucket stops for `jobs approve` by itself and the nightly deltas run unattended**. A changed object replaces its document; an object that disappears from the bucket takes its document with it at the end of the next full sync (`--deletions mirror`, the default; `keep` leaves them). A sync over an unchanged bucket costs nothing. Objects the extractors cannot read are skipped, not failed.
+
+**Scheduling is the app's.** The nightly sync is an ordinary cron interface job whose method calls the SDK:
+
+```typescript
+// methods/sync-archive.ts — scheduled "0 3 * * *" in the cron interface (see the Scheduled Jobs skill)
+export default async function () {
+  const { job } = await Archive.sync();   // returns at once; the job plans and runs in the background
+}
+```
+
+`Archive.jobs()` lists recent syncs and `Archive.job(id)` reads one — progress, plan, the last twenty per-document failures, and why it paused if it did. The connection itself (bucket, prefix, which secrets) is only ever made from the CLI or the dashboard; code can run a sync, not repoint one. `datasources disconnect --source archive` stops following the bucket and keeps every document.
+
+## Mapping raw objects into documents (mappers)
+
+A source takes every object it is given as one document through built-in extraction. When **the file is not the document** — JSON records that should become markdown plus metadata, a JSONL object that holds a thousand articles, a kill notice that means "remove this story", a bucket where only some keys matter — give the source a **mapper**: your code, one object in, documents out. This is how a structured archive becomes a corpus, and you write it.
+
+**Inspect before you write.** `remy-admin datasources inspect --source archive --connector` (or `--store raw --prefix 2024/`) profiles the objects without reading them all: counts and bytes by extension and by key shape (`2024/#/{uuid}.json`), size buckets, and ten sampled heads with their top-level JSON keys. Report what you found and ask the questions only the user can answer (which prefixes, which schema generations, what counts as a duplicate) before writing a line.
+
+```typescript
+// datasources/archive.mapper.ts  — beside archive.ts, which exports Archive
+import { defineMapper, documents, passthrough, skip, deletes } from '@mindstudio-ai/agent';
+import { Archive } from './archive';
+
+export default defineMapper(Archive, {
+  map: async (object) => {
+    if (!object.key.endsWith('.json')) return passthrough();           // PDFs etc. through built-in extraction
+    const { item } = await object.json();
+    if (item.type !== 'text') return skip(`not an article: ${item.type}`);
+    if (item.pubstatus === 'canceled') return deletes([item.uri]);     // a kill notice
+    return documents([{
+      externalId: item.uri,                                            // the identity the platform replaces by
+      title: item.headline ?? item.slugline,
+      markdown: toMarkdown(item),
+      metadata: { date: Number(item.versioncreated.slice(0, 10).replaceAll('-', '')), language: item.language },
+      replaces: item.altids?.original_id,                              // a writethru supersedes its original
+    }]);
+  },
+});
+```
+
+Declared in `mindstudio.json` — the compiler lifts it like a jewel and the platform runs it as an ordinary execution frame:
+
+```json
+"dataSources": [{ "slug": "archive", "mapper": { "path": "dist/datasources/archive.mapper.ts", "timeoutMs": 30000 } }]
+```
+
+`object` is `{ key, size, contentType, etag, lastModified, metadata }` plus lazy `bytes()`, `text()`, `json()`. Four outcomes: `documents([...])` (an array is one object → many documents; each `{ externalId, title, markdown, metadata?, replaces? }`), `passthrough({ metadata? })` (ingest the raw object as-is), `skip(reason)`, `deletes([externalId, ...])`. Throwing is the object's error. A mapper may call models (`runTask`) or `fetch` an API — it is your code — but every call is spend per object, so keep the common path cheap. `timeoutMs` is the per-object budget (default 30 s, max 300 s).
+
+**Everything entering a mapped source is mapped** — jobs, syncs and `Source.add()` alike, one rule. `add()` on a mapped source returns `{ documents, document, outcome }` (several documents from one object is normal) and throws `mapper_skipped` when the mapper refused the object: surface that to the user as what it is, not as a generic failure.
+
+The loop, in this order:
+
+```bash
+remy-admin datasources inspect --source archive --connector                        # look first
+remy-admin datasources map test --source archive --connector --limit 20 --dev      # the LOCAL mapper, real objects, nothing ingested
+# fix, re-run, until the outcomes read right; then deploy
+remy-admin datasources map test --source archive --connector --limit 20            # the compiled mapper, same objects
+remy-admin datasources sync --source archive --wait                                # or jobs start
+remy-admin datasources jobs quarantine <id>                                        # what it skipped or failed on, by reason
+remy-admin datasources jobs replay <id> --wait                                     # after a fix + deploy: just those objects again
+remy-admin datasources remap --source archive --wait                               # a changed mapper over every raw copy
+```
+
+`map test --dev` needs the dev session running (`npx mindstudio dev`); it runs the mapper from local source through the tunnel and prints every outcome with markdown previews. The plan of a mapped job records the mapper's outcome mix on its sample; a run whose skip share climbs past twice that pauses with `pauseReason: 'skips'` for a look at the quarantine. `remap` reads the platform's own raw copies — no origin traffic — skips unchanged markdown by hash, and supersedes changed documents, so a metadata tweak on a million-document source costs frames and little else. `externalId` is the identity everything replaces by; choose it deliberately (the record's stable id, never the key of a file that gets rewritten in place).
+
+## Dedicated capacity
+
+The shared pool holds a source up to a per-source cap of chunks. A corpus beyond it — a plan that answers `plan_requires_dedicated` — runs on dedicated retrieval capacity the workspace provisions and pays for hourly: `remy-admin infra list` shows the offering catalog with prices and any resources the app has, `infra provision --offering <id> --name <n> --wait` creates one, and `datasources move --source archive --to <resource-id> --wait` puts the source on it with its data intact (`create --placement <resource-id>` starts a new source there). Provisioning bills the workspace; **never provision without the user's explicit confirmation**, and show them the offering's price first. `infra --help` covers hibernate, resume, resize and destroy.
 
 ## Answering from results
 
@@ -105,4 +235,27 @@ remy-admin datasources promote --source policies                              #
 
 Search serves the current version throughout, so nothing degrades while the new one builds. `datasources drop` discards an unwanted candidate.
 
+## Evaluating retrieval — measure before you promote
+
+"Is the rebuilt version better?" and "which embedding model should this corpus use?" are measured, not guessed. The instrument is a query set (questions with the documents that should come back) run against a version, returning recall@k, MRR, nDCG, latency and cost per query as JSON. You present the table.
+
+```bash
+remy-admin datasources sample --source archive --size 300 --stratify year --wait      # a representative subset, same config as the parent
+remy-admin datasources eval create --source archive-sample --name base --size 150 --wait    # 150 queries generated from the sample
+remy-admin datasources eval run --set <setId> --label live --wait                            # score the live version
+remy-admin datasources revectorize --source archive-sample --max-chars 900 --wait            # the change under test
+remy-admin datasources eval run --set <setId> --candidate --label small-chunks --wait        # score the candidate
+remy-admin datasources eval compare <runA> <runB>                                            # deltas + per-query wins/losses
+remy-admin datasources eval result <runId> --worst 10                                        # what the misses retrieved instead
+```
+
+Rules that keep the numbers honest:
+
+- **Work on a sample.** A sample shares the parent's exact pinned config and its documents' content hashes, so a set built on it also scores the parent (`eval run --set <id> --source archive`). Re-vectorizing a 300-document sample costs cents; the full corpus costs real money.
+- **Generated queries are a start, not the truth.** The default `cloze` style holds a sentence out of a chunk — free and deterministic, but it flatters keyword matching because a real user does not type sentences from the document. `--style question` has a chat model write the question a user would ask (a model call per query). Add the questions the user actually cares about with `eval add --set <id> --query "..." --expect <filename>` or `eval import` from JSONL; tag them so `byTag` breaks the numbers down.
+- **Runs cost searches.** Every query is a real search (embedding + rerank spend), capped at 2,000 per run. Say so before running a large set.
+- **Read the branch breakdown.** `branches.{denseOnly, lexicalOnly, both}` says which half of hybrid found the answers; a corpus of part numbers and proper nouns lives on `lexicalOnly`, and that is the case for keeping hybrid on even when it costs latency.
+- **Compare like with like.** Same set, same target, one change at a time: a version (`--candidate`), a retrieval override (`--mode`, `--rerank false`, `--rerank-model <id>`, `--hybrid false`), never both in one run.
+- **Present, then recommend.** A table with run labels, recall@5, MRR, p50 latency and cost per query, then one sentence: promote or drop. Ask before promoting; it changes what the deployed app retrieves.
+
 For anything deeper on the SDK, ask `askMindStudioSdk` rather than guessing at an API.

package/dist/prompt/static/coding.md

@@ -82,7 +82,7 @@ You have access to the `mindstudio` CLI, which exposes every SDK action as a com
 ### Production App Management
 You have access to `remy-admin`, a CLI for managing the user's production app. Use it via your bash tool. All output is JSON. Run `remy-admin --help` or `remy-admin <command> --help` to discover usage and available options.
 
-Available commands: `requests` (server logs, errors, latency), `crashes` (frontend browser errors), `analytics` (traffic queries — lifetime metrics, sources, live counters), `releases`, `diagnostics` (Lighthouse audit), `domains`, `users` (list, set roles), `db` (query production sql), `data` (live db operations like lift-from-dev), `methods` (list, invoke), `secrets`, `files` (CDN files), `datasources` (document corpora), `prerender` (crawler snapshots), `voice` (phone numbers, call logs, voice policy), `issues` (externally-reported bugs), `settings` (app settings: signup restrictions, app-store-reviewer test accounts, embedding origins, toggles).
+Available commands: `requests` (server logs, errors, latency), `crashes` (frontend browser errors), `analytics` (traffic queries — lifetime metrics, sources, live counters), `releases`, `diagnostics` (Lighthouse audit), `domains`, `users` (list, set roles), `db` (query production sql), `data` (live db operations like lift-from-dev), `methods` (list, invoke), `secrets`, `files` (CDN files), `datasources` (document corpora: add, search, `inspect` a bucket or store, bulk `jobs` with a plan, S3 `connect`/`sync`, mappers via `map test`/`remap`, `eval`), `infra` (dedicated retrieval capacity for large corpora), `prerender` (crawler snapshots), `voice` (phone numbers, call logs, voice policy), `issues` (externally-reported bugs), `settings` (app settings: signup restrictions, app-store-reviewer test accounts, embedding origins, toggles).
 
 Two rules: buying a `voice` phone number bills $1/month — never buy without the user's explicit confirmation. `issues` is for externally-reported bugs only — read from it and resolve items when the user asks; never use it to track work you are doing with the user.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mindstudio-ai/remy",
-  "version": "0.1.320",
+  "version": "0.1.321",
   "description": "Remy coding agent",
   "repository": {
     "type": "git",