@wrangled/cli 0.3.2 → 0.3.4

package/dist/api.js

@@ -146,6 +146,16 @@ export async function putSigned(signedUrl, body, contentType) {
  * token, so no message ever includes it.
  */
 export async function getSigned(signedUrl) {
+    return new Uint8Array(await (await fetchSigned(signedUrl)).arrayBuffer());
+}
+/** Like `getSigned`, but hands over the file as it arrives instead of holding it whole. */
+export async function streamSigned(signedUrl) {
+    const response = await fetchSigned(signedUrl);
+    if (!response.body)
+        throw new CliError("File storage answered the download with no file. Try again.", exitCodes.error);
+    return response.body;
+}
+async function fetchSigned(signedUrl) {
     const signed = new URL(signedUrl);
     let response;
     try {
@@ -159,7 +169,7 @@ export async function getSigned(signedUrl) {
         await response.body?.cancel();
         throw new CliError(`File storage answered HTTP ${response.status} to the download. Try again.`, exitCodes.error);
     }
-    return new Uint8Array(await response.arrayBuffer());
+    return response;
 }
 async function request(connection, path, init) {
     let response;

package/dist/files.js

@@ -41,6 +41,7 @@ async function realLocation(path) {
         return join(await realLocation(parent), basename(path));
     }
 }
+/** `content` may arrive in pieces, which are written as they come; the file appears whole or not at all. */
 export async function writeFileAtomic(path, content, mode) {
     const temp = join(dirname(path), `.${basename(path)}.${randomBytes(6).toString("hex")}.tmp`);
     try {

package/dist/guide.js

@@ -37,7 +37,7 @@ To read the whole library with your own tools, such as "every email we have" or
 
 1. Run \`${cli} warehouse schema --summary\` before writing any SQL. It lists each table once: whether it is a table or a view, which source fills it, how many columns it has, and a one-line summary. Then run \`${cli} warehouse schema --table <name>\` (repeat for several) or \`--search <words>\` for the full detail of just the tables you need: every column with its Postgres type, nullability, and description. Plain \`${cli} warehouse schema\` prints every column of every table, which on a big warehouse is a lot. Read the descriptions: they say what null means and which metrics do not add up. Use only these names. Never guess a table or column name.
 2. Write one SQL statement in the PostgreSQL dialect. Refer to tables unqualified (\`FROM orders\`). The workspace schema is already on the search path.
-3. Run it: \`${cli} warehouse query "SELECT ..."\`. For long SQL, use \`--file query.sql\` or pipe it on stdin. To keep the rows out of your context, add \`--out results.csv\` (or \`.json\`): it saves every row the server returned and prints one line.
+3. Run it: \`${cli} warehouse query "SELECT ..."\`. For long SQL, use \`--file query.sql\` or pipe it on stdin. To keep the rows out of your context, add \`--out results.csv\` (or \`.json\`): it saves every row the server returned and prints one line. When you need more rows than one answer holds, add \`--all\` as well: the server saves the whole answer to a file and the CLI downloads it (see Limits).
 4. Aggregate in SQL (\`GROUP BY\`, \`SUM\`, \`date_trunc\`) and add \`LIMIT\`. Do not pull raw rows to aggregate them yourself.
 5. Before trusting recent or old numbers, run \`${cli} sources list\`. It shows when each source last synced, its last error, and when it syncs next. \`${cli} sources show <id>\` adds each table's row count and date range, the history still to fetch, and the last sync runs.
 
@@ -62,6 +62,7 @@ A webhook address lets a tool such as a form or webinar platform push events int
 - One statement per call, and it must return rows: \`SELECT\`, \`VALUES\`, or \`WITH ... SELECT\`. Anything else (\`INSERT\`, \`UPDATE\`, \`DELETE\`, DDL, \`SET\`, \`COPY\`) fails.
 - 15-second statement timeout.
 - Results stop at 1000 rows or about 5 MB. In JSON output, \`truncated\` is \`"row_limit"\` or \`"byte_limit"\` when a limit was hit, otherwise \`null\`.
+- \`--out <file> --all\` saves the whole answer instead, up to 1000000 rows or about 50 MB, through a download link that works for 10 minutes; the server deletes its copy after 24 hours. A \`.ndjson\` or \`.jsonl\` file gets one JSON row per line. Over the REST API, \`POST /api/v1/warehouse/export\` with \`{"sql": "..."}\` answers \`{ url, format, columns, rowCount, bytes, truncated, expiresAt }\`; the file at \`url\` is NDJSON.
 - In JSON output, \`columns\` lists each column's name and Postgres type. \`numeric\` and \`bigint\` values are JSON numbers with their exact digits, so parse them with a big-number-safe parser when precision matters. Dates are \`YYYY-MM-DD\` strings.
 - Every output column needs a unique name. Alias duplicates (\`a.id AS order_id\`).
 
@@ -142,6 +143,8 @@ Each workspace keeps one map of what the client sells (the offer stack) and the
 
 To change the map, write the whole map as JSON (\`{"offers": [...], "funnels": [...]}\`, the \`map\` of \`${cli} funnels show --format json\`) and run \`${cli} funnels draft --file map.json --note "<what changed and why>"\`. Add \`--evidence '<json>'\` or \`@file.json\` with what you looked at: counts and ids, never names, emails or phones. It is checked against synced data, and when every step's rule is filled in it goes live at once, confirmed by \`system:auto\`; a person can undo it in the app. A map with a rule not filled in stays on hold as the draft and counts nothing; saving replaces the draft on hold. Never invent an id or a number to fill a rule. Pass \`--based-on <n>\` with the version you read (the draft on hold, else the confirmed map): if someone saved since, the save is refused as \`draft_conflict\`, and you show again and redo your change on top. Leave out \`--based-on\` only for the first map, when there is neither a map nor a draft; once there is one, a save without it is refused as \`draft_conflict\`. A map that does not fit is refused with every problem by path, such as \`funnels[0].steps[1].offer_key: no offer has the key x\`. Saving prints whether the map went live, what is left to fill in (an asset not synced yet, a rule with no calendar chosen), and the parts synced data does not back up, such as an offer with fewer than 3 buyers in a year.
 
+\`--counts-from fix\` (the default) makes the map count for every past date the current map counted, as a fix; \`--counts-from <YYYY-MM-DD>\` makes it a change in the business that counts from that day (today or earlier). Save a fix and a change as two drafts, the fix first. \`--questions @questions.json\` asks a person what only they can settle: an array of \`{question_id, text, kind, evidence, options: [{label, patch}], default_option}\`, where the map already holds the default option and each other option's patch is JSON Patch ops on the map, naming offers, funnels and steps by key (\`/offers/course/matches/-\`). A question whose options all count the same sales and people, or that was answered already, is left out; at most 7 stay open, biggest money at stake first, and the checks add a yes/no question for each weak part no question covers. People answer them in the app; an answer saves the map as a fix, and later drafts keep it. \`${cli} funnels show\` lists the open and answered questions and says whether the map counts as a fix or from a date.
+
 \`${cli} funnels numbers\` counts each step of each funnel in the confirmed map over \`--from\` and \`--to\` (default the last 30 days; \`--funnel <key>\` for one funnel): how many reached it, and how many of those who reached the step before in those days moved on to it within the window, with the drop-off and the median days it took. A step whose rules name a person (webinar, GoHighLevel, payments) counts people and follows the same people from step to step; a step that cannot (ad clicks, page visitors) counts events, and its comparison is a rate of counts. A step whose data is not connected says unknown, never 0. Numbers are worked out after each sync by the rules of the map version confirmed at the time, so a renamed step keeps its numbers. For SQL, read \`saved_funnel_steps\`, \`funnel_step_daily\` and \`funnel_conversion\`.
 
 \`${cli} funnels evidence\` gathers what the workspace's own data says it sells, to draft a map from: the prices people paid in the last 365 days on each platform with buyer counts, the most common orders in which one person paid (\`1500 → 497 → 3500\`, with the median days between), GoHighLevel funnels and their steps, calendars with bookings and shows, pipeline stages, the most used tags, webinars with sign-ups, viewers and offer clicks, the landing pages ads link to, and active Facebook campaigns with 30-day spend. A part whose source is not connected says so. Ids, counts and amounts only. The \`map-offers\` playbook (\`${cli} skill install map-offers\` installs it as the \`${MAP_OFFERS_SKILL_NAME}\` skill, with a worked example map beside it; \`${cli} docs map-offers\` prints it) walks an agent from the evidence to a draft.
@@ -156,6 +159,18 @@ Kinds are \`exclusion\`, \`gap\`, \`baseline\`, \`change\`, \`definition\`, \`ca
 
 To suggest a fact, write it as JSON and run \`${cli} facts propose --file fact.json\` (or pipe it on stdin): \`{"kind": "gap", "subject_type": "table", "subject_ref": "meta_insights_daily", "key": "history_start", "value": {"table": "meta_insights_daily", "from": "2026-04-07"}, "summary": "Facebook ad numbers only go back to April 7, 2026.", "evidence": {"query": "SELECT min(date) FROM meta_insights_daily"}}\`. You only suggest: a person confirms, fixes or rejects it in the app. A fact with the same kind, subject and key already open is refused as \`fact_exists\`. Evidence is counts, queries and ids, never names, emails or phones.
 
+## The weekly story
+
+Each workspace keeps a weekly story: a one-screen read of what the data says this week, why, what is working, what the data cannot tell us, and what to do next. \`${cli} story\` prints the newest version, \`${cli} story list\` every version newest first, and \`${cli} story show <version>\` one of them (each takes \`--json\`). Every save is a new version; nothing is ever changed or deleted.
+
+To save one, write it as JSON and run \`${cli} story submit --file story.json\` (or pipe it on stdin):
+
+\`\`\`json
+{"period_end": "2026-09-27", "windows": [{"key": "7d", "from": "2026-09-21", "until": "2026-09-27", "vs_from": "2026-09-14", "vs_until": "2026-09-20"}], "headline": "Leads held steady while spend fell 12%.", "sections": [{"key": "what_happened", "title": "What happened", "audience": "both", "claims": [{"text": "Spend fell 12% week over week.", "confidence": "high", "evidence": ["spend"]}]}], "evidence": {"spend": {"sql": "SELECT sum(spend) FROM meta_insights_daily WHERE date >= '2026-09-14'"}}}
+\`\`\`
+
+Each section's \`audience\` is \`staff\`, \`client\` or \`both\`; a person on staff decides in the app when the client part may be shown. Every claim lists its evidence: keys of \`evidence\`, or \`fact:<id>\` for a settled fact. Every evidence query must be used by a claim, and a story carries at most 30 queries. The server runs each query itself, read-only, and keeps the rows it got back, so the numbers people see are what the warehouse said; any rows you send are ignored. A query that fails, is not one SELECT or WITH statement, or was cut off at the warehouse's row or size limit refuses the whole story with its key named, and so do rows past 256 KiB across the story: keep evidence to the totals the sentences use. Cite only proposed or confirmed facts. To correct a story, submit a new one with \`"previous_id"\` set to the id of the one it fixes.
+
 ## The Facebook ads agent
 
 The daily Facebook ads agent follows its own playbook: install it with \`${cli} skill install meta-ads\` (or \`--codex\` for an AGENTS.md in the current directory) and read it with \`${cli} docs meta-ads\`. It works in runs (\`${cli} agent start\`, \`${cli} agent finish\`), reads and suggests settled facts (\`${cli} facts\`), keeps a brand profile (\`${cli} brand\`), and makes proposals (\`${cli} proposals\`) that a person approves in the web app. Nothing it does is sent to Meta.
@@ -192,7 +207,7 @@ ${cli} whoami                       show the current workspace
 ${cli} guide                        what this workspace's data holds, how fresh it is, and what is missing: read it first
 ${cli} brief [--from <date>] [--to <date>] [--channel <channel>] [--campaign <id>] [--top <n>]   the offer, voice, best and worst ads, pages, results and audience: start here before writing ads
 ${cli} warehouse schema [--summary] [--table <name> ...] [--search <words>]   list tables and columns; --summary for one row per table
-${cli} warehouse query "<sql>"      run one read-only query (or --file <path>, or stdin); --out <file.csv | file.json> saves the rows
+${cli} warehouse query "<sql>"      run one read-only query (or --file <path>, or stdin); --out <file.csv | file.ndjson | file.json> saves the rows; add --all for every row
 ${cli} funnel [--from <date>] [--to <date>] [--grain day|week|month]   spend, leads, calls and sales per period, with totals
 ${cli} money [--from <date>] [--to <date>] [--grain day|week|month]   money in, refunds, ad spend, payouts and what is left, per period, with totals
 ${cli} campaigns [--from <date>] [--to <date>] [--channel <channel>] [--limit <n>]   each campaign's results, best first, with totals
@@ -240,9 +255,11 @@ ${cli} trail <plan-id | sku | item-id>   how a plan's items were made, where the
 ${cli} funnels show [--version <n>] | funnels versions [--diff <a> <b>]   the offers and funnels map, as text, and its versions
 ${cli} funnels numbers [--funnel <key>] [--from <date>] [--to <date>]   how many reached each funnel step and moved on
 ${cli} funnels evidence   prices paid, buyer paths, GoHighLevel funnels, calendars and tags, webinars, pages and campaigns, to draft the map from
-${cli} funnels draft --file <map.json> [--note <why>] [--evidence '<json>' | @file.json] [--based-on <n>]   save the whole map; it goes live when every step can be counted
+${cli} funnels draft --file <map.json> [--note <why>] [--evidence '<json>' | @file.json] [--based-on <n>] [--counts-from fix|<date>] [--questions @file.json]   save the whole map; it goes live when every step can be counted
 ${cli} facts [--kind <kind>] [--subject <type>[:<ref>]] [--status <status>] [--json] | facts show <id>   settled facts about the data, to treat as true
 ${cli} facts propose --file <fact.json>   suggest a fact for a person to confirm
+${cli} story [--json] | story list | story show <version>   the weekly story, its versions, and the rows behind each sentence
+${cli} story submit --file <story.json>   save the next version of the weekly story
 ${cli} worker list | worker runs <hand-id | skill> [--limit <n>] | worker run-now <hand-id | skill>   this workspace's scheduled hands, their runs and costs, and Run now
 ${cli} skill install [${PLAYBOOK_CHOICES}] [--project] [--codex]   install a Claude Code skill (or the playbook for Codex)
 ${cli} docs [${PLAYBOOK_CHOICES}]   print this guide, or a hand's playbook

package/dist/main.js

@@ -4,7 +4,7 @@ import { mkdir, readFile, stat } from "node:fs/promises";
 import { homedir } from "node:os";
 import { basename, dirname, extname, join, resolve } from "node:path";
 import { parseArgs } from "node:util";
-import { api, ApiError, apiDownload, apiText, apiUpload, CliError, exitCodes, getSigned, IMPORT_LIMITS, IMPORT_NAME_PATTERN, putSigned, readCredentials, removeCredentials, resolveApiUrl, resolveConnection, UnreadableCredentialsError, writeCredentials, } from "./api.js";
+import { api, ApiError, apiDownload, apiText, apiUpload, CliError, exitCodes, getSigned, IMPORT_LIMITS, IMPORT_NAME_PATTERN, putSigned, readCredentials, removeCredentials, resolveApiUrl, resolveConnection, UnreadableCredentialsError, writeCredentials, streamSigned, } from "./api.js";
 import { brand, envPrefix } from "./brand.js";
 import { firstSymlink, inRunFolder, writeFileAtomic } from "./files.js";
 import { guide, skill } from "./guide.js";
@@ -102,10 +102,16 @@ const commands = {
         },
     },
     "warehouse query": {
-        usage: 'warehouse query "<sql>" | --file <path> | < query.sql [--out <file.csv | file.json>]',
+        usage: 'warehouse query "<sql>" | --file <path> | < query.sql [--out <file.csv | file.ndjson | file.json> [--all]]',
         async run(args, options) {
+            if (options.all && options.out === undefined) {
+                throw new CliError("--all saves to a file: add --out <file.csv | file.ndjson | file.json>.", exitCodes.usage);
+            }
             const sql = await readSql(args, options.file);
-            const text = await apiText(await resolveConnection(options.apiUrl), "/api/v1/warehouse/query", { sql });
+            const connection = await resolveConnection(options.apiUrl);
+            if (options.all)
+                return saveWholeAnswer(connection, sql, options.out, options.format);
+            const text = await apiText(connection, "/api/v1/warehouse/query", { sql });
             if (options.out !== undefined)
                 return saveQueryResult(text, options.out, options.format);
             if (options.format === "json")
@@ -113,7 +119,7 @@ const commands = {
             const result = queryResultWithExactNumbers(text);
             console.log(renderTable(result.columns.map((c) => c.name), result.rows));
             console.log(`(${rowCount(result.rows.length)})`);
-            warnIfTruncated(result);
+            warnIfTruncated(result.truncated);
         },
     },
     funnel: {
@@ -983,24 +989,36 @@ const commands = {
         },
     },
     "funnels draft": {
-        usage: "funnels draft --file <map.json> [--note <why>] [--evidence '<json>' | @file.json] [--based-on <n>]   save a whole map; it counts at once when every step can be counted",
+        usage: "funnels draft --file <map.json> [--note <why>] [--evidence '<json>' | @file.json] [--based-on <n>] [--counts-from fix|<YYYY-MM-DD>] [--questions '<json>' | @file.json]   save a whole map; it counts at once when every step can be counted",
         async run(args, options) {
-            const usage = "Usage: funnels draft --file <map.json> [--note <why>] [--evidence '<json>' | @file.json] [--based-on <n>]";
+            const usage = "Usage: funnels draft --file <map.json> [--note <why>] [--evidence '<json>' | @file.json] [--based-on <n>] [--counts-from fix|<YYYY-MM-DD>] [--questions '<json>' | @file.json]";
             if (args.length > 0 || !options.file)
                 throw new CliError(usage, exitCodes.usage);
+            if (options.countsFrom !== undefined && options.countsFrom !== "fix" && !/^\d{4}-\d{2}-\d{2}$/.test(options.countsFrom)) {
+                throw new CliError("--counts-from is fix or a date, YYYY-MM-DD", exitCodes.usage);
+            }
             const map = await parseJson(`@${options.file}`, "--file");
             const evidence = options.evidence === undefined ? undefined : await parseJson(options.evidence, "--evidence");
+            const questions = options.questions === undefined ? undefined : await parseJson(options.questions, "--questions");
             const connection = await resolveConnection(options.apiUrl);
             const basedOn = options.basedOn === undefined ? null : wholeNumber(options.basedOn, "--based-on");
-            const saved = await api(connection, `${FUNNELS_PATH}/draft`, { map, note: options.note, evidence, based_on: basedOn }, "PUT");
+            const saved = await api(connection, `${FUNNELS_PATH}/draft`, { map, note: options.note, evidence, based_on: basedOn, counts_from: options.countsFrom, questions }, "PUT");
             if (options.format === "json")
                 return printJson(saved);
             const based = basedOn === null ? "" : `, based on version ${basedOn}`;
+            const counts = options.countsFrom === undefined || options.countsFrom === "fix" ? "it counts for past dates too, as a fix" : `it counts from ${options.countsFrom}, as a change`;
             console.log(saved.applied
-                ? `Map version ${saved.version.version} is live${based}: it counts from now. A person can undo it in the app.`
+                ? `Map version ${saved.version.version} is live${based}: ${counts}. A person can undo it in the app.`
                 : `Saved version ${saved.version.version}${based}. It is on hold: it counts nothing until every step can be counted.`);
-            if (saved.warnings.length > 0)
-                console.log(`\n${saved.applied ? "Not in synced data yet" : "To fill in"}:\n${saved.warnings.map((w) => `- ${w.path}: ${w.message}`).join("\n")}`);
+            if (saved.reapplied.length > 0)
+                console.log(`\nAnswers put back into the map: ${saved.reapplied.join(", ")}`);
+            if (saved.asked.length > 0)
+                console.log(`\nQuestions open for a person, biggest money at stake first: ${saved.asked.join(", ")}`);
+            const [left, warnings] = [saved.warnings.filter((w) => w.message.startsWith("question ")), saved.warnings.filter((w) => !w.message.startsWith("question "))];
+            if (warnings.length > 0)
+                console.log(`\n${saved.applied ? "Not in synced data yet" : "To fill in"}:\n${warnings.map((w) => `- ${w.path}: ${w.message}`).join("\n")}`);
+            if (left.length > 0)
+                console.log(`\nQuestions left out:\n${left.map((w) => `- ${w.path}: ${w.message}`).join("\n")}`);
             if (saved.weak.length > 0)
                 console.log(`\nNot backed up by synced data:\n${saved.weak.map((w) => `- ${w.path}: ${w.reason}`).join("\n")}`);
         },
@@ -1034,8 +1052,9 @@ const commands = {
                 "confirmed by": v.confirmedBy,
                 "confirmed at": v.confirmedAt,
                 note: v.note && excerpt(v.note),
+                counts: v.counts == null ? null : v.counts.kind === "fix" ? "for past dates too (fix)" : `from ${v.counts.from} (change)`,
             }));
-            console.log(renderTable(["version", "status", "created by", "created at", "confirmed by", "confirmed at", "note"], rows));
+            console.log(renderTable(["version", "status", "created by", "created at", "confirmed by", "confirmed at", "note", "counts"], rows));
         },
     },
     facts: {
@@ -1084,6 +1103,66 @@ const commands = {
             console.log(`Proposed fact ${fact.id}. A person confirms or rejects it in the app.`);
         },
     },
+    story: {
+        usage: "story   the newest weekly story, with the rows behind each sentence",
+        async run(args, options) {
+            if (args.length > 0)
+                throw new CliError("Usage: story | story list | story show <version> | story submit --file <story.json>", exitCodes.usage);
+            const { story } = await api(await resolveConnection(options.apiUrl), `${STORIES_PATH}/latest`);
+            if (options.format === "json")
+                return printJson({ story });
+            if (!story)
+                return console.log("No story yet.");
+            printStory(story);
+        },
+    },
+    "story list": {
+        usage: "story list   every version of the weekly story, newest first",
+        async run(args, options) {
+            if (args.length > 0)
+                throw new CliError("Usage: story list", exitCodes.usage);
+            const { stories } = await api(await resolveConnection(options.apiUrl), STORIES_PATH);
+            if (options.format === "json")
+                return printJson({ stories });
+            if (stories.length === 0)
+                return console.log("No story yet.");
+            const versions = new Map(stories.map((s) => [s.id, s.version]));
+            const rows = stories.map((s) => ({
+                version: s.version,
+                "period end": s.periodEnd,
+                corrects: s.previousId && versions.get(s.previousId),
+                author: s.author,
+                saved: s.createdAt,
+                headline: excerpt(s.headline, 80),
+            }));
+            console.log(renderTable(["version", "period end", "corrects", "author", "saved", "headline"], rows));
+        },
+    },
+    "story show": {
+        usage: "story show <version>   one version of the weekly story",
+        async run(args, options) {
+            if (args.length !== 1)
+                throw new CliError("Usage: story show <version>", exitCodes.usage);
+            // `show latest` reaches the newest-story route, which answers null before the first story.
+            const { story } = await api(await resolveConnection(options.apiUrl), `${STORIES_PATH}/${encodeURIComponent(args[0])}`);
+            if (options.format === "json")
+                return printJson({ story });
+            if (!story)
+                return console.log("No story yet.");
+            printStory(story);
+        },
+    },
+    "story submit": {
+        usage: "story submit --file <story.json> | < story.json   save the next version; the server runs each query and keeps its rows",
+        async run(args, options) {
+            if (args.length > 0)
+                throw new CliError("Usage: story submit --file <story.json> | < story.json", exitCodes.usage);
+            const { story } = await api(await resolveConnection(options.apiUrl), STORIES_PATH, await readToolInput(options));
+            if (options.format === "json")
+                return printJson({ story });
+            console.log(`Saved story version ${story.version}.`);
+        },
+    },
     "plans create": {
         usage: "plans create --name <name> --goal <text> --channel meta|ghl|klaviyo|web [--target '<json>' | @file.json] [--landing <sku | item-id> | --landing-url <url>] [--funnel <key> --step <key>] [--run <agent-run-id>]",
         async run(args, options) {
@@ -1471,6 +1550,20 @@ const FACT_COLUMNS = ["id", "kind", "subject", "key", "status", "summary"];
 function factRow(f) {
     return { id: f.id, kind: f.kind, subject: `${f.subjectType}:${f.subjectRef}`, key: f.key, status: f.status, summary: excerpt(f.summary, 80) };
 }
+const STORIES_PATH = "/api/v1/stories";
+function printStory(story) {
+    const corrects = story.previousId ? `, correcting ${story.previousId}` : "";
+    console.log(`Story version ${story.version}, up to ${story.periodEnd}, saved ${story.createdAt} by ${story.author}${corrects}.`);
+    console.log(`\n${story.headline}`);
+    for (const section of story.sections) {
+        console.log(`\n${section.title} (${section.audience})`);
+        for (const claim of section.claims)
+            console.log(`- ${claim.text} [${claim.confidence}: ${claim.evidence.join(", ")}]`);
+    }
+    const evidence = Object.entries(story.evidence).map(([key, e]) => ({ key, rows: e.rows.length, sql: excerpt(e.sql, 80) }));
+    if (evidence.length > 0)
+        console.log(`\nEvidence, as the server read it on saving:\n${renderTable(["key", "rows", "sql"], evidence)}`);
+}
 const PROPOSAL_COLUMNS = ["id", "kind", "status", "target", "change", "reason"];
 function proposalChange(p) {
     if (p.kind === "creative")
@@ -1523,6 +1616,8 @@ Options:
   --search <words>      warehouse schema: describe in full the tables whose name, description or columns mention a word
   --out <path>          the file, or existing folder, creatives download and warehouse query save to; warehouse query
                         writes CSV for a .csv path, otherwise JSON
+  --all                 with warehouse query --out, save the whole answer (up to 1,000,000 rows or 50 MiB) instead of
+                        the first 1,000 rows or 5 MiB; .ndjson or .jsonl saves one JSON row per line
   --open                open the file creatives download saved
   --media               with content pull, also save each item's files and HTML under media/ in the folder
   --full                with content pull, fetch every item, not only what changed since the last pull (brings new numbers)
@@ -1550,6 +1645,8 @@ Options:
   --based-on <n>        the map version funnels draft was written from; refused when someone saved since
                         (leave it out only for the first map, when there is neither a map nor a draft)
   --evidence <json | @file.json>   what funnels draft was written from: counts and ids, never names, emails or phones
+  --counts-from fix|<YYYY-MM-DD>   how funnels draft counts: fix (default) for past dates too, or a change from that day
+  --questions <json | @file.json>  the questions funnels draft asks a person, as an array
   --diff <a> <b>        with funnels versions, what changed from version a to version b
   --funnel <key>        only this funnel's steps in funnels numbers (default: every funnel in the confirmed map);
                         with plans create and --step, the funnel whose step the push feeds
@@ -1604,6 +1701,7 @@ function parseOptions(argv) {
             summary: { type: "boolean", default: false },
             search: { type: "string" },
             out: { type: "string" },
+            all: { type: "boolean", default: false },
             open: { type: "boolean", default: false },
             media: { type: "boolean", default: false },
             full: { type: "boolean", default: false },
@@ -1647,6 +1745,8 @@ function parseOptions(argv) {
             version: { type: "string" },
             "based-on": { type: "string" },
             evidence: { type: "string" },
+            "counts-from": { type: "string" },
+            questions: { type: "string" },
             diff: { type: "string" },
             funnel: { type: "string" },
             step: { type: "string" },
@@ -1692,6 +1792,7 @@ function parseOptions(argv) {
             summary: values.summary,
             search: values.search,
             out: values.out,
+            all: values.all,
             open: values.open,
             media: values.media,
             full: values.full,
@@ -1735,6 +1836,8 @@ function parseOptions(argv) {
             version: values.version,
             basedOn: values["based-on"],
             evidence: values.evidence,
+            countsFrom: values["counts-from"],
+            questions: values.questions,
             diff: values.diff,
             funnel: values.funnel,
             step: values.step,
@@ -2313,11 +2416,78 @@ async function saveQueryResult(text, out, format) {
     const path = await savePath(out, "results.json");
     const result = queryResultWithExactNumbers(text);
     await saveFile(path, extname(path).toLowerCase() === ".csv" ? toCsv(text) : text);
+    reportSaved(path, result.rows.length, result.truncated, format);
+}
+/**
+ * Has the server save the whole answer to storage, then streams it from there into the file: CSV for a `.csv` path,
+ * one JSON row per line for `.ndjson` or `.jsonl`, otherwise the same JSON `saveQueryResult` writes.
+ */
+async function saveWholeAnswer(connection, sql, out, format) {
+    const path = await savePath(out, "results.json");
+    const saved = await api(connection, "/api/v1/warehouse/export", { sql });
+    const rows = exportRows(await streamSigned(saved.url), saved.rowCount);
+    const extension = extname(path).toLowerCase();
+    const content = extension === ".csv"
+        ? csvLines(saved.columns, rows)
+        : extension === ".ndjson" || extension === ".jsonl"
+            ? suffixed(rows, "\n")
+            : queryJson(saved, rows);
+    await saveFile(path, content);
+    reportSaved(path, saved.rowCount, saved.truncated, format, true);
+}
+/** Each row of the export's NDJSON as it arrives; throws if the download ends with rows missing. */
+async function* exportRows(body, expected) {
+    const decoder = new TextDecoder();
+    let received = 0;
+    let pending = "";
+    const stopped = (why) => new CliError(`The download ${why} after ${rowCount(received)} of ${expected.toLocaleString("en-US")}. Try again.`, exitCodes.error);
+    const reader = body[Symbol.asyncIterator]();
+    for (;;) {
+        let next;
+        try {
+            next = await reader.next();
+        }
+        catch (error) {
+            throw stopped(`broke off (${error.message})`);
+        }
+        if (next.done)
+            break;
+        const lines = (pending + decoder.decode(next.value, { stream: true })).split("\n");
+        pending = lines.pop();
+        for (const line of lines) {
+            received++;
+            yield line;
+        }
+    }
+    if (pending + decoder.decode() !== "" || received !== expected)
+        throw stopped("stopped");
+}
+async function* suffixed(lines, suffix) {
+    for await (const line of lines)
+        yield line + suffix;
+}
+async function* csvLines(columns, rows) {
+    const csv = csvWriter(columns);
+    yield csv.header;
+    for await (const row of rows)
+        yield csv.row(JSON.parse(row, keepRawNumbers));
+}
+/** The query route's JSON, with the rows as the server wrote them. */
+async function* queryJson(saved, rows) {
+    yield `{"columns":${JSON.stringify(saved.columns)},"rows":[`;
+    let first = true;
+    for await (const row of rows) {
+        yield first ? row : `,${row}`;
+        first = false;
+    }
+    yield `],"rowCount":${saved.rowCount},"truncated":${JSON.stringify(saved.truncated)}}`;
+}
+function reportSaved(path, rows, truncated, format, whole = false) {
     if (format === "json")
-        printJson({ path: resolve(path), rowCount: result.rows.length, truncated: result.truncated });
+        printJson({ path: resolve(path), rowCount: rows, truncated });
     else
-        console.log(`Saved ${rowCount(result.rows.length)} to ${path}`);
-    warnIfTruncated(result);
+        console.log(`Saved ${rowCount(rows)} to ${path}`);
+    warnIfTruncated(truncated, whole);
 }
 /** Top-level numbers keep the digits the server sent, so a numeric column is never rounded. */
 function queryResultWithExactNumbers(text) {
@@ -2329,18 +2499,28 @@ function queryResultWithExactNumbers(text) {
 function rowCount(n) {
     return `${n.toLocaleString("en-US")} ${n === 1 ? "row" : "rows"}`;
 }
-function warnIfTruncated(result) {
-    if (result.truncated)
-        console.error(`Results were cut off (${result.truncated}). Aggregate in SQL or add LIMIT.`);
+/** `whole` is true when the answer came from --all, which has no larger limit to point to. */
+function warnIfTruncated(truncated, whole = false) {
+    if (!truncated)
+        return;
+    const more = whole ? "" : ", or save every row with --out <file> --all";
+    console.error(`Results were cut off (${truncated}). Aggregate in SQL or add LIMIT${more}.`);
 }
-/** Every number keeps the digits the server sent, nested ones too, and json columns are written as JSON text. */
 function toCsv(text) {
     const result = JSON.parse(text, keepRawNumbers);
-    const json = new Set(result.columns.filter((c) => c.type === "json" || c.type === "jsonb").map((c) => c.name));
+    const csv = csvWriter(result.columns);
+    return [csv.header, ...result.rows.map(csv.row)].join("");
+}
+/**
+ * CSV lines, each ending in a newline. Rows must be parsed with `keepRawNumbers`, so every number keeps the digits the
+ * server sent, nested ones too; json columns are written as JSON text.
+ */
+function csvWriter(columns) {
+    const json = new Set(columns.filter((c) => c.type === "json" || c.type === "jsonb").map((c) => c.name));
     const field = (name, value) => (value != null && json.has(name) ? JSON.stringify(value) : cell(value));
-    const names = result.columns.map((c) => c.name);
-    const line = (values) => values.map(csvField).join(",");
-    return [line(names), ...result.rows.map((row) => line(names.map((name) => field(name, row[name]))))].map((l) => `${l}\n`).join("");
+    const names = columns.map((c) => c.name);
+    const line = (values) => `${values.map(csvField).join(",")}\n`;
+    return { header: line(names), row: (row) => line(names.map((name) => field(name, row[name]))) };
 }
 function keepRawNumbers(_key, value, context) {
     return typeof value === "number" && context?.source !== undefined ? JSON.rawJSON(context.source) : value;

package/dist/playbooks.js

@@ -583,7 +583,7 @@ ${cli} funnels show --format json > current.json
 - **campaigns**: active Facebook campaigns with their last 30 days' spend.
 - **pastCampaigns**: every Facebook campaign that ever spent, all time rather than the evidence window, biggest spend first: its first and last day, spend, results by type with each type's average value per result, and the addresses of the landing pages its ads link to where known (\`landingPageUrls\`). A name here can belong to a campaign no longer listed in \`campaigns\`.
 
-\`current.json\` is the map now: \`map\` (the confirmed one) and \`draft\` (on hold until a rule is filled in), each null when there is none. Note the version you start from: the draft's when one is on hold, else the confirmed map's, else none.
+\`current.json\` is the map now: \`map\` (the confirmed one) and \`draft\` (on hold until a rule is filled in), each null when there is none. Note the version you start from: the draft's when one is on hold, else the confirmed map's, else none. \`questions.answered\` lists the questions a person already answered: keep each answer in your map, and never ask them again.
 
 ## 2. Check past campaigns for offers the map is missing
 
@@ -628,17 +628,51 @@ Write \`map.json\` as \`{"offers": [...], "funnels": [...]}\`, in this order.
 3. **Assets** on each step: \`{"asset_type": ..., "ref": ...}\` with refs from the evidence: \`meta_campaign\` (campaign id), \`url\` (page address), \`content_item\` (content item id), \`ghl_funnel_step\` (step id), \`ghl_calendar\`, \`ghl_pipeline_stage\`, \`ghl_tag\`, \`webinar\` (webinar id). Give the step after Traffic its page, the funnel's first page, as a \`url\`, a \`content_item\` from \`pages\` or a \`ghl_funnel_step\`: the Traffic step finds its ads from it. Only that step's pages count, so a checkout page that two funnels share is never mistaken for a first page.
 4. **Rules**, what counts as reaching each step, each \`{"kind": "...", "params": {...}}\`, such as \`{"kind": "purchase", "params": {"offer_key": "course"}}\`. The kinds and their params: \`ad_traffic\` for the Traffic step, with no params: it counts every ad whose link goes to the funnel's first page, or to a short link that forwards there, and finds new ones on every sync, so never list campaign ids. Its one param, \`exclude_campaign_ids\`, leaves a campaign out; use it only when a campaign's ads point at the page for another reason, and say why in the report. \`meta_ads\` is an older kind kept so old maps still read; never write it. The others: \`page_visit\` (\`host\`, \`path\`), \`webinar\` (\`event_types\` and \`webinar_ids\`), \`ghl_tag\` (\`tags\`, exact tag names), \`ghl_appointment\` (\`calendar_ids\`, \`outcome\` booked or showed), \`ghl_stage\` (\`pipeline_id\`, \`stage_ids\`), \`ghl_workflow_event\` (\`endpoint_id\` of a GoHighLevel workflow's intake endpoint, and an optional \`match\` on \`event_type\`, \`workflow_id\` or \`workflow_name\`), \`purchase\` (\`offer_key\`), or \`none\` for a step that counts nothing. An empty \`webinar_ids\` counts every webinar: fill it in with the one webinar id when the evidence lists one, and with this step's ids when it lists more.
 
-## 6. Save the map
+## 6. Write your questions
+
+Some things the evidence can't tell you, such as whether the $497 flash-sale buyers are course buyers. Make your best guess in the map, then ask a person with a question card. Write \`questions.json\`, an array of questions. Each is \`{"question_id", "text", "kind", "evidence", "options", "default_option"}\`:
+
+- **Ask only when both are true:** the evidence can't tell, and the answer changes which sales or people count. A question whose answers would only rename something, or change a step's type or a label, is never asked: pick one yourself. ${brand.name} drops any question whose options all count the same sales and people.
+- **Your best guess is the default.** The map you save already holds it. Its option's \`patch\` is \`[]\`, and \`default_option\` is its index.
+- **One card per decision, at most 7.** ${brand.name} keeps the 7 that move the most money, and adds a yes/no card of its own for each weak part no card of yours covers.
+- \`question_id\`: lowercase letters, digits, \`-\` and \`_\`, such as \`flash_sale\`. Keep the same id for the same question on every run.
+- \`text\`: the question in plain words, at most 200 characters.
+- \`kind\`: \`yes_no\` (exactly two options, labelled \`Yes\` then \`No\`) or \`pick_one\` (two to four options).
+- \`evidence\`: one to four lines from \`evidence.json\`, counts and amounts only, such as "728 people paid $497 last year". Never a name, email or phone number.
+- \`options\`: each \`{"label", "patch"}\`. A patch is JSON Patch ops, \`add\`, \`remove\` or \`replace\`, on the map you save. Name offers, funnels and steps by key, not by index, so the patch still fits after the map changes: \`/offers/course/matches/-\` adds a match to the course offer, \`/funnels/webinar_to_course/steps/upsell_1/offer_key\` is that step's offer.
+- **Keep the answered ones.** Read \`questions.answered\` in \`current.json\`: keep each answer in your map, and never ask it again. ${brand.name} puts an answer back if your map drops it.
+
+A worked example, from DLC's first map. These were questions, one card each: whether the flash-sale buyers count as course buyers; whether VIP and guest tickets count as event tickets; whether "Financing - Protégé Mastery" payments are coaching sales; whether the strategy call also sells Protégé; whether other products such as 10K Club belong on the map. These were not: whether Sales Mastery is an order bump or an upsell page (only a step type), and how the webinar tags are named (only a label). The flash-sale card, with the guess that they are course buyers already in the map:
+
+\`\`\`json
+{
+  "question_id": "flash_sale",
+  "text": "Do the $497 flash-sale buyers count as course buyers?",
+  "kind": "yes_no",
+  "evidence": ["728 people paid $497 for Overages Blueprint 2.0 flash sale last year", "1,703 people paid $1,497 for the course"],
+  "options": [
+    {"label": "Yes", "patch": []},
+    {"label": "No", "patch": [{"op": "remove", "path": "/offers/course/matches/1"}]}
+  ],
+  "default_option": 0
+}
+\`\`\`
+
+A question whose patch does not apply, or gives a map that does not fit, is refused with its path, such as \`questions[2].options[1].patch\`. Fix it and save again.
+
+## 7. Save the map
 
 Write \`evidence-summary.json\` with what you read, in counts and ids only, such as \`{"buyer_paths": [{"purchases": [1500, 497, 3500], "people": 41}], "pages_read": ["<content item id>"], "webinars": ["<id>"]}\`. The note is one or two plain sentences, with no quotes or apostrophes: where the map came from and what a person should check. Then:
 
 \`\`\`sh
-${cli} funnels draft --file map.json --note "<the note>" --evidence @evidence-summary.json --based-on <version from step 1>
+${cli} funnels draft --file map.json --note "<the note>" --evidence @evidence-summary.json --questions @questions.json --counts-from fix --based-on <version from step 1>
 \`\`\`
 
-Leave out \`--based-on\` only when \`current.json\` has neither a map nor a draft. A map that does not fit is refused with every problem by path: fix each and save again. A \`draft_conflict\` means someone saved since you read: run \`${cli} funnels show --format json\` again and redo your change on top. Saving prints whether the map went live, what is left to fill in (such as a page rule with no address), and the weak parts: what the data does not back up, such as an offer with fewer than 3 buyers in the last year. A rule you cannot fill in from the evidence stays unfilled, and the map waits on hold; that is better than a guess.
+\`--counts-from\` says from when the map counts. Use \`fix\` when the data before today already shows the business working this way: the map then counts that way for every past date too. Use a date, \`--counts-from 2026-09-01\`, when the evidence for the change only starts recently, such as a new offer whose first sale was on 1 September: give the first date of that evidence, and the old map keeps counting the dates before it. When unsure, use \`fix\`. A draft that holds both a fix and a change is saved as two drafts: first the fix with \`--counts-from fix\`, then the change on top with its date, based on the version the first save made.
+
+Leave out \`--based-on\` only when \`current.json\` has neither a map nor a draft. A map that does not fit is refused with every problem by path: fix each and save again. A \`draft_conflict\` means someone saved since you read: run \`${cli} funnels show --format json\` again and redo your change on top. Saving prints whether the map went live, the answers it put back, the questions now open and those it left out, what is left to fill in (such as a page rule with no address), and the weak parts: what the data does not back up, such as an offer with fewer than 3 buyers in the last year. A rule you cannot fill in from the evidence stays unfilled, and the map waits on hold; that is better than a guess.
 
-## 7. Report
+## 8. Report
 
 Finish with a short report in plain words for the owner, who is not technical:
 
@@ -646,6 +680,8 @@ Finish with a short report in plain words for the owner, who is not technical:
 - what you mapped: the offer stack in one line each, and each funnel's steps;
 - offers found only in past campaigns, each with its campaign family and spend;
 - what you guessed, one line each: "Two payments of $497 and $297 follow the $1,500 one, usually within minutes; I named them Upsell 1 and Upsell 2.";
+- the questions now open, one line each, and any the save left out and why;
+- whether the map counts as a fix or from a date, and why;
 - what you could not tell, and why: a source not connected, a price nobody paid in the last year, a page with no ads;
 - the weak parts the save listed, one line each, in plain words;
 - that a person can undo the map in the app.
@@ -658,7 +694,7 @@ ${cli} funnels show [--format json]              the map now and the draft on ho
 ${cli} content show <sku | item-id>              a page's words or a video's transcript
 ${cli} content search --kind <kind> --text <words>
 ${cli} content add-page <url> [--format json]    read a landing page at a full address now
-${cli} funnels draft --file map.json [--note <why>] [--evidence @file.json] [--based-on <n>]
+${cli} funnels draft --file map.json [--note <why>] [--evidence @file.json] [--questions @file.json] [--counts-from fix|<YYYY-MM-DD>] [--based-on <n>]
 \`\`\`
 `;
 export const mapOffersSkill = `---

package/dist/pull.js

@@ -103,9 +103,9 @@ export function renderItem(item) {
         ["Subject", item.subject],
         ["Preview text", item.previewText],
         ["Headline", item.headline],
-        ["Body", item.body || null],
+        [item.transcripts.some((t) => t.mediaPosition === null) ? "Transcript" : "Body", item.body || null],
         ["Fields", Object.keys(item.fields).length === 0 ? null : `\`\`\`json\n${sortedJson(item.fields)}\n\`\`\``],
-        ...item.transcripts.map((t) => [`Transcript (file ${t.mediaPosition})`, t.text]),
+        ...item.transcripts.flatMap((t) => (t.mediaPosition === null ? [] : [[`Transcript (file ${t.mediaPosition})`, t.text]])),
     ];
     const body = sections.flatMap(([title, text]) => (text === null ? [] : [`## ${title}\n\n${text.trimEnd()}\n`]));
     return ["---", ...Object.entries(front).flatMap(([k, v]) => yamlLines(k, v, "")), "---", "", ...body.join("\n").split("\n")].join("\n").trimEnd() + "\n";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wrangled/cli",
-  "version": "0.3.2",
+  "version": "0.3.4",
   "description": "Command-line access to your marketing data warehouse, for people and coding agents",
   "type": "module",
   "bin": {