mulmoterminal 6.8.0 → 7.0.0

package/README.md

@@ -1296,6 +1296,18 @@ finder, the search or a clicked terminal path alike. Switching tabs saves the on
 opening another file does, so only the front tab can hold unsaved edits; each tab keeps its own place
 (Preview or editor, caret, scroll) and the whole strip is remembered with the rest of the pane. Close a
 tab with its **×**, a middle click, or **Delete** while it has focus; ←/→, Home and End move between them.
+The tree follows the tab in front, as VS Code's explorer does: its folders open and its row comes into
+view (a reload still puts the tree back where you left it).
+
+**The tree shows what git sees.** In a git repository a changed file is tinted and lettered as in VS
+Code's explorer — `M` modified, `A` added, `U` untracked, `R` renamed — and a folder holding changes
+carries a dot, so a collapsed tree still says where the agent has been. It is read when the tree
+loads or reloads, after a save or an outside change to the open file, and every thirty seconds.
+
+**The editor marks what changed since the last commit.** A bar beside each line that is new (green)
+or changed (amber), and a notch where lines were removed, as VS Code's gutter shows them — kept up
+to date as you type. **Changes** in the header also shows the removed lines in place, as a unified
+diff. Nothing is marked for a file outside git or one not yet committed.
 
 **HTML pages and images show in the pane too.** An `.html` file has a **Preview** like a Markdown
 file: the page itself, sandboxed as presentHtml pages are — its scripts run on an opaque origin with

package/blueprints/ask/checks/replies.mjs

@@ -1,7 +1,7 @@
 // Reads .blueprint/replies.json — an answer to each question the person asked — and answers one question
 // per mode. Exit 0 is yes.
 //   answer  every question has exactly one reply; a reply found in the documents quotes them (chaff cite)
-//           and names every address it quotes; a reply not found says what was searched. Records the
+//           and names every place it quotes (the address, or the heading of the section there); a reply not found says what was searched. Records the
 //           documents' and FAQ.md's fingerprints.
 //   keep    the documents are unchanged; FAQ.md holds every question after what it held before when the
 //           person asked to keep the answers, and is untouched otherwise
@@ -9,6 +9,7 @@ import { existsSync, readFileSync, writeFileSync } from "node:fs";
 import { fromBase } from "./base.mjs";
 const { fail, quotationProblems, readJson } = await import(fromBase("chaff.mjs"));
 const { digest, documentSource, documentsNamed, fingerprint } = await import(fromBase("documents.mjs"));
+const { headingsOf, namesPlace } = await import(fromBase("places.mjs"));
 
 const REPLIES = ".blueprint/replies.json";
 const REPLIES_PAGE = ".blueprint/replies.md";
@@ -29,6 +30,15 @@ if (answers?.keep === KEEP && documents.includes(FAQ)) fail(`${FAQ} is one of th
 const indented = (line) => "  " + line;
 const nonEmpty = (value) => typeof value === "string" && value.trim() !== "";
 
+// Read once per document, and only for one of the documents: a source that is not is refused by the quotation check.
+const headingCache = new Map();
+const headingAt = (source, address) => {
+  const resolved = documentSource(documents)(source);
+  if (!resolved.path) return undefined;
+  if (!headingCache.has(resolved.path)) headingCache.set(resolved.path, headingsOf(resolved.path));
+  return headingCache.get(resolved.path).get(address.trim());
+};
+
 const replyProblem = (reply) => {
   if (typeof reply !== "object" || reply === null) return "a reply is not an object";
   if (!nonEmpty(reply.question)) return "a reply names no question";
@@ -39,7 +49,10 @@ const replyProblem = (reply) => {
   if (!Array.isArray(citations)) return `${which}: citations must be an array`;
   if (reply.found && citations.length === 0) return `${which}: an answer found in the documents quotes them`;
   if (!citations.every((citation) => nonEmpty(citation?.address))) return `${which}: every quotation needs an "address"`;
-  const unnamed = citations.filter((citation) => !reply.answer.includes(citation.address.trim()));
+  const unnamed = citations.filter(
+    (citation) =>
+      !reply.answer.includes(citation.address.trim()) && !namesPlace(reply.answer, citation.address, headingAt(String(citation.source), citation.address)),
+  );
   if (unnamed.length > 0) return `${which}: the answer does not name ${unnamed.map((citation) => citation.address).join(", ")}, which it quotes`;
   if (!reply.found && (!Array.isArray(reply.searched) || !reply.searched.some(nonEmpty))) return `${which}: say what was searched ("searched")`;
   return null;

package/blueprints/ask/skills/answer/SKILL.md

@@ -48,11 +48,15 @@ the answers before anything is kept.
 - `question`: the question exactly as it is in `questions`. One reply per question, none extra.
 - `found`: `true` when the documents answer it. Then at least one citation.
 - `citations`: `source` is the document's path as `documents` names it; `address` is where; `quote` is
-  copied character for character — the check runs `chaff cite` on every one. **Write every cited address in
-  the `answer` text** so the person can open the document there.
+  copied character for character — the check runs `chaff cite` on every one. **Name every cited place in
+  the `answer` text** so the person can open the document there: an article or item as the document numbers it
+  (第4条第2項, Section 3.1), and a section of a Markdown document by its heading, word for word, in 「」 (「作業用フォルダを信頼しておく」;
+  in an English answer, "Getting started") — the check looks for it in those marks.
+  chaff's index for a section (`h1.3`) goes in `address` only — it means nothing to the person.
 - `searched`: when `found` is `false`, the words and places you looked for.
 
-Also write `.blueprint/replies.md` for the person: each question as a heading, the answer, and the quotations.
+Also write `.blueprint/replies.md` for the person: each question as a heading, the answer, and the quotations,
+each with its place named as in the answer (never `h1.3`).
 
 ## Done when
 

package/blueprints/ask/skills/keep/SKILL.md

@@ -17,7 +17,8 @@ run the check and stop.
 Otherwise add to `FAQ.md` in this folder (create it if there is none). **Add after what it already holds;
 never edit or reorder what is there** — the check compares the beginning of the file with what it held
 before. For each question: the question as a `##` heading, word for word; the answer; and where it is
-written (document and address). For a question the documents do not answer, say so — that is worth keeping
+written (the document, and the place named as in the answer: 第4条第2項, or a section's heading in 「」 —
+never chaff's `h1.3`). For a question the documents do not answer, say so — that is worth keeping
 too.
 
 ## Done when

package/blueprints/docs/checks/chaff.sh

@@ -4,4 +4,4 @@
 if [ -n "${CHAFF_BIN:-}" ]; then
   exec sh -c "$CHAFF_BIN \"\$@\"" chaff "$@"
 fi
-exec npx -y chaffjs@0.11 "$@"
+exec npx -y chaffjs@0.13 "$@"

package/blueprints/docs/checks/places.d.mts

@@ -0,0 +1,7 @@
+// Types for places.mjs, which the packs run with plain node and the specs import.
+export function headingsIn(tree: unknown): Map<string, string>;
+export function headingsOf(file: string): Map<string, string>;
+export function namesPlace(text: string, address: string, heading: string | undefined): boolean;
+export function placeNamesIn(tree: unknown): Map<string, string>;
+export function placeNamesOf(file: string): Map<string, string>;
+export function placeNamer(sourcePath: (source: string) => { path?: string; problem?: string }): (source: string, address: string) => string;

package/blueprints/docs/checks/places.mjs

@@ -0,0 +1,102 @@
+// Where a quotation sits, named for the person who reads it: a section of a Markdown document by its heading,
+// since chaff's address for it (h1.3) tells nobody anything. Other addresses (第4条第2項, 3.1) already read as the
+// document writes them.
+import { runChaff } from "./chaff.mjs";
+
+const sectionsIn = (node) => [
+  ...(node?.kind === "section" && typeof node.address === "string" && typeof node.attrs?.heading === "string" ? [[node.address, node.attrs.heading]] : []),
+  ...(Array.isArray(node?.children) ? node.children.flatMap(sectionsIn) : []),
+];
+
+const PARTS = new Set(["section", "article"]);
+
+const text = (value) => (typeof value === "string" && value !== "" ? value : null);
+
+// A Japanese article (第4条, 第二十一条の二): what is under it is named as Japanese law cites it.
+const JA_ARTICLE = /^第[^条\s]+条/u;
+const halfWidth = (digits) => digits.replace(/[０-９]/gu, (digit) => String.fromCharCode(digit.charCodeAt(0) - 0xfee0));
+
+// Under a Japanese article, one level down is a paragraph (第2項, numbered from the address when chaff gives it no
+// label) and anything deeper an item (第三号, by its label). chaff calls both "item", so the depth tells them apart.
+function japaneseName(label, own, above, article) {
+  const levels = own.slice(article.address.length + 1).split(".");
+  if (levels.length === 1) return `${above}第${halfWidth(label ?? levels[0])}項`;
+  return `${above}第${label ?? levels.at(-1)}号`;
+}
+
+// A part (an article, a section) is named by its label, or by its quoted heading when it has none (a Markdown
+// section); anything below it (an item, a paragraph) adds its own label to the name of what holds it.
+function nameOf(node, own, above, article) {
+  const label = text(node?.attrs?.label);
+  if (PARTS.has(node?.kind)) {
+    const heading = text(node?.attrs?.heading);
+    return label ?? (heading === null ? own : `「${heading}」`);
+  }
+  if (above === null) return label ?? own;
+  return article?.japanese && own.startsWith(`${article.address}.`) ? japaneseName(label, own, above, article) : `${above} ${label ?? own}`;
+}
+
+const partOf = (node, own, name, article) => (PARTS.has(node?.kind) && own !== null ? { address: own, japanese: JA_ARTICLE.test(name ?? "") } : article);
+
+const namesIn = (node, above, article) => {
+  const own = text(node?.address);
+  const name = own === null ? above : nameOf(node, own, above, article);
+  const here = own === null ? [] : [[own, name]];
+  const within = partOf(node, own, name, article);
+  return [...here, ...(Array.isArray(node?.children) ? node.children.flatMap((child) => namesIn(child, name, within)) : [])];
+};
+
+/** Every address in a chaff tree, with the name a person reads for it: 第4条第2項, Section 3.2 (a) (i), 「用意するもの」. */
+export const placeNamesIn = (tree) => new Map(namesIn(tree, null, null));
+
+/** The place names of `file`, by address; none when chaff cannot read its tree. */
+export const placeNamesOf = (file) => {
+  const run = runChaff(["tree", file, "--format", "json"]);
+  if (run.code !== 0) return new Map();
+  try {
+    return placeNamesIn(JSON.parse(run.stdout));
+  } catch {
+    return new Map();
+  }
+};
+
+/**
+ * Where a citation sits, for a view a person reads: its place name from the document's tree, read once per document
+ * and only for one of `documents` (through `sourcePath`, which resolves or refuses a source); the address itself
+ * when there is no name.
+ */
+export const placeNamer = (sourcePath) => {
+  const cache = new Map();
+  return (source, address) => {
+    const resolved = sourcePath(source);
+    if (!resolved.path) return address;
+    if (!cache.has(resolved.path)) cache.set(resolved.path, placeNamesOf(resolved.path));
+    return cache.get(resolved.path).get(String(address).trim()) ?? address;
+  };
+};
+
+/** Every section address in a chaff tree (`tree --format json`), with its heading. */
+export const headingsIn = (tree) => new Map(sectionsIn(tree));
+
+/** The section headings of `file`, by address; none when chaff cannot read its tree. */
+export const headingsOf = (file) => {
+  const run = runChaff(["tree", file, "--format", "json"]);
+  if (run.code !== 0) return new Map();
+  try {
+    return headingsIn(JSON.parse(run.stdout));
+  } catch {
+    return new Map();
+  }
+};
+
+// A heading counts only in quotation marks: a short one (期限) is also an ordinary word, and bare it would be found by accident.
+const QUOTES = [
+  ["「", "」"],
+  ['"', '"'],
+  ["“", "”"],
+];
+
+/** Whether `text` names a place: by its address as written, or by the quoted heading of the section at that address. */
+export const namesPlace = (text, address, heading) =>
+  text.includes(address.trim()) ||
+  (typeof heading === "string" && heading !== "" && QUOTES.some(([open, close]) => text.includes(`${open}${heading}${close}`)));

package/blueprints/docs/skills/workspace/SKILL.md

@@ -12,7 +12,7 @@ repository, `.git/info/exclude`) until the person has approved the models.
 1. `.blueprint/` already exists (the build created it). Everything this build records goes there.
 2. Run chaff once through the base pack's wrapper, so its first download happens now and not in the
    middle of a later step: `sh <base pack>/checks/chaff.sh rules --json > /dev/null`. The wrapper runs the
-   pinned chaff (`npx -y chaffjs@0.11`) unless `CHAFF_BIN` names another. If it fails, show the person the
+   pinned chaff (`npx -y chaffjs@0.13`) unless `CHAFF_BIN` names another. If it fails, show the person the
    error and stop — do not work around it.
 3. If the folder is a git repository, add `.blueprint/` to `.git/info/exclude`. That keeps the build's
    working files out of commits without touching `.gitignore`, which belongs to the person.

package/blueprints/firebase/checks/spec.sh

@@ -6,3 +6,7 @@ if grep -n '{{' .blueprint/spec.md >&2; then
   echo "unfilled placeholders are left in .blueprint/spec.md" >&2
   exit 1
 fi
+# A usecase may hold its spec to more than the template: when it ships its own spec check, that runs too.
+if [ -n "${BLUEPRINT_USECASE:-}" ] && [ -f "$BLUEPRINT_USECASE/checks/spec.sh" ]; then
+  sh "$BLUEPRINT_USECASE/checks/spec.sh"
+fi

package/blueprints/from-collection/checks/actions.sh

@@ -0,0 +1,48 @@
+#!/bin/sh
+# What the spec decided for the source's actions and ingests is done: each one to be built (`feature`) has a test that
+# names it and the tests pass; each one left to a person (`manual`) has its steps in the README. A source with no
+# actions or ingests has nothing to do.
+set -eu
+base="$1"
+file=.blueprint/actions.json
+# The record itself first: a build that got here without the spec check is held to it all the same.
+node --no-warnings "$(dirname "$0")/decisions.mjs"
+[ -s "$file" ] || { echo "no $file: the source has no actions or ingests to handle"; exit 0; }
+# A key for a model call lives in .env; the folder becomes a repository whenever its person adds git, and .env must not go with it.
+if [ -f .env ] && ! grep -qxE '/?\.env(\*)?' .gitignore 2>/dev/null; then
+  echo ".env exists and .gitignore does not ignore it; add a line .env to .gitignore before anything secret goes in" >&2
+  exit 1
+fi
+case "$base" in
+  local) tests="test/actions.test.ts" ;;
+  firebase) tests="test/blueprint/actions.spec.ts" ;;
+  *) echo "usage: actions.sh local|firebase" >&2; exit 2 ;;
+esac
+# The titles of the tests the file really declares — parsed, so a name in a comment or a string is not a test. Only read
+# when something is to be built.
+wanted=$(node -e 'process.stdout.write(String((JSON.parse(require("fs").readFileSync(process.argv[1], "utf8")).actions ?? []).filter((entry) => entry.decision === "feature").length))' "$file")
+titles="[]"
+[ "$wanted" -eq 0 ] || titles=$(node --no-warnings "$(dirname "$0")/test-titles.mjs" "$tests")
+features=$(node -e '
+const fs = require("node:fs");
+const [file, tests, titlesJson] = process.argv.slice(1);
+const entries = JSON.parse(fs.readFileSync(file, "utf8")).actions ?? [];
+const titles = JSON.parse(titlesJson);
+const readme = fs.existsSync("README.md") ? fs.readFileSync("README.md", "utf8") : "";
+// A name counts only where it does its job: in the title of a test the file declares, or in a README heading.
+const escaped = (name) => name.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+const inTestTitle = (name) => titles.some((title) => title.includes(name));
+const inHeading = (name) => new RegExp(`^#{1,6}\\s.*${escaped(name)}`, "m").test(readme);
+const problems = entries.flatMap(({ name, decision }) => {
+  if (decision === "feature" && !inTestTitle(name)) return [`${name} is to be built, and ${tests} has no test titled with it`];
+  if (decision === "manual" && !inHeading(name)) return [`${name} is left to a person, and README.md has no heading naming it`];
+  return [];
+});
+if (problems.length > 0) { console.error(problems.join("\n")); process.exit(1); }
+process.stdout.write(String(entries.filter((entry) => entry.decision === "feature").length));
+' "$file" "$tests" "$titles")
+[ "$features" -gt 0 ] || exit 0
+case "$base" in
+  local) sh "$BLUEPRINT_BASE/checks/tests-pass.sh" actions ;;
+  firebase) sh "$BLUEPRINT_BASE/checks/emulator-test.sh" actions ;;
+esac

package/blueprints/from-collection/checks/compare.mjs

@@ -0,0 +1,42 @@
+// What the import checks share: the copied source, which fields are compared, and how a stored value is held against
+// the source's. Each check reads its own store (SQLite, Firestore) and hands the values here.
+import { readFileSync } from "node:fs";
+
+export const SOURCE = ".blueprint/source";
+const MAX_REPORTED = 20;
+// The field kinds whose value is kept as it is, under the field's key (spec/conversion.md).
+const PLAIN = new Set(["string", "text", "email", "markdown", "number", "date", "datetime", "enum", "ref", "image", "file", "boolean"]);
+
+const readJson = (file) => JSON.parse(readFileSync(file, "utf8"));
+
+export const sourceCollections = () => readJson(`${SOURCE}/source.json`).collections;
+export const schemaOf = (slug) => readJson(`${SOURCE}/collections/${slug}/schema.json`);
+export const recordsOf = (slug) =>
+  readFileSync(`${SOURCE}/collections/${slug}/records.jsonl`, "utf8")
+    .split("\n")
+    .filter((line) => line.trim() !== "")
+    .map((line) => JSON.parse(line));
+
+export const plainFields = (schema) => Object.entries(schema.fields ?? {}).filter(([, spec]) => PLAIN.has(spec.type));
+export const isFileField = (spec) => spec.type === "image" || spec.type === "file";
+// A field the record left empty has nothing to carry over.
+export const hasValue = (value) => value !== undefined && value !== null && value !== "";
+export const labelOf = (schema, record) => `${schema.primaryKey}=${JSON.stringify(record[schema.primaryKey])}`;
+
+/**
+ * Whether a stored value is the source's. `booleanAs` says how the store holds a boolean — 0/1 in SQLite, itself in
+ * Firestore; a number is compared as a number, anything else as its text.
+ */
+export function sameValue(type, sourceValue, stored, booleanAs) {
+  if (type === "number") return Number(sourceValue) === Number(stored);
+  const want = type === "boolean" ? booleanAs(sourceValue === true) : sourceValue;
+  return String(want) === String(stored);
+}
+
+/** Prints the problems, at most a screenful, and exits 1 when there are any. */
+export function report(problems) {
+  if (problems.length === 0) return;
+  console.error(problems.slice(0, MAX_REPORTED).join("\n"));
+  if (problems.length > MAX_REPORTED) console.error(`… and ${problems.length - MAX_REPORTED} more`);
+  process.exit(1);
+}

package/blueprints/from-collection/checks/decisions.mjs

@@ -0,0 +1,69 @@
+// What was decided for the source's actions and ingests, held against the source: every action (`actions`,
+// `collectionActions`) and ingest has exactly one entry in .blueprint/actions.json, under its own name and kind, with a
+// decision the actions step can act on — and a mutate is always built. A source with neither needs no file. Both the
+// spec check and the actions check run this, so a build that reaches the actions step without a spec check behind it
+// is held to the same record. Prints each problem, and exits 1 when there is any.
+import { existsSync, readFileSync } from "node:fs";
+
+const SOURCE = ".blueprint/source";
+const FILE = ".blueprint/actions.json";
+const DECISIONS = new Set(["feature", "manual", "drop"]);
+
+const readJson = (file) => JSON.parse(readFileSync(file, "utf8"));
+
+// What the source has to be decided on: each action and ingest, by the name the spec uses, with the kind it was.
+function wanted() {
+  return readJson(`${SOURCE}/source.json`).collections.flatMap((slug) => {
+    const schema = readJson(`${SOURCE}/collections/${slug}/schema.json`);
+    return [
+      ...[...(schema.actions ?? []), ...(schema.collectionActions ?? [])].map((action) => ({ name: `${slug}.actions.${action.id}`, kind: action.kind })),
+      ...(schema.ingest ? [{ name: `${slug}.ingest`, kind: schema.ingest.kind }] : []),
+    ];
+  });
+}
+
+// The recorded entries, or null when the file is not JSON with an "actions" list.
+function recorded() {
+  try {
+    const parsed = readJson(FILE);
+    return Array.isArray(parsed?.actions) ? parsed.actions : null;
+  } catch {
+    return null;
+  }
+}
+
+function entryProblems({ name, kind }, found) {
+  if (found.length !== 1) return [`${FILE}: ${name} has ${found.length} entries, it needs one`];
+  const [entry] = found;
+  if (entry.kind !== kind) return [`${FILE}: ${name} is recorded as kind ${JSON.stringify(entry.kind)}; the source has it as ${JSON.stringify(kind)}`];
+  if (!DECISIONS.has(entry.decision)) return [`${FILE}: ${name} has decision ${JSON.stringify(entry.decision)}; it is feature, manual or drop`];
+  return kind === "mutate" && entry.decision !== "feature" ? [`${FILE}: ${name} is a mutate, which is always built (feature)`] : [];
+}
+
+function problems() {
+  const items = wanted();
+  if (items.length === 0) return [];
+  // A copy is taken from a schema the collection engine accepted, which always names a kind; one without it is not a
+  // source this can judge — above all, not whether it is a mutate that has to be built.
+  const kindless = items.filter((item) => typeof item.kind !== "string" || item.kind === "");
+  if (kindless.length > 0) return kindless.map((item) => `${SOURCE}: ${item.name} has no kind, so what it was cannot be decided on`);
+  if (!existsSync(FILE)) return [`${FILE} is missing; it records how each of the ${items.length} actions and ingests is handled`];
+  const entries = recorded();
+  if (entries === null) return [`${FILE} is not JSON with an "actions" list`];
+  const known = new Set(items.map((item) => item.name));
+  return [
+    ...items.flatMap((item) =>
+      entryProblems(
+        item,
+        entries.filter((entry) => entry?.name === item.name),
+      ),
+    ),
+    ...entries.filter((entry) => !known.has(entry?.name)).map((entry) => `${FILE}: ${JSON.stringify(entry?.name)} is not an action or ingest of the source`),
+  ];
+}
+
+const found = problems();
+if (found.length > 0) {
+  console.error(found.join("\n"));
+  process.exit(1);
+}

package/blueprints/from-collection/checks/firestore-verify.mjs

@@ -0,0 +1,115 @@
+// Reads Firestore (and Cloud Storage) after `yarn import-source` and holds it against the copied source, over the REST
+// APIs rather than the app's own code: every collection has as many documents as it had records, every stored field of
+// every record reads back equal, and every file a record points at is in the bucket. Prints each mismatch, and exits 1
+// when there is any.
+//
+//   node firestore-verify.mjs emulator   inside `firebase emulators:exec` (FIRESTORE_EMULATOR_HOST, GCLOUD_PROJECT)
+//   node firestore-verify.mjs prod       with PROJECT_ID and ACCESS_TOKEN (gcloud auth print-access-token)
+//
+// The bucket the files went to is named in .blueprint/storage-bucket; in production it must be one of the project's.
+import { existsSync, readFileSync } from "node:fs";
+import { SOURCE, hasValue, isFileField, labelOf, plainFields, recordsOf, report, sameValue, schemaOf, sourceCollections } from "./compare.mjs";
+
+const PAGE_SIZE = 300;
+const TIMEOUT_MS = 20000;
+const target = process.argv[2];
+
+function endpoints() {
+  if (target === "emulator") {
+    const project = process.env.GCLOUD_PROJECT || "demo-blueprint";
+    return {
+      documents: `http://${process.env.FIRESTORE_EMULATOR_HOST}/v1/projects/${project}/databases/(default)/documents`,
+      object: (bucket, file) => `http://${process.env.FIREBASE_STORAGE_EMULATOR_HOST}/v0/b/${bucket}/o/${encodeURIComponent(file)}`,
+      headers: { Authorization: "Bearer owner" },
+      bucketAllowed: () => true,
+    };
+  }
+  const project = process.env.PROJECT_ID;
+  return {
+    documents: `https://firestore.googleapis.com/v1/projects/${project}/databases/(default)/documents`,
+    object: (bucket, file) => `https://storage.googleapis.com/storage/v1/b/${bucket}/o/${encodeURIComponent(file)}`,
+    headers: { Authorization: `Bearer ${process.env.ACCESS_TOKEN}` },
+    bucketAllowed: (bucket) => bucket === `${project}.appspot.com` || bucket === `${project}.firebasestorage.app`,
+  };
+}
+
+async function get(url, headers) {
+  const response = await fetch(url, { headers, signal: AbortSignal.timeout(TIMEOUT_MS) });
+  return { status: response.status, body: response.ok ? await response.json() : null };
+}
+
+// A Firestore REST value as the plain value it holds.
+function decode(value) {
+  if ("stringValue" in value) return value.stringValue;
+  if ("integerValue" in value) return Number(value.integerValue);
+  if ("doubleValue" in value) return value.doubleValue;
+  if ("booleanValue" in value) return value.booleanValue;
+  if ("timestampValue" in value) return value.timestampValue;
+  if ("referenceValue" in value) return value.referenceValue.split("/").at(-1);
+  if ("nullValue" in value) return null;
+  return value;
+}
+
+async function documentsOf(api, collection, pageToken = "") {
+  const query = new URLSearchParams({ pageSize: String(PAGE_SIZE), ...(pageToken ? { pageToken } : {}) });
+  const { status, body } = await get(`${api.documents}/${encodeURIComponent(collection)}?${query}`, api.headers);
+  if (status !== 200) throw new Error(`reading ${collection} from Firestore answered ${status}`);
+  const page = (body.documents ?? []).map((doc) => ({
+    id: doc.name.split("/").at(-1),
+    fields: Object.fromEntries(Object.entries(doc.fields ?? {}).map(([key, value]) => [key, decode(value)])),
+  }));
+  return body.nextPageToken ? [...page, ...(await documentsOf(api, collection, body.nextPageToken))] : page;
+}
+
+function fieldProblems(slug, key, spec, record, doc, label) {
+  const value = record[key];
+  if (!hasValue(value)) return [];
+  if (!(key in doc.fields)) return [`${slug} ${label}: the document has no field ${key}`];
+  return sameValue(spec.type, value, doc.fields[key], (flag) => flag)
+    ? []
+    : [`${slug} ${label}: ${key} is ${JSON.stringify(doc.fields[key])}, the source had ${JSON.stringify(value)}`];
+}
+
+async function collectionProblems(api, slug) {
+  const schema = schemaOf(slug);
+  const records = recordsOf(slug);
+  const docs = new Map((await documentsOf(api, slug)).map((doc) => [doc.id, doc]));
+  const counted = docs.size === records.length ? [] : [`${slug}: Firestore has ${docs.size} documents, the source had ${records.length} records`];
+  // The primary key is the document id, matched here; the document need not repeat it as a field.
+  const fields = plainFields(schema).filter(([key]) => key !== schema.primaryKey);
+  const perRecord = records.flatMap((record) => {
+    const doc = docs.get(String(record[schema.primaryKey]));
+    if (!doc) return [`${slug}: no document ${JSON.stringify(record[schema.primaryKey])}`];
+    return fields.flatMap(([key, spec]) => fieldProblems(slug, key, spec, record, doc, labelOf(schema, record)));
+  });
+  return [...counted, ...perRecord];
+}
+
+// Every file the records point at and the source holds, once.
+const pointedAt = () => [
+  ...new Set(
+    sourceCollections().flatMap((slug) => {
+      const fileKeys = plainFields(schemaOf(slug))
+        .filter(([, spec]) => isFileField(spec))
+        .map(([key]) => key);
+      return recordsOf(slug).flatMap((record) =>
+        fileKeys.map((key) => record[key]).filter((value) => hasValue(value) && existsSync(`${SOURCE}/files/${value}`)),
+      );
+    }),
+  ),
+];
+
+async function fileProblems(api) {
+  const files = pointedAt();
+  if (files.length === 0) return [];
+  const bucketFile = ".blueprint/storage-bucket";
+  if (!existsSync(bucketFile)) return [`the records point at ${files.length} files, and ${bucketFile} does not name the bucket they went to`];
+  const bucket = readFileSync(bucketFile, "utf8").trim();
+  if (!api.bucketAllowed(bucket)) return [`${bucketFile} names ${bucket}, which is not this project's bucket`];
+  const found = await Promise.all(files.map(async (file) => (await get(api.object(bucket, file), api.headers)).status === 200));
+  return files.filter((_file, index) => !found[index]).map((file) => `${file} is not in the bucket ${bucket}`);
+}
+
+const api = endpoints();
+const problems = [...(await Promise.all(sourceCollections().map((slug) => collectionProblems(api, slug)))).flat(), ...(await fileProblems(api))];
+report(problems);

package/blueprints/from-collection/checks/import-firebase.sh

@@ -0,0 +1,36 @@
+#!/bin/sh
+# The records the build copied are in Firestore, and the files they point at in Cloud Storage — read back over the REST
+# APIs, not through the app. `emulator`: `yarn import-source --target emulator` runs twice inside the emulators, then the
+# emulators are read. `prod`: production is read with the person's gcloud token (the step itself did the import, after
+# its approval). A build that copied only the shape has nothing to move.
+set -eu
+target="$1"
+here="$(dirname "$0")"
+source=.blueprint/source/source.json
+[ -s "$source" ] || { echo "missing $source; the build was started without a collection" >&2; exit 1; }
+if [ "$(node -e 'process.stdout.write(String(JSON.parse(require("fs").readFileSync(process.argv[1], "utf8")).records === true))' "$source")" != true ]; then
+  echo "only the shape was copied; there are no records to move"
+  exit 0
+fi
+grep -q '"import-source"' package.json || { echo "package.json has no import-source script" >&2; exit 1; }
+verify="node --no-warnings $here/firestore-verify.mjs"
+case "$target" in
+  emulator)
+    . "$BLUEPRINT_BASE/checks/java21.sh"
+    only="firestore"
+    [ ! -d .blueprint/source/files ] || only="firestore,storage"
+    # Twice: moving the records again must leave the same documents, not a second copy of each.
+    firebase emulators:exec --project demo-blueprint --only "$only" \
+      "yarn -s import-source --target emulator && yarn -s import-source --target emulator && $verify emulator"
+    ;;
+  prod)
+    # Taken one at a time: a failure inside an assignment that prefixes a command would not stop `set -e`.
+    prod=$(sh "$BLUEPRINT_BASE/checks/project-id.sh" prod)
+    token=$(gcloud auth print-access-token)
+    PROJECT_ID="$prod" ACCESS_TOKEN="$token" $verify prod
+    ;;
+  *)
+    echo "usage: import-firebase.sh emulator|prod" >&2
+    exit 2
+    ;;
+esac

package/blueprints/from-collection/checks/import-verify.mjs

@@ -0,0 +1,45 @@
+// Reads a database `yarn import-source` filled and holds it against the copied source: every collection's table has as
+// many rows as it had records, every stored field of every record reads back equal, and every file a record points at
+// is in data/files/. Prints each mismatch, and exits 1 when there is any.
+import { existsSync } from "node:fs";
+import { DatabaseSync } from "node:sqlite";
+import { SOURCE, hasValue, isFileField, labelOf, plainFields, recordsOf, report, sameValue, schemaOf, sourceCollections } from "./compare.mjs";
+
+const tableOf = (slug) => slug.replaceAll("-", "_");
+const quoted = (name) => `"${name.replaceAll('"', '""')}"`;
+const asColumn = (flag) => (flag ? 1 : 0);
+
+function fieldProblems(slug, key, spec, record, row, label) {
+  const value = record[key];
+  if (!hasValue(value)) return [];
+  if (!(key in row)) return [`${slug}: table ${tableOf(slug)} has no column ${key}`];
+  const problems = sameValue(spec.type, value, row[key], asColumn)
+    ? []
+    : [`${slug} ${label}: ${key} is ${JSON.stringify(row[key])}, the source had ${JSON.stringify(value)}`];
+  const missingFile = isFileField(spec) && existsSync(`${SOURCE}/files/${value}`) && !existsSync(`data/files/${value}`);
+  return missingFile ? [...problems, `${slug} ${label}: ${key} points at ${value}, which is not in data/files/`] : problems;
+}
+
+function collectionProblems(db, slug) {
+  const schema = schemaOf(slug);
+  const records = recordsOf(slug);
+  const table = tableOf(slug);
+  const exists = db.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?").get(table);
+  if (!exists) return [`${slug}: there is no table ${table}`];
+  const count = db.prepare(`SELECT COUNT(*) AS n FROM ${quoted(table)}`).get().n;
+  const counted = count === records.length ? [] : [`${slug}: table ${table} has ${count} rows, the source had ${records.length} records`];
+  const primary = schema.primaryKey;
+  const fields = plainFields(schema);
+  const find = db.prepare(`SELECT * FROM ${quoted(table)} WHERE ${quoted(primary)} = ?`);
+  const perRecord = records.flatMap((record) => {
+    const row = find.get(record[primary]);
+    if (!row) return [`${slug}: no row for ${primary} = ${JSON.stringify(record[primary])}`];
+    return fields.flatMap(([key, spec]) => fieldProblems(slug, key, spec, record, row, labelOf(schema, record)));
+  });
+  return [...counted, ...perRecord];
+}
+
+const db = new DatabaseSync(process.argv[2], { readOnly: true });
+const problems = sourceCollections().flatMap((slug) => collectionProblems(db, slug));
+db.close();
+report(problems);

package/blueprints/from-collection/checks/import.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+# The records the build copied are in the app: `yarn import-source` is run twice into a fresh database, and the
+# database is then held against the source by reading it directly — the import's own word is not taken. A build that
+# copied only the shape has nothing to move.
+set -eu
+source=.blueprint/source/source.json
+[ -s "$source" ] || { echo "missing $source; the build was started without a collection" >&2; exit 1; }
+if [ "$(node -e 'process.stdout.write(String(JSON.parse(require("fs").readFileSync(process.argv[1], "utf8")).records === true))' "$source")" != true ]; then
+  echo "only the shape was copied; there are no records to move"
+  exit 0
+fi
+grep -q '"import-source"' package.json || { echo "package.json has no import-source script" >&2; exit 1; }
+work=$(mktemp -d)
+trap 'rm -rf "$work"' EXIT INT TERM
+db="$work/app.db"
+yarn -s import-source "$db" >/dev/null || { echo "yarn import-source failed" >&2; yarn -s import-source "$db" >&2 || true; exit 1; }
+# Twice: moving the records again must leave the same rows, not a second copy of each.
+yarn -s import-source "$db" >/dev/null || { echo "yarn import-source failed the second time over the same database" >&2; exit 1; }
+node --no-warnings "$(dirname "$0")/import-verify.mjs" "$db"
+yarn test >/dev/null || { echo "yarn test fails" >&2; yarn test >&2 || true; exit 1; }