@haystackeditor/cli 0.29.0 → 0.30.2

package/README.md

@@ -118,7 +118,9 @@ answered or finished (the bugs
 it found are in the output) or is still running under `--no-wait`; 2 when it ended
 without finishing (stopped early, or cancelled) or the machines it used could
 not be proven shut down;
-1 when the command failed or no crawl could be started.
+1 when the command failed or no crawl could be started; 3 when onboarding is
+blocked; 4 for the handshake (run it again with `--intent`); 5 when the app
+copy is broken and the change could not be tested.
 
 **The stop hook.** `haystack init` installs a Claude Code Stop hook that runs
 `haystack verify precompute --hook` whenever an agent turn stops. It captures

package/dist/commands/crawl-contract.js

@@ -15,6 +15,9 @@ export const CRAWL_WAIT_MAX_MS = 25_000;
  * (how many times the copy's calls reached the stand-in across the crawl's preparation and all its replays, not user actions). */
 export const CRAWL_MAX_STAND_IN_ROWS = 200;
 export const CRAWL_MAX_STAND_IN_LINES = 20;
+/** Amendment 22: each health evidence list holds at most this many lines, and the commonest difference lines at most this many. */
+export const CRAWL_HEALTH_MAX_LINES = 40;
+export const CRAWL_HEALTH_MAX_DIFFERENCES = 8;
 /** Amendment 18: the brief's bounds. */
 export const CRAWL_BRIEF_MAX_FUNCTIONS = 2000;
 export const CRAWL_BRIEF_MAX_GROUPS = 2000;

package/dist/commands/crawl-report.js

@@ -11,7 +11,7 @@ export const STATUS_WORDS = {
     cancelled: 'cancelled',
 };
 export const STAGE_WORDS = {
-    reconcile: 'cleaning up after an earlier try',
+    reconcile: 'checking for leftovers of an earlier try',
     source: 'rebuilding your change',
     worlds: 'building and starting the app with and without your change',
     blast: 'waiting for the analysis of what your change touches',
@@ -66,6 +66,38 @@ export function currentStep(view) {
 export function results(view) {
     return view.manifest ?? (view.answer?.status === 'answered' ? view.answer : null);
 }
+/** CRAWL-V1 amendment 22: the copy of the app the crawl ran was itself broken, so it tested nothing; null when it was not judged
+ * broken (or the results carry no health). */
+export function brokenCopy(view) {
+    const health = results(view)?.health;
+    return health !== undefined && health.status === 'app-broken' ? health : null;
+}
+/** What `haystack verify` says first of a broken copy. */
+export const BROKEN_COPY_HEADLINE = "We couldn't test your change: the app copy is broken.";
+/** What the coding agent does about a broken copy: the app's own setup says what the errors need; until a broken copy updates its
+ * setup (onboarding amendment 38 updates it only from a check that could not start the app), that is sent to the Haystack team
+ * (`haystack feedback`), and the user is told the change was not tested. */
+export const BROKEN_COPY_STEPS = [
+    "Find what these errors point to in the repository's own setup (docker compose files, env examples, docs): the services, settings and data the app needs to run.",
+    'Send what you found with `haystack feedback "<what the app needs to run>"`: a copy that starts but is broken does not update its setup yet.',
+    'Tell your user this check did not test the change; do not read it as passing.',
+];
+/** Text that reads as an error, for the lines a broken copy's start screen shows (every one is in --raw). */
+const ERROR_LIKE = /\b(?:errors?|fail(?:ed|s|ure)?|unsupported|exception|denied|refused|unavailable|unauthori[sz]ed|forbidden|invalid|cannot|can't|could not|unable|timed out|timeout|not found|internal server)\b/iu;
+/** What a broken copy's evidence says, as lines: what its start screen shows (its messages, then its texts that read as errors),
+ * and the requests that failed and the console errors logged with nobody clicking, each with the builds it was in. */
+export function brokenCopyEvidence(health) {
+    const { evidence } = health;
+    const shows = [...new Set([...evidence.messages, ...evidence.texts.filter(text => ERROR_LIKE.test(text))])];
+    const idle = (kind) => {
+        const of = (lines) => lines.filter(line => line.startsWith(`${kind}:`)).map(line => line.slice(kind.length + 1));
+        const old = of(evidence.background.old);
+        const fresh = of(evidence.background.new);
+        return [...new Set([...old, ...fresh])].map(entry => `${entry} (${old.includes(entry) && fresh.includes(entry) ? 'both builds'
+            : old.includes(entry) ? 'without your change only' : 'with your change only'})`);
+    };
+    return { shows, failedRequests: idle('failed'), consoleErrors: idle('errors') };
+}
 /** The status the command reports: a running crawl that has answered (amendment 9: its budget is up and its work is
  * done; only its shutdown remains) reads as completed. */
 export function reportedStatus(view) {
@@ -76,9 +108,21 @@ export function headline(view) {
     const found = bugs > 0 ? ` It found ${plural(bugs, 'bug')} before it stopped.` : '';
     switch (reportedStatus(view)) {
         case 'completed':
+            // Amendment 22: a broken copy tested nothing, whatever it found.
+            if (brokenCopy(view) !== null)
+                return BROKEN_COPY_HEADLINE;
             return bugs > 0 ? `Found ${plural(bugs, 'bug')} in your change.` : 'No bugs found in your change.';
         case 'incomplete': {
             const error = results(view)?.error ?? view.error;
+            // Onboarding rule 18 (amendment 38): which world did not start says whose it is, and what Haystack does about it.
+            if (error?.code === 'world-build-failed' && view.worlds.base === null) {
+                return 'The crawl could not finish: the app did not build or start at the base commit, on its setup. Haystack updates the '
+                    + 'setup for it and runs this check again (`haystack verify onboarding` shows the update).';
+            }
+            if (error?.code === 'world-build-failed' && view.worlds.head === null) {
+                return 'The crawl could not finish: your change did not build or start, though the base did. Haystack checks whether the change '
+                    + 'needs something new in the setup and, if it does, adds it and runs this check again (`haystack verify onboarding` shows it).';
+            }
             return `The crawl could not finish: ${error ? errorWords(error) : 'no reason was recorded'}.${found}`;
         }
         case 'prepared':
@@ -137,6 +181,9 @@ export function verifyReport(view) {
         } : null,
         notFinishedInTime: explored ? published.notFinishedInTime : 0,
         outsideServiceGaps: explored ? published.standIns?.gaps ?? [] : [],
+        health: !explored || published.health === undefined ? null : published.health.status === 'app-broken'
+            ? { status: 'app-broken', reason: published.health.reason, ...brokenCopyEvidence(published.health), whatToDo: [...BROKEN_COPY_STEPS] }
+            : { status: published.health.status, reason: published.health.reason, shows: [], failedRequests: [], consoleErrors: [], whatToDo: null },
         error: error ? { code: error.code, message: error.message } : null,
         machinesNotProvenShutDown: TERMINAL.has(view.status) ? view.totals.cleanupUnproven : 0,
     };
@@ -145,6 +192,12 @@ export function verifyReport(view) {
 export function reportMarkdown(report) {
     const lines = [`# ${report.headline}`, '',
         `Change: ${report.title}`, `Check ${report.runId} of ${report.repository}: base ${report.baseCommit.slice(0, 12)}, change ${report.workCommit.slice(0, 12)}.`];
+    // Amendment 22: a broken copy is said first, with why and what to do, whatever became of the run after it answered (a cancelled
+    // or failed run's published health still says it); its findings follow, marked.
+    const broken = report.health?.status === 'app-broken' ? report.health : null;
+    if (broken !== null) {
+        lines.splice(1, 0, '', broken.reason, ...(broken.shows.length ? ['', 'The start screen shows:', ...broken.shows.map(line => `- ${line}`)] : []), ...(broken.failedRequests.length ? ['', 'Requests that failed with nobody clicking:', ...broken.failedRequests.map(line => `- ${line}`)] : []), ...(broken.consoleErrors.length ? ['', 'Console errors with nobody clicking:', ...broken.consoleErrors.map(line => `- ${line}`)] : []), '', '## What to do', ...(broken.whatToDo ?? []).map((step, index) => `${index + 1}. ${step}`));
+    }
     if (report.error && report.status !== 'completed')
         lines.push(`Details: ${report.error.message}`);
     if (report.neverRan === null) {
@@ -161,7 +214,9 @@ export function reportMarkdown(report) {
         if (spot.reachedBy)
             lines.push(`  - reached by: ${[spot.reachedBy.start, ...spot.reachedBy.steps].join(' > ')}${spot.reachedBy.page ? ` (page ${spot.reachedBy.page})` : ''}`);
     }
-    lines.push('', '## What the crawl found');
+    lines.push('', broken === null ? '## What the crawl found' : '## What the crawl found in the broken copy');
+    if (broken !== null)
+        lines.push('The copy was broken, so these differences are the environment\'s, not evidence about the change.');
     if (report.findings.length === 0)
         lines.push('No confirmed differences between the app with and without the change.');
     report.findings.forEach((finding, index) => {

package/dist/commands/verify-onboarding.js

@@ -113,6 +113,23 @@ export function parseOnboardingStatus(value, expected) {
     if (ready ? !(typeof value.recordPublishedAt === 'string' && !Number.isNaN(Date.parse(value.recordPublishedAt)))
         : value.recordPublishedAt !== null)
         invalid('its record time');
+    // Rule 18 (amendment 38): a ready record says what it updated, and the update of it under way, if any.
+    const recordUpdate = value.recordUpdate;
+    if (!(recordUpdate === null || (ready && isRecord(recordUpdate) && typeof recordUpdate.of === 'string' && SHA256.test(recordUpdate.of)
+        && typeof recordUpdate.crawlRunId === 'string' && (recordUpdate.side === 'base' || recordUpdate.side === 'head')
+        && typeof recordUpdate.workCommit === 'string' && COMMIT.test(recordUpdate.workCommit)
+        && Array.isArray(recordUpdate.changes) && recordUpdate.changes.every(change => isRecord(change) && typeof change.field === 'string'
+        && (change.name === null || typeof change.name === 'string') && (change.kind === 'added' || change.kind === 'changed')
+        && Array.isArray(change.evidence) && change.evidence.every(id => typeof id === 'string')))))
+        invalid('its record update');
+    const update = value.update;
+    if (!(update === null || (ready && isRecord(update) && typeof update.onboardRunId === 'string' && ONBOARD_RUN_ID.test(update.onboardRunId)
+        && (update.side === 'base' || update.side === 'head') && typeof update.crawlRunId === 'string'
+        && (update.state === 'updating' || update.state === 'blocked' || update.state === 'stopped')
+        && (update.state === 'blocked') === (update.block !== null))))
+        invalid('its update');
+    if (isRecord(update) && update.block !== null)
+        parseOnboardingBlock(update.block);
     if (value.block !== null)
         parseOnboardingBlock(value.block);
     const checkpoint = value.checkpoint;
@@ -313,6 +330,9 @@ export function formatOnboarding(status) {
         lines.push('', ...stages);
     if (status.block)
         lines.push('', ...blockLines(status.block));
+    const updates = updateLines(status);
+    if (updates.length)
+        lines.push('', ...updates);
     if (status.dataAbsences.length)
         lines.push('', ...status.dataAbsences.map(absence => chalk.yellow(`Note: ${safe(absence)}`)));
     if (status.versionChoices.length)
@@ -325,6 +345,37 @@ export function formatOnboarding(status) {
         lines.push('', ...unreviewed.map(note => chalk.yellow(`Note: ${safe(note)}`)));
     return lines.join('\n');
 }
+/** Rule 18 (amendment 38): what the ready setup's last update was, and the update of it under way or ended without one: one
+ * line each, and a blocked update's block (what it needs, and what to tell Haystack). */
+export function updateLines(status) {
+    const lines = [];
+    const failed = (side) => side === 'base' ? 'the app did not start at its base commit' : 'the change did not start';
+    if (status.recordUpdate) {
+        const changes = status.recordUpdate.changes.map(change => `${change.kind} ${change.field === 'env' ? 'variable' : change.field}`
+            + `${change.name === null ? '' : ` ${change.name}`}${change.evidence.length ? ` (from ${change.evidence.join(', ')})` : ''}`);
+        lines.push(`Its setup was updated in place after check ${status.recordUpdate.crawlRunId}, where ${failed(status.recordUpdate.side)}`
+            + `${changes.length ? `: ${changes.join('; ')}` : ''}.`);
+    }
+    const update = status.update;
+    if (update === null)
+        return lines;
+    if (update.state === 'updating') {
+        lines.push(`Updating the setup (${update.onboardRunId}): check ${update.crawlRunId} found ${failed(update.side)}. The check runs `
+            + 'again on the updated setup when it is ready.');
+    }
+    else if (update.state === 'stopped') {
+        lines.push(`The update of the setup for check ${update.crawlRunId} (${update.onboardRunId}) stopped before finishing; the next check `
+            + 'that fails the same way starts it again.');
+    }
+    else {
+        lines.push(update.side === 'head'
+            ? chalk.bold(`Finding: check ${update.crawlRunId}'s change does not start, and no change to the setup started it.`)
+            : chalk.bold(`Haystack could not update the setup for check ${update.crawlRunId}, where the app did not start at its base commit.`));
+        if (update.block)
+            lines.push(...blockLines(update.block));
+    }
+    return lines;
+}
 /** Rule 2 (amendment 34): the repository's one setup, which base it was made at and how long ago, in one sentence. */
 export function readyLine(status, now = Date.now()) {
     if (status.recordBaseCommit === null || status.recordPublishedAt === null)
@@ -339,12 +390,16 @@ function age(ms) {
         : [Math.floor(minutes / 1_440), 'day'];
     return count === 0 ? 'just now' : `${count} ${unit}${count === 1 ? '' : 's'} ago`;
 }
-/** Rule 17 (amendment 32): an onboarding the independent review skipped, as the sentence every result shows; none otherwise. */
+/** Rule 17 (amendments 32 and 39): an onboarding the independent review skipped, as the sentence every result shows, saying why
+ * for each reason the contract names (a reason this CLI does not know yet is still shown, as the reason the server gave); none
+ * otherwise. */
 export function reviewNotes(review) {
     if (review?.verdict !== 'skipped')
         return [];
-    return [`This onboarding was NOT reviewed: the independent review that checks the setup did not cheat is turned off for this `
-            + `team (${review.reason}).`];
+    const why = review.reason === 'internal-test-tenant' ? 'is turned off for this team'
+        : review.reason === 'internal-test-requester' ? 'is turned off for onboardings Haystack\'s own people or automation start'
+            : 'did not run';
+    return [`This onboarding was NOT reviewed: the independent review that checks the setup did not cheat ${why} (${review.reason}).`];
 }
 /** Rule 15(g), amendment 3: a stand-in gap as the sentence every result shows. */
 export function standInGapNotes(gaps) {

package/dist/commands/verify.js

@@ -29,9 +29,9 @@ import { findGitRoot } from '../utils/hooks.js';
 import { resolveAuthContext } from '../utils/auth.js';
 import { classifyHttpError, readWithRetries, SERVICE_SILENCE_LIMIT_MS, ServiceSilentError, transientReadFailure, repoApiPath, } from '../utils/haystack-api.js';
 import { GATEWAY_TIMEOUT_MS, gatewayFetch } from './case-batch.js';
-import { CRAWL_BRIEF_MAX_FUNCTIONS, CRAWL_BRIEF_MAX_FLOW_NAME_CHARS, CRAWL_BRIEF_MAX_GROUPS, CRAWL_BRIEF_MAX_PRODUCTION_ROWS, CRAWL_BUDGET_MAX_MS, CRAWL_BUDGET_MIN_MS, CRAWL_MAX_FINDINGS, CRAWL_MAX_FINDING_STEPS, CRAWL_MAX_IDEAS, CRAWL_MAX_STAND_IN_LINES, CRAWL_MAX_STAND_IN_ROWS, CRAWL_MAX_TEXT_CHARS, CRAWL_WAIT_MAX_MS, CRAWL_POOLS, } from './crawl-contract.js';
+import { CRAWL_BRIEF_MAX_FUNCTIONS, CRAWL_BRIEF_MAX_FLOW_NAME_CHARS, CRAWL_BRIEF_MAX_GROUPS, CRAWL_BRIEF_MAX_PRODUCTION_ROWS, CRAWL_BUDGET_MAX_MS, CRAWL_BUDGET_MIN_MS, CRAWL_HEALTH_MAX_DIFFERENCES, CRAWL_HEALTH_MAX_LINES, CRAWL_MAX_FINDINGS, CRAWL_MAX_FINDING_STEPS, CRAWL_MAX_IDEAS, CRAWL_MAX_STAND_IN_LINES, CRAWL_MAX_STAND_IN_ROWS, CRAWL_MAX_TEXT_CHARS, CRAWL_WAIT_MAX_MS, CRAWL_POOLS, } from './crawl-contract.js';
 import { captureCheckout, crawlUnavailableText, EXPLICIT_WALL_MS, normalModeFailure, postPrecomputeCapture, } from './verify-precompute.js';
-import { SPOT_WORDS, TERMINAL, VERDICT_WORDS, VERDICTS, currentStep, errorWords, headline, plural, reportedStatus, results, verifyReport, } from './crawl-report.js';
+import { BROKEN_COPY_STEPS, SPOT_WORDS, TERMINAL, VERDICT_WORDS, VERDICTS, brokenCopy, brokenCopyEvidence, currentStep, errorWords, headline, plural, reportedStatus, results, verifyReport, } from './crawl-report.js';
 import { formatOnboarding, onboardingExitCode, readOnboardingStatus, readyLine, reportOnboardingState, reviewNotes, standInGapNotes, waitForOnboarding, } from './verify-onboarding.js';
 import { formatCapture, preVerifyCapture, readCaptureWindows } from './capture-brief.js';
 import { flowDetail, flowSummaries } from './verify-flows.js';
@@ -183,9 +183,30 @@ function checkResults(value, view, what) {
         invalid('its ideas');
     if (value.brief !== undefined)
         checkBrief(value.brief);
+    if (value.health !== undefined)
+        checkHealth(value.health);
     if (value.error !== null)
         checkError(value.error, 'its error');
 }
+/** Amendment 22: the health of the app copy, as the worker bounds it: `not-judged` with its reason, or a judgment. */
+function checkHealth(value) {
+    const lines = (item) => isStrings(item, CRAWL_HEALTH_MAX_LINES);
+    if (!isRecord(value) || typeof value.reason !== 'string' || value.reason.length === 0)
+        invalid('its app health');
+    if (value.status === 'not-judged')
+        return;
+    const evidence = value.evidence;
+    const images = value.images;
+    if ((value.status !== 'ok' && value.status !== 'app-broken') || typeof value.working !== 'number' || !(value.working >= 0 && value.working <= 1)
+        || !isMember(value.tested, ['tested', 'app_broken', 'not_reached', 'unclear'])
+        || !isRecord(evidence) || !lines(evidence.texts) || !lines(evidence.messages) || !lines(evidence.overlays)
+        || !isRecord(evidence.background) || !lines(evidence.background.old) || !lines(evidence.background.new)
+        || !Array.isArray(evidence.commonDifferences) || evidence.commonDifferences.length > CRAWL_HEALTH_MAX_DIFFERENCES
+        || !evidence.commonDifferences.every(row => isRecord(row) && typeof row.line === 'string' && isCount(row.count))
+        || !isRecord(images) || (images.old !== null && typeof images.old !== 'string')
+        || (images.new !== null && typeof images.new !== 'string'))
+        invalid('its app health');
+}
 /** Amendment 18: a brief's shape and bounds, as the worker checks them. */
 function checkBrief(value) {
     const text = (item, empty = false) => typeof item === 'string' && (empty || item.length > 0) && [...item].length <= CRAWL_MAX_TEXT_CHARS;
@@ -388,6 +409,15 @@ function findingLines(finding, position) {
 export function formatCrawl(view) {
     const status = reportedStatus(view);
     const lines = [chalk.bold(headline(view))];
+    // Amendment 22: a broken copy tested nothing: said first (the headline), then why, what both builds showed, and what to do. Its
+    // findings still follow, marked as the broken copy's.
+    // Amendment 22: whatever became of the run after it answered; the exit code keeps the run's own status (crawlExitCode).
+    const broken = brokenCopy(view);
+    if (broken !== null) {
+        const { shows, failedRequests, consoleErrors } = brokenCopyEvidence(broken);
+        const listed = (title, items) => (items.length ? [`  ${title}`, ...items.map(item => `    ${safe(item)}`)] : []);
+        lines.push(`  ${safe(broken.reason)}`, ...listed('The start screen shows:', shows), ...listed('Requests that failed with nobody clicking:', failedRequests), ...listed('Console errors with nobody clicking:', consoleErrors), chalk.bold('  What to do:'), ...BROKEN_COPY_STEPS.map((step, index) => `    ${index + 1}. ${step}`), '');
+    }
     const published = results(view);
     const error = published?.error ?? view.error;
     if (error && status !== 'completed')
@@ -411,7 +441,9 @@ export function formatCrawl(view) {
         lines.push('  No changed spots were recorded.');
     for (const spot of published.reach.spots)
         lines.push(...spotLines(spot));
-    lines.push('', chalk.bold('What the crawl found'));
+    lines.push('', chalk.bold(broken === null ? 'What the crawl found' : 'What the crawl found in the broken copy'));
+    if (broken !== null)
+        lines.push(chalk.yellow('  The copy was broken, so these differences are the environment\'s, not evidence about your change.'));
     // Bugs first, then the undecided, then what the judge called intended.
     const findings = VERDICTS.flatMap(verdict => published.findings.filter(finding => finding.verdict === verdict));
     if (findings.length === 0)
@@ -475,12 +507,16 @@ export function formatCrawl(view) {
  * is done and only its shutdown remains), or still running with --no-wait;
  * 2 ended without finishing (incomplete, cancelled) or finished with cleanup
  * unproven (an answered crawl has not finished its cleanup, so it is not judged); bugs
- * found never change it. The command's own failures exit 1 elsewhere. */
+ * found never change it. 5 (amendment 22): it finished, but the copy of the app was
+ * broken, so the change was not tested; it wins over unproven cleanup, since what the
+ * agent does next is fix how the app runs. The command's own failures exit 1 elsewhere. */
 export { verifyReport };
 export function crawlExitCode(view) {
     const status = reportedStatus(view);
     if (!TERMINAL.has(status))
         return 0;
+    if (status === 'completed' && brokenCopy(view) !== null)
+        return 5;
     if (status !== 'completed' || (TERMINAL.has(view.status) && view.totals.cleanupUnproven > 0))
         return 2;
     return 0;
@@ -651,12 +687,13 @@ function staleCrawl(found, identity) {
     return found.mode === 'prepare' || found.status === 'prepared' ? 'prepare-only' : null;
 }
 function staleWords(found, stale) {
+    const kind = found.mode === 'prepare' ? 'preparation' : 'crawl';
     switch (stale) {
-        case 'retitled': return `The last crawl of this code ran under another title, "${safe(found.changeTitle)}" (${found.runId})`;
-        case 'other-time': return `The last crawl of this code was asked for ${timeWords(found.budgetMs)} (${found.runId})`;
-        case 'cancelled': return `This change's last crawl was cancelled (${found.runId})`;
-        case 'incomplete': return `This change's last crawl stopped before finishing (${found.runId})`;
-        case 'cancelling': return `This change's crawl is being cancelled (${found.runId})`;
+        case 'retitled': return `The last ${kind} of this code ran under another title, "${safe(found.changeTitle)}" (${found.runId})`;
+        case 'other-time': return `The last ${kind} of this code was asked for ${timeWords(found.budgetMs)} (${found.runId})`;
+        case 'cancelled': return `This change's last ${kind} was cancelled (${found.runId})`;
+        case 'incomplete': return `This change's last ${kind} stopped before finishing (${found.runId})`;
+        case 'cancelling': return `This change's ${kind} is being cancelled (${found.runId})`;
         case 'prepare-only': return found.status === 'prepared'
             ? `Your change was built and frozen ahead of time (${found.runId})`
             : `Your change is being built and frozen ahead of time (${found.runId})`;
@@ -671,6 +708,8 @@ async function submitCapture(capture, token, found) {
     // sender does: admission is idempotent for one capture, so a repeat can only find what the first created.
     const acknowledgment = await readWithRetries(() => postPrecomputeCapture(capture, token, Date.now() + EXPLICIT_WALL_MS)).catch((error) => { throw normalModeFailure(error); });
     const crawl = acknowledgment.crawl;
+    // Amendment 12: the handshake submits mode 'prepare', which builds and prepares the change and never starts a crawl.
+    const preparing = capture.derivation.request.crawl?.mode === 'prepare';
     const before = found === null ? 'No crawl existed for this change' : staleWords(found.view, found.stale);
     if (crawl === undefined) {
         return { runId: null, onboarding: null,
@@ -682,22 +721,22 @@ async function submitCapture(capture, token, found) {
     }
     if (crawl.status === 'onboarding') {
         return { runId: null, onboarding: 'onboarding', line: `${before}. The app is not onboarded yet: `
-                + `the service is preparing it (${crawl.onboardRunId}) and runs the crawl when it is ready.` };
+                + `the service is preparing it (${crawl.onboardRunId}) and continues with this change when it is ready.` };
     }
     if (crawl.status === 'onboarding-blocked') {
         return { runId: null, onboarding: 'onboarding-blocked', line: `${before}, and the app's onboarding is blocked.` };
     }
     if (found === null) {
         return { runId: crawl.runId, onboarding: null, line: crawl.status === 'queued'
-                ? `No crawl existed for this change yet; started one (${crawl.runId}).`
-                : `The service already had a crawl for this change (${crawl.runId}).` };
+                ? `No crawl existed for this change yet; ${preparing ? 'started preparing it' : 'started one'} (${crawl.runId}).`
+                : `The service already had a check for this change (${crawl.runId}).` };
     }
     if (!anotherCrawl(found.stale) && crawl.runId === found.view.runId) {
         return { runId: crawl.runId, onboarding: null, line: found.stale === 'prepare-only'
                 ? `${before}; checking it now, reusing what was built.` : `${before}; asked the service to run it again.` };
     }
     return { runId: crawl.runId, onboarding: null, line: crawl.status === 'queued'
-            ? `${before}; started one for this change (${crawl.runId}).`
+            ? `${before}; ${preparing ? 'started preparing the change again' : 'started one for this change'} (${crawl.runId}).`
             : `${before}; the service already had one for this change (${crawl.runId}).` };
 }
 function checkedChange(baseCommit, workCommit) {
@@ -980,16 +1019,24 @@ export function formatBrief(brief) {
     }
     return lines.join('\n');
 }
-/** The handshake's question, asked after everything it shows. */
 /** The command to steer with; it keeps the options the handshake was given that choose the crawl (where, and for how long). */
 export function steerCommand(options) {
     const kept = [options.repo ? ` --repo ${options.repo}` : '', options.account ? ` --account ${options.account}` : '',
         options.minutes ? ` --minutes ${options.minutes}` : ''].join('');
     return `haystack verify${kept} --intent "<what you were asked to do>" --idea "<something to try>" [--idea ...]`;
 }
-const steerLines = (command) => ['',
-    'Next: pick the ideas worth trying, add your own, and say what you were asked to do (the task in the user\'s words, not what your code does):',
-    `  ${command}`, 'Your ideas are explored first; the ideas above are explored in every crawl too.'];
+/** The handshake's question, asked after everything it shows; it speaks of "the ideas above" only when it showed some. */
+export function steerLines(command, shownIdeas) {
+    return shownIdeas
+        ? ['', 'Next: pick the ideas worth trying, add your own, and say what you were asked to do (the task in the user\'s words, not what your code does):',
+            `  ${command}`, 'Your ideas are explored first; the ideas above are explored in every crawl too.']
+        : ['', 'Next: say what you were asked to do (the task in the user\'s words, not what your code does) and what to try:',
+            `  ${command}`, 'Your ideas are explored first.'];
+}
+/** Why a run stopped, as its report says it: the published error, else the run's own. */
+function stoppedBecause(view) {
+    return results(view)?.error ?? view.error ?? null;
+}
 /** The handshake (Akshay, 10/7): a plain `haystack verify` on a change no crawl has started for shows what the change touches,
  * how production and real users run it, and Haystack's own ideas, then asks the coding agent to run `haystack verify` again
  * with --intent (and any --idea). It submits the capture in mode 'prepare' when no run of it is preparing, prepared or
@@ -999,8 +1046,10 @@ async function handshake(capture, identity, change, found, stale, token, options
     let view = found;
     let onboarding = null;
     const finish = (brief, captured, flows) => {
+        const error = view === null ? null : stoppedBecause(view);
         printVerifyJson(options, change, onboarding, null, {
-            run: view === null ? null : { runId: view.runId, status: view.status }, brief, capture: captured, flows, next: steerCommand(options),
+            run: view === null ? null : { runId: view.runId, status: view.status, error: error ? { code: error.code, message: error.message } : null },
+            brief, capture: captured, flows, next: steerCommand(options),
         });
     };
     if (view === null || (stale !== null && stale !== 'prepare-only')) {
@@ -1055,12 +1104,18 @@ async function handshake(capture, identity, change, found, stale, token, options
         console.log(`What your change touches is not worked out yet (${currentStep(view)}); run \`haystack verify\` again to see it, or steer now.`);
     }
     else {
-        console.log(`Run ${view.runId} ended (${view.status}) without a brief${view.error ? `: ${errorWords(view.error)}` : ''}; steer the crawl without one.`);
+        // A failed prepare leaves no brief (amendment 18); the agent sees why, and that a steered crawl runs those steps again.
+        const error = stoppedBecause(view);
+        console.log(`Run ${view.runId} ended (${view.status}) without a brief${error ? `: ${errorWords(error)}` : ''}.`);
+        if (error) {
+            console.log(`  Details: ${safe(error.message)}`);
+            console.log('A steered crawl runs these steps again, so it may stop the same way; if it does, tell us with `haystack feedback`.');
+        }
     }
     if (captured !== null)
         for (const section of captured)
             console.log(`\n${formatCapture(section)}`);
-    console.log(steerLines(steerCommand(options)).join('\n'));
+    console.log(steerLines(steerCommand(options), (brief?.planning?.ideas.length ?? 0) > 0).join('\n'));
 }
 /** verify's one JSON document: the report (`--json`) or the service's whole record (`--raw`). */
 function printVerifyJson(options, change, onboarding, view, handshake) {

package/dist/index.js

@@ -120,7 +120,8 @@ an hour). The file is JSON with any of three notes, each plain text:
 Leave out what you do not know. Onboarding still proves everything it takes
 from the notes. The app is onboarded once: notes written before that feed its
 onboarding (an onboarding that stopped on a question starts again with them);
-after it, new notes start no onboarding.
+after it, the next check that cannot start the app uses them to update the
+setup in place.
 
 Every run also plans telemetry for the app: --app names it when the repository
 has several (init never picks one), --origin its production origins and
@@ -209,6 +210,16 @@ changed spot, how far the crawl got with it, and the steps that reached it),
 what the crawl found (bugs first, with the steps to see each), and the changed
 code it never ran.
 
+When the copy of the app was itself broken (error popups or messages from its
+own setup, its own requests failing in both builds), the crawl could not test
+your change, and it says so first: "We couldn't test your change: the app copy
+is broken.", why, what the start screen showed, the requests that failed and
+the console errors, and what to do: find what those errors need in the
+repository's own setup (docker compose files, env examples, docs), send that
+with \`haystack feedback\` (a copy that starts but is broken does not update its
+setup yet), and tell your user the change was not tested. What the crawl
+found in the broken copy is listed after it, marked as such.
+
 The first verify on a base the service has not onboarded yet prepares the app
 (reads the repository, plans how to run it, builds its runtime, starts it,
 prepares its data and test accounts, proves a workflow); the command shows each
@@ -225,6 +236,8 @@ Exit codes:
      stopped before finishing
   3  onboarding is blocked: the output says what would unblock it
   4  the handshake: run it again with --intent (and any --idea)
+  5  the crawl finished, but the copy of the app was broken, so your change
+     was not tested: the output says what to do
 
 The hosted fleet run of pushed commits is \`haystack verify hosted start\`.
 

package/dist/schema.js

@@ -12,9 +12,9 @@
 export const SCHEMA_VERSIONS = {
     'cloud-verifier': '1.0.0',
     'case-batch': '1.0.1',
-    verify: '2.1.1',
+    verify: '2.3.0',
     'verify-flow': '1.0.0',
-    'verify-raw': '1.1.0',
+    'verify-raw': '1.3.0',
     'verify-answer': '1.0.1',
     'verify-onboarding': '1.0.0',
     init: '1.0.2',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@haystackeditor/cli",
-  "version": "0.29.0",
+  "version": "0.30.2",
   "description": "haystack verify: run your app with and without a change, and see what the change broke",
   "type": "module",
   "bin": {

package/schemas/verify-raw.v1.json

@@ -21,7 +21,7 @@
       ]
     },
     "schema_version": {
-      "const": "1.1.0"
+      "const": "1.3.0"
     },
     "change": {
       "description": "Schema 1.0.2: what this verify checked: the capture's base commit (where HEAD meets origin's default branch), its work commit (committed and uncommitted changes), and the files that differ between them.",
@@ -65,6 +65,7 @@
         "run",
         "brief",
         "capture",
+        "flows",
         "next"
       ],
       "properties": {
@@ -77,7 +78,8 @@
               "type": "object",
               "required": [
                 "runId",
-                "status"
+                "status",
+                "error"
               ],
               "properties": {
                 "runId": {
@@ -95,6 +97,29 @@
                     "incomplete",
                     "cancelled"
                   ]
+                },
+                "error": {
+                  "oneOf": [
+                    {
+                      "type": "null"
+                    },
+                    {
+                      "type": "object",
+                      "additionalProperties": false,
+                      "required": [
+                        "code",
+                        "message"
+                      ],
+                      "properties": {
+                        "code": {
+                          "type": "string"
+                        },
+                        "message": {
+                          "type": "string"
+                        }
+                      }
+                    }
+                  ]
                 }
               }
             }
@@ -127,6 +152,84 @@
         },
         "next": {
           "type": "string"
+        },
+        "flows": {
+          "description": "Amendment 21 (2.1.1): the brief's flows in its order, each with its number (from 1, as `haystack verify flow <n>` takes it), files, function counts and, per app with capture set up, its routes' share of captured sessions (null without capture). Empty when the brief has none.",
+          "type": "array",
+          "items": {
+            "type": "object",
+            "required": [
+              "number",
+              "id",
+              "name",
+              "files",
+              "functions",
+              "changedFunctions",
+              "capture"
+            ],
+            "properties": {
+              "number": {
+                "type": "integer",
+                "minimum": 1
+              },
+              "id": {
+                "type": "string"
+              },
+              "name": {
+                "type": "string"
+              },
+              "files": {
+                "type": "array",
+                "items": {
+                  "type": "string"
+                }
+              },
+              "functions": {
+                "type": "integer",
+                "minimum": 0
+              },
+              "changedFunctions": {
+                "type": "integer",
+                "minimum": 0
+              },
+              "capture": {
+                "oneOf": [
+                  {
+                    "type": "null"
+                  },
+                  {
+                    "type": "array",
+                    "items": {
+                      "type": "object",
+                      "required": [
+                        "app",
+                        "applicationId",
+                        "state"
+                      ],
+                      "properties": {
+                        "app": {
+                          "type": "string"
+                        },
+                        "state": {
+                          "enum": [
+                            "ready",
+                            "no-data",
+                            "unavailable"
+                          ]
+                        },
+                        "routes": {
+                          "$ref": "#/$defs/captureWindow"
+                        },
+                        "reason": {
+                          "type": "string"
+                        }
+                      }
+                    }
+                  }
+                ]
+              }
+            }
+          }
         }
       }
     }
@@ -931,6 +1034,56 @@
             "planning"
           ]
         },
+        "health": {
+          "description": "Schema 1.2.0, CRAWL-V1 amendment 22, in the final answer and the sealed manifest only: whether the copy of the app worked, so the crawl could test the change. not-judged carries only its reason. Otherwise status (app-broken: Clef answered app_broken and scored it working under 0.3, so the findings are the environment's; ok), working (0 to 1), tested (tested, app_broken, not_reached or unclear), reason (one sentence), evidence (the start screen's texts, messages and overlays and what each build reported with nobody clicking, at most 40 lines each, and at most 8 commonest difference lines with their counts) and images (health/old.png and health/new.png in the bundle, null in answers).",
+          "type": "object",
+          "required": [
+            "status",
+            "reason"
+          ],
+          "properties": {
+            "status": {
+              "enum": [
+                "ok",
+                "app-broken",
+                "not-judged"
+              ]
+            },
+            "reason": {
+              "type": "string"
+            },
+            "working": {
+              "type": "number",
+              "minimum": 0,
+              "maximum": 1
+            },
+            "tested": {
+              "enum": [
+                "tested",
+                "app_broken",
+                "not_reached",
+                "unclear"
+              ]
+            },
+            "evidence": {
+              "type": "object",
+              "required": [
+                "texts",
+                "messages",
+                "overlays",
+                "background",
+                "commonDifferences"
+              ]
+            },
+            "images": {
+              "type": "object",
+              "required": [
+                "old",
+                "new"
+              ]
+            }
+          }
+        },
         "notFinishedInTime": {
           "$ref": "#/$defs/count"
         },

package/schemas/verify.v2.json

@@ -11,7 +11,7 @@
   ],
   "properties": {
     "schema_version": {
-      "const": "2.1.1"
+      "const": "2.3.0"
     },
     "change": {
       "description": "Schema 1.0.2: what this verify checked: the capture's base commit (where HEAD meets origin's default branch), its work commit (committed and uncommitted changes), and the files that differ between them.",
@@ -70,6 +70,7 @@
             "neverRan",
             "notFinishedInTime",
             "outsideServiceGaps",
+            "health",
             "error",
             "machinesNotProvenShutDown"
           ],
@@ -342,6 +343,70 @@
               },
               "description": "Calls the copy's stand-ins could not serve: missing coverage, never bugs."
             },
+            "health": {
+              "description": "Schema 2.2.0 (CRAWL-V1 amendment 22): whether the copy of the app worked, so the crawl could test the change; null when the results say nothing of it (a crawl that did not explore, or a service from before it). app-broken: the copy was itself broken, so the change was not tested and the findings are the environment's; the command says so first and exits 5 (a run cancelled or failed after it answered keeps its own exit code). For app-broken, shows (what the start screen shows that reads as an error), failedRequests and consoleErrors (logged with nobody clicking, each with the builds it was in) and whatToDo; empty and null otherwise.",
+              "oneOf": [
+                {
+                  "type": "null"
+                },
+                {
+                  "type": "object",
+                  "additionalProperties": false,
+                  "required": [
+                    "status",
+                    "reason",
+                    "shows",
+                    "failedRequests",
+                    "consoleErrors",
+                    "whatToDo"
+                  ],
+                  "properties": {
+                    "status": {
+                      "enum": [
+                        "ok",
+                        "app-broken",
+                        "not-judged"
+                      ]
+                    },
+                    "reason": {
+                      "type": "string",
+                      "description": "One sentence in plain words."
+                    },
+                    "shows": {
+                      "type": "array",
+                      "items": {
+                        "type": "string"
+                      }
+                    },
+                    "failedRequests": {
+                      "type": "array",
+                      "items": {
+                        "type": "string"
+                      }
+                    },
+                    "consoleErrors": {
+                      "type": "array",
+                      "items": {
+                        "type": "string"
+                      }
+                    },
+                    "whatToDo": {
+                      "oneOf": [
+                        {
+                          "type": "null"
+                        },
+                        {
+                          "type": "array",
+                          "items": {
+                            "type": "string"
+                          }
+                        }
+                      ]
+                    }
+                  }
+                }
+              ]
+            },
             "error": {
               "oneOf": [
                 {
@@ -393,7 +458,8 @@
               "type": "object",
               "required": [
                 "runId",
-                "status"
+                "status",
+                "error"
               ],
               "properties": {
                 "runId": {
@@ -411,6 +477,29 @@
                     "incomplete",
                     "cancelled"
                   ]
+                },
+                "error": {
+                  "oneOf": [
+                    {
+                      "type": "null"
+                    },
+                    {
+                      "type": "object",
+                      "additionalProperties": false,
+                      "required": [
+                        "code",
+                        "message"
+                      ],
+                      "properties": {
+                        "code": {
+                          "type": "string"
+                        },
+                        "message": {
+                          "type": "string"
+                        }
+                      }
+                    }
+                  ]
                 }
               }
             }