@haystackeditor/cli 0.21.0 → 0.22.0

package/dist/commands/verify-onboarding.js

@@ -10,7 +10,7 @@
 import chalk from 'chalk';
 import { withSchema } from '../schema.js';
 import { resolveAuthContext } from '../utils/auth.js';
-import { classifyHttpError, HaystackApiError } from '../utils/haystack-api.js';
+import { classifyHttpError, readWithRetries, SERVICE_SILENCE_LIMIT_MS, ServiceSilentError, transientReadFailure, } from '../utils/haystack-api.js';
 import { trackOnboardingEvent } from '../utils/telemetry.js';
 import { gatewayFetch } from './case-batch.js';
 import { ONBOARDING_STAGES, ONBOARDING_VERSION, } from './onboarding-contract.js';
@@ -93,6 +93,8 @@ export function parseOnboardingStatus(value, expected) {
         invalid('its data absences');
     if (!isStrings(value.versionChoices))
         invalid('its version choices');
+    if (!isStrings(value.standIns) || !isStrings(value.unsetProviders))
+        invalid('its stand-ins');
     if (value.block !== null)
         parseOnboardingBlock(value.block);
     const checkpoint = value.checkpoint;
@@ -128,6 +130,9 @@ async function body(response, what) {
         return await response.json();
     }
     catch (error) {
+        // A connection that dropped mid-body is not a bad body: a wait reads again.
+        if (transientReadFailure(error) !== null)
+            throw error;
         throw new OnboardingResponseError(`The onboarding service returned ${what} that is not JSON: ${error instanceof Error ? error.message : String(error)}`);
     }
 }
@@ -159,8 +164,11 @@ const ONBOARDING_READ_EVERY_MS = 15_000;
 const delay = (ms) => new Promise(resolve => setTimeout(resolve, ms));
 /** Polls the base's onboarding until it is no longer running. No wall-time cap:
  * an interrupt stops waiting and leaves the onboarding running (`resume` is the
- * command that follows it again). A momentary 429/5xx (a server or runner
- * restarting) is read again at the next poll. */
+ * command that follows it again). A read that may work if asked again (a 429/5xx
+ * while a server or runner restarts, or no answer at all: a dropped connection
+ * ended a 1 h wait on 2026-10-02 with "fetch failed") is read again at the next
+ * poll, said once per run of failures, until the service has given no answer for
+ * SERVICE_SILENCE_LIMIT_MS (stall detection). */
 export async function waitForOnboarding(first, token, progress, resume) {
     const interrupted = () => {
         console.error(`\nStopped waiting. The onboarding keeps running; run \`${resume}\` again to follow it.`);
@@ -171,6 +179,8 @@ export async function waitForOnboarding(first, token, progress, resume) {
         let status = first;
         progress(`Waiting for onboarding ${status.onboardRunId ?? ''} (Ctrl-C stops waiting; the onboarding keeps running).`);
         progress(onboardingStep(status));
+        let failing = false;
+        let answeredAt = Date.now();
         while (status.state === 'onboarding') {
             await delay(ONBOARDING_READ_EVERY_MS);
             let next;
@@ -178,11 +188,18 @@ export async function waitForOnboarding(first, token, progress, resume) {
                 next = await readOnboardingStatus(status.repository, status.baseCommit, token);
             }
             catch (error) {
-                if (!(error instanceof HaystackApiError && [429, 502, 503, 504].includes(error.status)))
+                const why = transientReadFailure(error);
+                if (why === null)
                     throw error;
-                progress(`The onboarding status is unavailable for now (${error.status}); reading it again.`);
+                if (Date.now() - answeredAt >= SERVICE_SILENCE_LIMIT_MS)
+                    throw new ServiceSilentError('onboarding', resume);
+                if (!failing)
+                    progress(`The onboarding status is unavailable for now (${why}); reading it again.`);
+                failing = true;
                 continue;
             }
+            failing = false;
+            answeredAt = Date.now();
             if (onboardingStep(next) !== onboardingStep(status))
                 progress(onboardingStep(next));
             status = next;
@@ -273,6 +290,9 @@ export function formatOnboarding(status) {
         lines.push('', ...status.dataAbsences.map(absence => chalk.yellow(`Note: ${safe(absence)}`)));
     if (status.versionChoices.length)
         lines.push('', ...status.versionChoices.map(choice => chalk.yellow(`Note: ${safe(choice)}`)));
+    const outside = [...status.standIns, ...status.unsetProviders];
+    if (outside.length)
+        lines.push('', ...outside.map(note => chalk.yellow(`Note: ${safe(note)}`)));
     return lines.join('\n');
 }
 /** Telemetry: onboarding started, blocked or ready, once per state a command sees. */
@@ -351,7 +371,8 @@ export async function verifyOnboardingCommand(options) {
     const origin = resolveOriginRepository(gitRoot);
     const repository = `${origin.owner}/${origin.repository}`;
     const auth = await resolveAuthContext({ preferredLogin: options.account, owner: origin.owner, repo: origin.repository });
-    let status = await readOnboardingStatus(repository, deriveBaseSha(gitRoot, Date.now() + EXPLICIT_WALL_MS), auth.token);
+    const baseCommit = deriveBaseSha(gitRoot, Date.now() + EXPLICIT_WALL_MS);
+    let status = await readWithRetries(() => readOnboardingStatus(repository, baseCommit, auth.token));
     reportOnboardingState(status);
     if (options.wait && status.state === 'onboarding') {
         status = await waitForOnboarding(status, auth.token, line => console.error(chalk.dim(`${clock()} ${line}`)), 'haystack verify onboarding --wait');

package/dist/commands/verify.js

@@ -26,7 +26,7 @@ import chalk from 'chalk';
 import { withSchema } from '../schema.js';
 import { findGitRoot } from '../utils/hooks.js';
 import { resolveAuthContext } from '../utils/auth.js';
-import { classifyHttpError, HaystackApiError } from '../utils/haystack-api.js';
+import { classifyHttpError, readWithRetries, SERVICE_SILENCE_LIMIT_MS, ServiceSilentError, transientReadFailure, } from '../utils/haystack-api.js';
 import { GATEWAY_TIMEOUT_MS, gatewayFetch } from './case-batch.js';
 import { CRAWL_BUDGET_MAX_MS, CRAWL_BUDGET_MIN_MS, CRAWL_MAX_FINDINGS, CRAWL_MAX_FINDING_STEPS, CRAWL_WAIT_MAX_MS, } from './crawl-contract.js';
 import { captureCheckout, crawlUnavailableText, EXPLICIT_WALL_MS, normalModeFailure, postPrecomputeCapture, } from './verify-precompute.js';
@@ -260,6 +260,9 @@ async function responseBody(response) {
         return await response.json();
     }
     catch (error) {
+        // A connection that dropped mid-body is not a bad body: a wait reads again.
+        if (transientReadFailure(error) !== null)
+            throw error;
         throw new CrawlResponseError(`The crawl service returned a body that is not JSON: ${error instanceof Error ? error.message : String(error)}`);
     }
 }
@@ -485,7 +488,11 @@ function printNewBugs(view, printed) {
 }
 /** Follows the same capture until its crawl answers at its budget or ends (amendment 9). Each read is held by the
  * service until the crawl changes, so there is no pause between reads. No wall-time cap: an interrupt stops waiting
- * and leaves the crawl running. */
+ * and leaves the crawl running. A read that may work if asked again (transientReadFailure) is asked again after
+ * TRANSIENT_READ_PAUSE_MS, said once per run of failures: one dropped connection used to end the command. Only a
+ * service that has given no answer for SERVICE_SILENCE_LIMIT_MS stops the wait (stall detection). */
+/** The pause before a followed crawl is read again after a read that may work if asked again. */
+const TRANSIENT_READ_PAUSE_MS = 5_000;
 async function waitForCrawl(first, identity, token) {
     const interrupted = () => {
         console.error('\nStopped waiting. The crawl keeps running; run `haystack verify` again to see it.');
@@ -498,17 +505,36 @@ async function waitForCrawl(first, identity, token) {
         progress(`Waiting for crawl ${view.runId} (Ctrl-C stops waiting; the crawl keeps running).`);
         progress(currentStep(view));
         printNewBugs(view, printed);
+        let failing = false;
+        let answeredAt = Date.now();
         while (!TERMINAL.has(reportedStatus(view))) {
-            // The followed crawl's own read is held, so only a change to it wakes the wait (Astra review: a capture read could sit up to
-            // 25 s on another title's crawl of the same code).
-            let next = await readCrawlRun(view.runId, identity, token, view.updatedAt);
-            // Once this crawl is being cancelled, the capture's newest crawl is read at once (not held) and followed if it is this same
-            // change (amendment 12: no crawl replaces another, but one can be cancelled and asked for again).
-            if (next.cancelRequested) {
-                const newest = await readCrawl(identity, token);
-                if (newest !== null && sameChange(newest, identity) && newest.runId !== view.runId)
-                    next = newest;
+            let next;
+            try {
+                // The followed crawl's own read is held, so only a change to it wakes the wait (Astra review: a capture read could sit up
+                // to 25 s on another title's crawl of the same code).
+                next = await readCrawlRun(view.runId, identity, token, view.updatedAt);
+                // Once this crawl is being cancelled, the capture's newest crawl is read at once (not held) and followed if it is this
+                // same change (amendment 12: no crawl replaces another, but one can be cancelled and asked for again).
+                if (next.cancelRequested) {
+                    const newest = await readCrawl(identity, token);
+                    if (newest !== null && sameChange(newest, identity) && newest.runId !== view.runId)
+                        next = newest;
+                }
             }
+            catch (error) {
+                const why = transientReadFailure(error);
+                if (why === null)
+                    throw error;
+                if (Date.now() - answeredAt >= SERVICE_SILENCE_LIMIT_MS)
+                    throw new ServiceSilentError('crawl', 'haystack verify');
+                if (!failing)
+                    progress(`The crawl's status is unavailable for now (${why}); reading it again.`);
+                failing = true;
+                await new Promise(resolve => setTimeout(resolve, TRANSIENT_READ_PAUSE_MS));
+                continue;
+            }
+            failing = false;
+            answeredAt = Date.now();
             if (next.runId !== view.runId)
                 progress(`A newer crawl of this same change took over (${next.runId}).`);
             if (!TERMINAL.has(reportedStatus(next)) && (next.runId !== view.runId || currentStep(next) !== currentStep(view))) {
@@ -555,21 +581,9 @@ function staleWords(found, stale) {
  * when no crawl could be started), whether the base is being onboarded or its
  * onboarding is blocked, and the line to report. */
 async function submitCapture(capture, token, found) {
-    let acknowledgment;
-    // A momentary 429/5xx (a server or runner restarting) is asked again, as the stop hook's sender does: admission is
-    // idempotent for one capture, so a repeat can only find what the first created.
-    const pauses = [2_000, 5_000, 10_000];
-    for (let attempt = 0; acknowledgment === undefined; attempt += 1) {
-        try {
-            acknowledgment = await postPrecomputeCapture(capture, token, Date.now() + EXPLICIT_WALL_MS);
-        }
-        catch (error) {
-            const transient = error instanceof HaystackApiError && [429, 502, 503, 504].includes(error.status);
-            if (!transient || attempt >= pauses.length)
-                throw normalModeFailure(error);
-            await new Promise(resolve => setTimeout(resolve, pauses[attempt]));
-        }
-    }
+    // A momentary 429/5xx or a dropped connection (a server or runner restarting) is asked again, as the stop hook's
+    // sender does: admission is idempotent for one capture, so a repeat can only find what the first created.
+    const acknowledgment = await readWithRetries(() => postPrecomputeCapture(capture, token, Date.now() + EXPLICIT_WALL_MS)).catch((error) => { throw normalModeFailure(error); });
     const crawl = acknowledgment.crawl;
     const before = found === null ? 'No crawl existed for this change' : staleWords(found.view, found.stale);
     if (crawl === undefined) {
@@ -651,7 +665,8 @@ export async function verifyCommand(options) {
         budgetMs: request.crawl.budgetMs ?? null,
     };
     const auth = await resolveAuthContext({ preferredLogin: options.account, owner: request.owner, repo: request.repository });
-    let view = await readCrawl(identity, auth.token);
+    // One-shot reads ask again after a failure that may work if asked again, as the waits do.
+    let view = await readWithRetries(() => readCrawl(identity, auth.token));
     let onboarding = null;
     const stale = view === null ? null : staleCrawl(view, identity);
     if (view === null || stale !== null) {
@@ -659,7 +674,7 @@ export async function verifyCommand(options) {
         let submitted = await submitCapture(capture, auth.token, found);
         note(submitted.line);
         if (submitted.onboarding !== null) {
-            onboarding = await readOnboardingStatus(identity.repository, identity.baseCommit, auth.token);
+            onboarding = await readWithRetries(() => readOnboardingStatus(identity.repository, identity.baseCommit, auth.token));
             reportOnboardingState(onboarding);
             if (options.wait !== false && onboarding.state === 'onboarding') {
                 onboarding = await waitForOnboarding(onboarding, auth.token, progress, 'haystack verify');
@@ -681,7 +696,8 @@ export async function verifyCommand(options) {
             process.exitCode = 1;
             return;
         }
-        view = await readCrawlRun(submitted.runId, identity, auth.token);
+        const runId = submitted.runId;
+        view = await readWithRetries(() => readCrawlRun(runId, identity, auth.token));
         if (found !== null && !anotherCrawl(found.stale) && view.runId === found.view.runId && TERMINAL.has(view.status)) {
             note('The service did not run it again; its last result follows.');
         }
@@ -693,7 +709,8 @@ export async function verifyCommand(options) {
         process.stdout.write(`${JSON.stringify(withSchema('verify', { change, onboarding, crawl: view }), null, 2)}\n`);
     else {
         console.log(formatCrawl(view));
-        const notes = [...(onboarding?.dataAbsences ?? []), ...(onboarding?.versionChoices ?? [])];
+        const notes = [...(onboarding?.dataAbsences ?? []), ...(onboarding?.versionChoices ?? []), ...(onboarding?.standIns ?? []),
+            ...(onboarding?.unsetProviders ?? [])];
         if (notes.length)
             console.log(['', ...notes.map(note => `Note: ${safe(note)}`)].join('\n'));
     }

package/dist/utils/haystack-api.js

@@ -25,6 +25,70 @@ export class HaystackApiError extends Error {
         return this.status === 401 || this.status === 403;
     }
 }
+/** The network failures fetch (undici) reports as its TypeError's `cause.code`: the connection refused, reset, dropped,
+ * unreachable or timed out, or the name not resolved for now. A request it could not even build (ERR_INVALID_URL) is not one. */
+const TRANSPORT_FAILURE_CODES = new Set(['ECONNREFUSED', 'ECONNRESET', 'ECONNABORTED', 'EPIPE', 'ETIMEDOUT', 'ENETUNREACH',
+    'ENETDOWN', 'EHOSTUNREACH', 'EHOSTDOWN', 'EAI_AGAIN', 'ENOTFOUND', 'UND_ERR_SOCKET', 'UND_ERR_CLOSED',
+    'UND_ERR_CONNECT_TIMEOUT', 'UND_ERR_HEADERS_TIMEOUT', 'UND_ERR_BODY_TIMEOUT']);
+/**
+ * Why a read failed in a way that asking again may fix, or null when it did not: the service or a runner restarting
+ * (429, 502, 503, 504), no answer at all (fetch's TypeError whose cause is one of TRANSPORT_FAILURE_CODES), or a request
+ * past its own deadline (TimeoutError). A wait loop reads again on these; anything else (sign-in, a bad request, a request
+ * that could not be built, a bug) is the caller's to report.
+ */
+export function transientReadFailure(error) {
+    if (error instanceof HaystackApiError)
+        return [429, 502, 503, 504].includes(error.status) ? `HTTP ${error.status}` : null;
+    if (error instanceof Error && error.name === 'TimeoutError')
+        return 'no answer in time';
+    return error instanceof TypeError ? transportCause(error.cause, 0) : null;
+}
+/** The transport failure somewhere in a fetch TypeError's cause chain. A connection dropped while the body is read fails as
+ * TypeError "terminated", whose cause may be the socket error or the request's TimeoutError (Bugbot, #3760); several
+ * addresses tried at once (IPv4 and IPv6) fail as one AggregateError. */
+function transportCause(cause, depth) {
+    if (!(cause instanceof Error) || depth > 4)
+        return null;
+    if (cause.name === 'TimeoutError')
+        return 'no answer in time';
+    const code = cause.code;
+    if (typeof code === 'string' && TRANSPORT_FAILURE_CODES.has(code))
+        return `no answer: ${code}`;
+    if (cause instanceof AggregateError) {
+        for (const inner of cause.errors) {
+            const found = transportCause(inner, depth + 1);
+            if (found !== null)
+                return found;
+        }
+    }
+    return transportCause(cause.cause, depth + 1);
+}
+/** How long a wait keeps asking while the service gives no answer, counted from its last answer. Stall detection, not a
+ * cap on the work: a service that answers is followed for as long as the work runs. */
+export const SERVICE_SILENCE_LIMIT_MS = 10 * 60_000;
+/** A wait stopped because the service gave no answer for SERVICE_SILENCE_LIMIT_MS (a dead network, a wrong
+ * HAYSTACK_API_BASE, a long outage); the work it was following keeps running on the service. */
+export class ServiceSilentError extends Error {
+    constructor(work, resume) {
+        super(`Haystack has not answered for ${SERVICE_SILENCE_LIMIT_MS / 60_000} minutes; the ${work} keeps running on the service. `
+            + `Run \`${resume}\` again to follow it.`);
+        this.name = 'ServiceSilentError';
+    }
+}
+/** A one-shot read, asked again after a failure that may work if asked again (transientReadFailure), at most three more times. */
+export async function readWithRetries(read) {
+    const pauses = [2_000, 5_000, 10_000];
+    for (let attempt = 0;; attempt += 1) {
+        try {
+            return await read();
+        }
+        catch (error) {
+            if (transientReadFailure(error) === null || attempt >= pauses.length)
+                throw error;
+            await new Promise(resolve => setTimeout(resolve, pauses[attempt]));
+        }
+    }
+}
 function headers(token, extra) {
     const result = new Headers(extra);
     result.set('Authorization', `Bearer ${token}`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@haystackeditor/cli",
-  "version": "0.21.0",
+  "version": "0.22.0",
   "description": "haystack verify: run your app with and without a change, and see what the change broke",
   "type": "module",
   "bin": {