@haystackeditor/cli 0.23.3 → 0.24.1

package/dist/telemetry/next.js

@@ -0,0 +1,168 @@
+/**
+ * Next.js production telemetry without an application SDK.
+ *
+ *   // next.config.ts
+ *   import { withHaystackTelemetry } from '@haystackeditor/cli/next';
+ *   export default withHaystackTelemetry(nextConfig);
+ *
+ * `next build` (Turbopack or --webpack) then instruments every git-tracked
+ * server source module as the bundler compiles it, and writes the runtime
+ * beside the build in `<distDir>/.haystack-telemetry/`. Start the server with
+ *
+ *   NODE_OPTIONS=--require=<abs distDir>/.haystack-telemetry/register.cjs
+ *
+ * plus the usual HAYSTACK_TELEMETRY / _ENDPOINT / _TOKEN variables; without
+ * them every probe is a pass-through.
+ *
+ * Why this shape:
+ *  - Source, not output: Next packs server code into chunks that mirror no
+ *    source file, so `haystack telemetry instrument` cannot map them. A loader
+ *    sees each module's real path and text, so every site records its
+ *    repository path and line, which is what joins it to a change's functions.
+ *  - Loaders run in bundler worker processes (Turbopack always), so they hand
+ *    each module's sites to the post-compile step through a per-build session
+ *    directory, not memory. The session id travels in the environment, which
+ *    every process `next build` spawns after loading this config inherits.
+ *  - The runtime stays outside the bundle and is installed by a preload, which
+ *    runs before the server's first module under `next start` and standalone
+ *    `server.js` alike, so no customer source file (instrumentation.ts) changes.
+ *  - Only `next build` is instrumented; dev and `next start` get the config
+ *    untouched.
+ */
+import { execFileSync } from 'node:child_process';
+import { createHash, randomBytes } from 'node:crypto';
+import { existsSync, readdirSync, readFileSync, realpathSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { authenticateTelemetryRuntime, BUNDLED_SOURCE_EXTENSIONS, writeBundledTelemetryRuntime, } from '../commands/telemetry.js';
+const PRODUCTION_BUILD_PHASE = 'phase-production-build';
+const SESSION_ENVIRONMENT = '__HAYSTACK_TELEMETRY_NEXT_SESSION';
+// Must equal the directory names in ./next-loader.cts.
+const MODULES_DIRECTORY = 'modules';
+const UNPARSED_DIRECTORY = 'unparsed';
+function readRecords(directory) {
+    if (!existsSync(directory))
+        return [];
+    return readdirSync(directory)
+        .filter(name => name.endsWith('.json'))
+        .map(name => JSON.parse(readFileSync(join(directory, name), 'utf8')));
+}
+const LOADER_PATH = fileURLToPath(new URL('./next-loader.cjs', import.meta.url));
+function plainObject(value) {
+    return value && typeof value === 'object' && !Array.isArray(value) ? value : {};
+}
+function gitOutput(cwd, args) {
+    try {
+        return execFileSync('git', args, { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], maxBuffer: 256 * 1024 * 1024 });
+    }
+    catch (error) {
+        throw new Error(`Haystack telemetry needs the app's git checkout to name source files (git ${args.join(' ')} failed in ${cwd}): ` +
+            `${error instanceof Error ? error.message : String(error)}`);
+    }
+}
+export function withHaystackTelemetry(nextConfig) {
+    return async (phase, context) => {
+        const resolved = typeof nextConfig === 'function' ? await nextConfig(phase, context) : nextConfig;
+        return phase === PRODUCTION_BUILD_PHASE ? instrumentProductionBuild(resolved) : resolved;
+    };
+}
+function instrumentProductionBuild(config) {
+    if (plainObject(config.experimental).turbopackFileSystemCacheForBuild) {
+        // A persisted Turbopack result skips the loader, so that module's sites
+        // would never reach the manifest and its probes would count nothing.
+        throw new Error('Haystack telemetry cannot instrument a build that reuses cached Turbopack modules; turn off experimental.turbopackFileSystemCacheForBuild');
+    }
+    // Site paths are repository-relative so they match the crawl's blast radius
+    // (`apps/web/src/...` in a monorepo), not relative to the Next project.
+    const sourceRoot = realpathSync(gitOutput(process.cwd(), ['rev-parse', '--show-toplevel']).trim());
+    // Evaluating this config has no side effects beyond naming the session:
+    // `next build` re-evaluates it in every worker it spawns (page-data and
+    // static-generation workers too), and loaders create what they write.
+    const session = process.env[SESSION_ENVIRONMENT] ||= randomBytes(12).toString('hex');
+    const sessionDirectory = join(tmpdir(), `haystack-telemetry-next-${session}`);
+    // The channel only has to be unique per instrumented build sharing a
+    // process; deriving it from the checkout keeps identical builds identical.
+    const controlChannel = `__haystackTelemetryControl_${createHash('sha256')
+        .update(`next\0${sourceRoot}`).digest('hex').slice(0, 24)}`;
+    const loaderOptions = {
+        sourceRoot,
+        sessionDirectory,
+        controlChannel,
+        runtimeIntegrity: authenticateTelemetryRuntime(controlChannel).integrity,
+    };
+    const loader = { loader: LOADER_PATH, options: loaderOptions };
+    // Turbopack (Next 16's default bundler): server-side ("node") modules outside
+    // node_modules ("foreign"). Edge and browser contexts are left alone: the
+    // runtime is Node-only.
+    const turbopack = plainObject(config.turbopack);
+    const rules = { ...plainObject(turbopack.rules) };
+    for (const extension of BUNDLED_SOURCE_EXTENSIONS) {
+        const pattern = `*${extension}`;
+        const rule = { condition: { all: ['node', { not: 'foreign' }] }, loaders: [loader] };
+        const existing = rules[pattern];
+        rules[pattern] = existing === undefined ? rule : [rule, ...(Array.isArray(existing) ? existing : [existing])];
+    }
+    const originalWebpack = typeof config.webpack === 'function' ? config.webpack : null;
+    const compiler = plainObject(config.compiler);
+    const originalAfterCompile = typeof compiler.runAfterProductionCompile === 'function'
+        ? compiler.runAfterProductionCompile
+        : null;
+    return {
+        ...config,
+        turbopack: { ...turbopack, rules },
+        // webpack (`next build --webpack`): the Node server compilation only, as
+        // a pre-loader so it sees the source before SWC strips types and lines.
+        webpack(webpackConfig, context) {
+            const applied = originalWebpack ? originalWebpack(webpackConfig, context) : webpackConfig;
+            const addRule = (resolved) => {
+                if (context?.isServer !== true || context?.nextRuntime !== 'nodejs')
+                    return resolved;
+                const moduleConfig = plainObject(resolved.module);
+                moduleConfig.rules = [{
+                        test: /\.(?:[cm]?[jt]sx?)$/,
+                        exclude: /[\\/]node_modules[\\/]/,
+                        enforce: 'pre',
+                        use: [loader],
+                    }, ...(Array.isArray(moduleConfig.rules) ? moduleConfig.rules : [])];
+                resolved.module = moduleConfig;
+                return resolved;
+            };
+            return applied instanceof Promise ? applied.then(addRule) : addRule(applied);
+        },
+        compiler: {
+            ...compiler,
+            async runAfterProductionCompile(context) {
+                if (originalAfterCompile)
+                    await originalAfterCompile(context);
+                writeNextTelemetryRuntime(loaderOptions, resolve(context.projectDir, context.distDir));
+            },
+        },
+    };
+}
+function writeNextTelemetryRuntime(options, distDirectory) {
+    const modules = readRecords(join(options.sessionDirectory, MODULES_DIRECTORY));
+    const unparsed = readRecords(join(options.sessionDirectory, UNPARSED_DIRECTORY));
+    if (modules.length === 0) {
+        throw new Error(`Haystack telemetry: the bundler compiled no instrumentable tracked server module through its loader${unparsed.length > 0 ? ` (${unparsed.length} could not be parsed)` : ''}; nothing was instrumented`);
+    }
+    const { registerPath, manifest } = writeBundledTelemetryRuntime({
+        outputDirectory: distDirectory,
+        controlChannel: options.controlChannel,
+        bundler: 'next',
+        sourceRoot: options.sourceRoot,
+        modules,
+        unparsed,
+    });
+    // The session is spent. Its id stays in the environment so the workers
+    // `next build` spawns next resolve the same (now empty) session.
+    rmSync(options.sessionDirectory, { recursive: true, force: true });
+    console.log(`Haystack telemetry: instrumented ${modules.length} server module(s), ${manifest.sites.length} site(s); build ${manifest.build_id}.`);
+    if (manifest.skipped_unparsed_files.length > 0) {
+        // Named, never silent: these modules run but count nothing.
+        console.log(`Haystack telemetry: skipped ${manifest.skipped_unparsed_files.length} server module(s) Babel could not parse (shipped uninstrumented; listed in instrumentation.json skipped_unparsed_files):`);
+        for (const file of manifest.skipped_unparsed_files)
+            console.log(`  ${file.source_path}: ${file.reason}`);
+    }
+    console.log(`Haystack telemetry: start the server with NODE_OPTIONS=--require=${registerPath}`);
+}

package/dist/utils/haystack-api.js

@@ -41,8 +41,15 @@ export function transientReadFailure(error) {
         return [429, 502, 503, 504].includes(error.status) ? `HTTP ${error.status}` : null;
     if (error instanceof Error && error.name === 'TimeoutError')
         return 'no answer in time';
-    return error instanceof TypeError ? transportCause(error.cause, 0) : null;
-}
+    return error instanceof TypeError ? transportCause(causeOf(error), 0) : null;
+}
+/** An error's cause and, for an AggregateError, its inner errors, read structurally rather than through ES2022's Error.cause
+ * and AggregateError typings: the frontend's build type-checks this file against ES2020 (#3854 broke deploy-frontend). */
+const causeOf = (error) => error.cause;
+const innerErrors = (error) => {
+    const errors = error.errors;
+    return Array.isArray(errors) ? errors : [];
+};
 /** The transport failure somewhere in a fetch TypeError's cause chain. A connection dropped while the body is read fails as
  * TypeError "terminated", whose cause may be the socket error or the request's TimeoutError (Bugbot, #3760); several
  * addresses tried at once (IPv4 and IPv6) fail as one AggregateError. */
@@ -54,14 +61,12 @@ function transportCause(cause, depth) {
     const code = cause.code;
     if (typeof code === 'string' && TRANSPORT_FAILURE_CODES.has(code))
         return `no answer: ${code}`;
-    if (cause instanceof AggregateError) {
-        for (const inner of cause.errors) {
-            const found = transportCause(inner, depth + 1);
-            if (found !== null)
-                return found;
-        }
+    for (const inner of innerErrors(cause)) {
+        const found = transportCause(inner, depth + 1);
+        if (found !== null)
+            return found;
     }
-    return transportCause(cause.cause, depth + 1);
+    return transportCause(causeOf(cause), depth + 1);
 }
 /** How long a wait keeps asking while the service gives no answer, counted from its last answer. Stall detection, not a
  * cap on the work: a service that answers is followed for as long as the work runs. */

package/package.json

@@ -1,13 +1,20 @@
 {
   "name": "@haystackeditor/cli",
-  "version": "0.23.3",
+  "version": "0.24.1",
   "description": "haystack verify: run your app with and without a change, and see what the change broke",
   "type": "module",
   "bin": {
     "haystack": "dist/index.js"
   },
+  "exports": {
+    "./next": {
+      "types": "./dist/telemetry/next.d.ts",
+      "default": "./dist/telemetry/next.js"
+    },
+    "./package.json": "./package.json"
+  },
   "scripts": {
-    "build": "rm -rf dist && tsc && cp -r src/assets dist/assets",
+    "build": "rm -rf dist && tsc && tsc -p tsconfig.next-types.json && cp -r src/assets dist/assets",
     "dev": "tsc --watch",
     "lint": "eslint src",
     "start": "node dist/index.js",

package/schemas/init.v1.json

@@ -2,12 +2,12 @@
   "$schema": "https://json-schema.org/draft/2020-12/schema",
   "$id": "https://docs.haystackeditor.com/cli/schemas/init.v1.json",
   "title": "Haystack init: this repository's setup for `haystack verify`",
-  "description": "Machine-readable output of `haystack init --json`. `changes` are the project-local changes init planned (made when status is done); `onboarding` is the base's OnboardingStatusV1 (onboarding_contracts.ts, APP-ONBOARDING-V1) as POST /api/agent/cloud-verifier/onboarding returns it, null unless status is done. Exit codes: 0 done (onboarding running or ready), 1 failed (the error envelope), 2 planned (nothing written; run with --yes), 3 done but onboarding blocked, 4 login-required, 5 app-required (install the Haystack GitHub App), 6 done but onboarding stopped before finishing (run init again).",
+  "description": "Machine-readable output of `haystack init --json`. `changes` are the project-local changes init planned (made when status is done); `onboarding` is the base's OnboardingStatusV1 (onboarding_contracts.ts, APP-ONBOARDING-V1) as POST /api/agent/cloud-verifier/onboarding returns it, null unless status is done. `notes` says what became of `--notes`: \"written\" (the repository's onboarding notes are now these), \"unchanged\" (they already were), null without `--notes` or when init stopped before writing them. Exit codes: 0 done (onboarding running or ready), 1 failed (the error envelope), 2 planned (nothing written; run with --yes), 3 done but onboarding blocked, 4 login-required, 5 app-required (install the Haystack GitHub App), 6 done but onboarding stopped before finishing (run init again).",
   "type": "object",
   "additionalProperties": false,
-  "required": ["schema_version", "status", "repository", "changes", "notices", "onboarding"],
+  "required": ["schema_version", "status", "repository", "changes", "notices", "notes", "onboarding"],
   "properties": {
-    "schema_version": { "const": "1.0.0" },
+    "schema_version": { "const": "1.0.1" },
     "status": { "enum": ["done", "planned", "login-required", "app-required"] },
     "repository": { "type": "string" },
     "changes": {
@@ -25,6 +25,7 @@
       }
     },
     "notices": { "type": "array", "items": { "type": "string" } },
+    "notes": { "oneOf": [{ "type": "null" }, { "enum": ["written", "unchanged"] }] },
     "onboarding": {
       "oneOf": [
         { "type": "null" },

package/schemas/pre-verify.v1.json

@@ -0,0 +1,178 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://docs.haystackeditor.com/cli/schemas/pre-verify.v1.json",
+  "title": "haystack pre-verify --json",
+  "description": "CRAWL-V1 amendment 18: the brief of the current change before any crawl: the functions it touches (the front half's blast radius, nearest first) and, when incident planning ran, Haystack's own ideas of what could break. `run` is the run of this capture that holds or will hold the brief; `brief` is null until it does (or when the run ended without one).",
+  "type": "object",
+  "required": ["schema_version", "change", "onboarding", "run", "brief"],
+  "properties": {
+    "schema_version": { "const": "1.0.1" },
+    "change": {
+      "type": "object",
+      "required": ["baseCommit", "workCommit", "files"],
+      "properties": {
+        "baseCommit": { "type": "string", "pattern": "^[0-9a-f]{40}$" },
+        "workCommit": { "type": "string", "pattern": "^[0-9a-f]{40}$" },
+        "files": { "type": "array", "items": { "type": "string" } }
+      }
+    },
+    "onboarding": { "description": "The app's onboarding for this base when this command met it (see `haystack schema verify-onboarding`), else null.", "oneOf": [{ "type": "null" }, { "type": "object" }] },
+    "run": {
+      "oneOf": [
+        { "type": "null" },
+        {
+          "type": "object",
+          "required": ["runId", "status"],
+          "properties": {
+            "runId": { "type": "string", "pattern": "^cv_[0-9a-f]{48}$" },
+            "status": { "enum": ["queued", "preparing", "crawling", "draining", "completed", "prepared", "incomplete", "cancelled"] }
+          }
+        }
+      ]
+    },
+    "brief": { "oneOf": [{ "type": "null" }, { "$ref": "#/$defs/brief" }] }
+  },
+  "$defs": {
+    "brief": {
+      "type": "object",
+      "required": ["functions", "leftOut", "planning"],
+      "properties": {
+        "functions": {
+          "description": "Every affected function the front half listed (at most 2000), nearest first. distance 0 is edited by the change; 1 is one engine edge away.",
+          "type": "array",
+          "maxItems": 2000,
+          "items": {
+            "type": "object",
+            "required": ["file", "name", "lines", "distance", "direction"],
+            "properties": {
+              "file": { "type": "string" },
+              "name": { "type": "string" },
+              "lines": { "type": "array", "minItems": 2, "maxItems": 2, "items": { "type": "integer", "minimum": 0 } },
+              "distance": { "type": "integer", "minimum": 0 },
+              "direction": { "enum": ["changed", "upstream", "downstream"] }
+            }
+          }
+        },
+        "leftOut": {
+          "description": "Functions the blast radius found but did not list, counted with why.",
+          "type": "array",
+          "items": { "type": "object", "required": ["reason", "count"], "properties": { "reason": { "type": "string" }, "count": { "type": "integer", "minimum": 0 } } }
+        },
+        "production": {
+          "description": "CRAWL-V1 amendment 19, only when the repository's own Haystack telemetry holds data in the last 7 days: how the brief's functions ran in production, by their index in `functions` (production runs the code before the change; lines are moved through the change's diff). A function with no row is unknown, not unused.",
+          "type": "object",
+          "required": ["windowDays", "payloads", "newestAt", "partial", "functions"],
+          "properties": {
+            "windowDays": { "type": "integer", "minimum": 1 },
+            "payloads": { "type": "integer", "minimum": 1 },
+            "newestAt": { "type": "string" },
+            "partial": { "type": "boolean", "description": "The read stopped at its byte budget: every count is a lower bound." },
+            "functions": {
+              "type": "array",
+              "items": {
+                "type": "object",
+                "required": ["index", "branches", "branchesOmitted", "values", "valuesOmitted"],
+                "properties": {
+                  "index": { "type": "integer", "minimum": 0 },
+                  "branches": {
+                    "type": "array",
+                    "maxItems": 100,
+                    "items": {
+                      "type": "object",
+                      "required": ["line", "whenTrue", "whenFalse", "capped"],
+                      "properties": {
+                        "line": { "type": "integer", "minimum": 1 },
+                        "whenTrue": { "type": "integer", "minimum": 0 },
+                        "whenFalse": { "type": "integer", "minimum": 0 },
+                        "capped": { "type": "boolean", "description": "A counter hit its cap: the counts are lower bounds." }
+                      }
+                    }
+                  },
+                  "branchesOmitted": { "type": "integer", "minimum": 0 },
+                  "values": {
+                    "type": "array",
+                    "maxItems": 100,
+                    "items": {
+                      "type": "object",
+                      "required": ["line", "what", "shapes"],
+                      "properties": {
+                        "line": { "type": "integer", "minimum": 1 },
+                        "what": { "type": "string", "description": "The value observed, e.g. parameter:email or binding:options." },
+                        "shapes": {
+                          "type": "array",
+                          "items": {
+                            "type": "object",
+                            "required": ["shape", "count"],
+                            "properties": {
+                              "shape": { "type": "string", "description": "A value-free category, e.g. undefined, string:empty:empty, array:few." },
+                              "count": { "type": "integer", "minimum": 0 }
+                            }
+                          }
+                        }
+                      }
+                    }
+                  },
+                  "valuesOmitted": { "type": "integer", "minimum": 0 }
+                }
+              }
+            }
+          }
+        },
+        "planning": {
+          "description": "Incident planning; null when it did not run for this change.",
+          "oneOf": [
+            { "type": "null" },
+            {
+              "type": "object",
+              "required": ["groups", "unresolved", "ideas"],
+              "properties": {
+                "groups": {
+                  "description": "Behavior groups of the change, each with the failure patterns selected for it.",
+                  "type": "array",
+                  "items": {
+                    "type": "object",
+                    "required": ["id", "changeKeys", "patterns"],
+                    "properties": {
+                      "id": { "type": "string" },
+                      "changeKeys": { "type": "array", "items": { "type": "string" } },
+                      "patterns": {
+                        "type": "array",
+                        "items": { "type": "object", "required": ["entryId", "mechanism", "symptom"],
+                          "properties": { "entryId": { "type": "string" }, "mechanism": { "type": "string" }, "symptom": { "type": "string" } } }
+                      }
+                    }
+                  }
+                },
+                "unresolved": {
+                  "description": "Changes no behavior group holds, and why.",
+                  "type": "array",
+                  "items": { "type": "object", "required": ["changeKey", "reason"], "properties": { "changeKey": { "type": "string" }, "reason": { "type": "string" } } }
+                },
+                "ideas": {
+                  "description": "Haystack's own ideas: the selected situations, in plan order (at most 64). Every crawl explores them; pass the ones you want first, and your own, to `haystack verify --idea`.",
+                  "type": "array",
+                  "maxItems": 64,
+                  "items": {
+                    "type": "object",
+                    "required": ["id", "groupId", "pattern", "why", "failure", "setup", "exercise", "watch", "start"],
+                    "properties": {
+                      "id": { "type": "string" },
+                      "groupId": { "type": "string" },
+                      "pattern": { "type": "string", "description": "The failure pattern's library id." },
+                      "why": { "type": "string", "description": "How that pattern could arise in this code." },
+                      "failure": { "type": "string", "description": "What would go wrong." },
+                      "setup": { "type": "array", "items": { "type": "string" }, "description": "What must be set up or true first." },
+                      "exercise": { "type": "string", "description": "What to do." },
+                      "watch": { "type": "string", "description": "What to watch." },
+                      "start": { "type": "string", "description": "Where it starts: the start screen, or a prepared start." }
+                    }
+                  }
+                }
+              }
+            }
+          ]
+        }
+      }
+    }
+  }
+}

package/schemas/verify-answer.v1.json

@@ -11,7 +11,7 @@
   ],
   "properties": {
     "schema_version": {
-      "const": "1.0.0"
+      "const": "1.0.1"
     },
     "facts": {
       "type": "object",
@@ -52,6 +52,13 @@
               "additionalProperties": {
                 "type": "string"
               }
+            },
+            "notes": {
+              "description": "What the repository's coding agent said about the app (APP-ONBOARDING-V1 rule 7, amendment 18), present once written with `haystack init --notes`; empty where it said nothing.",
+              "type": "object",
+              "additionalProperties": false,
+              "required": ["run", "signIn", "workflow"],
+              "properties": { "run": { "type": "string" }, "signIn": { "type": "string" }, "workflow": { "type": "string" } }
             }
           }
         },

package/schemas/verify.v1.json

@@ -17,7 +17,7 @@
         }
       ]
     },
-    "schema_version": { "const": "1.0.2" },
+    "schema_version": { "const": "1.0.4" },
     "change": {
       "description": "Schema 1.0.2: what this verify checked: the capture's base commit (where HEAD meets origin's default branch), its work commit (committed and uncommitted changes), and the files that differ between them.",
       "type": "object",
@@ -452,6 +452,27 @@
             "gaps": { "type": "array", "maxItems": 200, "items": { "type": "string" } }
           }
         },
+        "ideas": {
+          "description": "CRAWL-V1 amendment 17, only when `--idea` or `--ideas-file` gave ideas: one entry per idea in the order given. 'covered': an independent review saw it done in both builds; 'blocked': it could not be done in this copy (reason says why); 'unfinished': the crawl's time ended first. findingIds names the findings found while exploring it.",
+          "type": "array",
+          "maxItems": 64,
+          "items": {
+            "type": "object",
+            "required": ["id", "text", "state", "reason", "findingIds"],
+            "properties": {
+              "id": { "type": "string", "pattern": "^idea-[1-9][0-9]*$" },
+              "text": { "type": "string" },
+              "state": { "enum": ["covered", "blocked", "unfinished"] },
+              "reason": { "type": "string" },
+              "findingIds": { "$ref": "#/$defs/strings" }
+            }
+          }
+        },
+        "brief": {
+          "description": "CRAWL-V1 amendment 18: what Haystack knew of the change before exploring it (see `haystack schema pre-verify`, $defs.brief); absent when the run ended before its prepare.",
+          "type": "object",
+          "required": ["functions", "leftOut", "planning"]
+        },
         "notFinishedInTime": { "$ref": "#/$defs/count" },
         "error": { "oneOf": [{ "type": "null" }, { "$ref": "#/$defs/error" }] }
       }