@haystackeditor/cli 0.23.2 → 0.24.0

package/README.md

@@ -159,7 +159,10 @@ Intent is optional there: `--intent-file <path>` accepts JSON with exactly
 `problem`, `goal`, and `intended_outcomes`. It waits up to 35 minutes by
 default; pass `--no-wait` to return after the fleet run is queued. Repeating the
 same repository, commits, and intent reuses the same run; choose a new
-`--idempotency-key` when a fresh execution is intentional. A wait that expires
+`--idempotency-key` when a fresh execution is intentional. A fresh run on the
+same base reuses the repository's onboarding: phases that finished are not
+redone, and a phase that stopped terminally starts again once the fleet runs a
+different engine release (on the same release it stays stopped). A wait that expires
 prints `timed_out: true` and `next_command`, then exits 2. Use the returned
 account-bound `next_command` verbatim on machines with multiple saved accounts.
 

package/dist/assets/telemetry/runtime.cjs

@@ -416,6 +416,14 @@ function loadInstrumentation() {
       allowedFields: safeArrayIsArray(site.allowed_fields)
         ? safeArraySlice(site.allowed_fields, 0)
         : [],
+      // Source position, present only when the build instrumented source
+      // itself (bundler loader). Joins to the change's function line ranges.
+      ...(safeNumberIsSafeInteger(site.line) && site.line >= 1
+        ? {
+            line: site.line,
+            ...(safeNumberIsSafeInteger(site.column) && site.column >= 0 ? { column: site.column } : {}),
+          }
+        : {}),
     })]);
     const sites = new SafeMap();
     for (let index = 0; index < siteEntries.length; index++) {

package/dist/commands/crawl-contract.js

@@ -4,6 +4,10 @@ export const CRAWL_BUDGET_MIN_MS = 60_000;
 export const CRAWL_BUDGET_MAX_MS = 30 * 60_000;
 export const CRAWL_MAX_FINDINGS = 50;
 export const CRAWL_MAX_FINDING_STEPS = 64;
+/** Amendment 17: an intent or one idea is at most this many characters, and a crawl takes at most CRAWL_MAX_IDEAS ideas. */
+export const CRAWL_MAX_TEXT_CHARS = 2000;
+export const CRAWL_MAX_IDEAS = 64;
+export const CRAWL_POOLS = ['fleet', 'freestyle', 'auto'];
 /** Amendment 9: what a crawl has found so far (`exploring`) or its answer at the budget (`answered`), before the sealed manifest;
  * findings carry no images. A read may pass `waitAfter=<updatedAt>` to be held until the crawl changes (at most CRAWL_WAIT_MAX_MS). */
 export const CRAWL_WAIT_MAX_MS = 25_000;
@@ -11,3 +15,8 @@ export const CRAWL_WAIT_MAX_MS = 25_000;
  * (how many times the copy's calls reached the stand-in across the crawl's preparation and all its replays, not user actions). */
 export const CRAWL_MAX_STAND_IN_ROWS = 200;
 export const CRAWL_MAX_STAND_IN_LINES = 20;
+/** Amendment 18: the brief's bounds. */
+export const CRAWL_BRIEF_MAX_FUNCTIONS = 2000;
+export const CRAWL_BRIEF_MAX_GROUPS = 2000;
+/** Amendment 19: rows one brief function keeps per kind of production evidence (the rest counted). */
+export const CRAWL_BRIEF_MAX_PRODUCTION_ROWS = 100;

package/dist/commands/db-profile-upload.js

@@ -14,7 +14,7 @@ import chalk from 'chalk';
 import { DATABASE_PROFILE_UPLOAD_REQUEST_VERSION, parseDatabaseProfileUpload, } from './db-profile-contract.js';
 import { readDatabaseProfileFile } from './db-profile-show.js';
 import { resolveAuthContext } from '../utils/auth.js';
-import { classifyHttpError, haystackApiUrl } from '../utils/haystack-api.js';
+import { classifyHttpError, haystackApiUrl, readWithRetries, transientReadFailure } from '../utils/haystack-api.js';
 export const DATABASE_PROFILES_PATH = '/api/agent/cloud-verifier/database-profiles';
 /** GitHub's owner and repository name alphabets. */
 const REPOSITORY = /^([A-Za-z0-9](?:[A-Za-z0-9-]{0,38}))\/([A-Za-z0-9_.-]{1,100})$/u;
@@ -43,7 +43,23 @@ export function buildDatabaseProfileUploadRequest(file, options) {
         },
     };
 }
+// The service stores a profile by its content hash, so sending the same
+// request again after a dropped connection or a restarting service stores
+// nothing new: the upload is asked again on the same transient failures a
+// read is (readWithRetries: three more tries, 2 s, 5 s, 10 s apart).
 export async function uploadDatabaseProfile(request, token) {
+    try {
+        return await readWithRetries(() => uploadDatabaseProfileOnce(request, token));
+    }
+    catch (error) {
+        const transient = transientReadFailure(error);
+        if (transient === null)
+            throw error;
+        throw new Error(`Haystack API ${DATABASE_PROFILES_PATH} did not store the profile after 4 tries (${transient}). `
+            + 'Nothing was stored twice: run the same command again once the network or service is back.', { cause: error });
+    }
+}
+async function uploadDatabaseProfileOnce(request, token) {
     const response = await fetch(haystackApiUrl(DATABASE_PROFILES_PATH), {
         method: 'POST',
         headers: {
@@ -62,6 +78,9 @@ export async function uploadDatabaseProfile(request, token) {
         body = await response.json();
     }
     catch (error) {
+        // A connection dropped while the body is read is a transport failure, so the upload is asked again.
+        if (transientReadFailure(error) !== null)
+            throw error;
         throw new Error(`Haystack API ${DATABASE_PROFILES_PATH} returned a body that is not JSON: ${error instanceof Error ? error.message : String(error)}`);
     }
     const upload = parseDatabaseProfileUpload(body);

package/dist/commands/init.js

@@ -11,7 +11,9 @@
  *   2. a short marked block in AGENTS.md (and in CLAUDE.md when the repository
  *      keeps one) telling coding agents to run `haystack verify` after a change.
  * It then starts onboarding the app at the base `haystack verify` would use, so
- * the app is usually ready by the first verify. Running it again changes only
+ * the app is usually ready by the first verify. With --notes, the coding agent's
+ * notes on how the app runs, signs people in and is used (onboarding-notes.ts)
+ * become the repository's onboarding notes first, so onboarding plans from them. Running it again changes only
  * what is missing or out of date and shows where onboarding is.
  *
  * Exit codes: 0 set up (onboarding running or ready), 1 failed, 2 changes shown
@@ -30,7 +32,8 @@ import { HAYSTACK_APP_INSTALL_URL, HaystackApiError, INSTALLATION_REQUIRED_CODE
 import { findGitRoot } from '../utils/hooks.js';
 import { assertNoSymlinkOnPath } from '../utils/safe-write.js';
 import { CLAUDE_LOCAL_SETTINGS, CLAUDE_SHARED_SETTINGS, HAYSTACK_PRECOMPUTE_HOOK_COMMAND, claudeEntries, claudeSettingsPath, isCLIInstalled, isGitIgnored, isInstalledGlobally, isPrecomputeCommand, readClaudeSettings, } from './install-session-hooks.js';
-import { formatOnboarding, reportOnboardingState, startOnboarding } from './verify-onboarding.js';
+import { NOTE_FIELDS, readNotes } from './onboarding-notes.js';
+import { formatOnboarding, reportOnboardingState, startOnboarding, writeOnboardingNotes } from './verify-onboarding.js';
 import { deriveBaseSha, EXPLICIT_WALL_MS, resolveOriginRepository } from './verify-precompute.js';
 const NOTE_BEGIN = '<!-- haystack:begin -->';
 const NOTE_END = '<!-- haystack:end -->';
@@ -168,7 +171,10 @@ function ignoreRule(gitRoot, path) {
 }
 /** Claude Code's Stop hook in the per-developer settings, and the exclude line that keeps that file out of commits. */
 function planStopHook(gitRoot, notices) {
-    if (!isPlainDirectory(join(gitRoot, '.claude')) && !isCLIInstalled('claude'))
+    // Claude Code sets CLAUDECODE=1 in every command it runs, including from the desktop app and IDE
+    // extensions, where `claude` is often not on PATH.
+    const claudeCodeInUse = process.env.CLAUDECODE === '1' || isPlainDirectory(join(gitRoot, '.claude')) || isCLIInstalled('claude');
+    if (!claudeCodeInUse)
         return [];
     const settingsPath = `.claude/${CLAUDE_LOCAL_SETTINGS}`;
     const localPath = claudeSettingsPath(gitRoot, CLAUDE_LOCAL_SETTINGS);
@@ -328,11 +334,15 @@ export async function initCommand(options) {
         throw new Error('This checkout has no GitHub `origin` remote. Haystack verifies GitHub repositories: add the repository as `origin`, then run `haystack init` again.');
     }
     const repository = `${origin.owner}/${origin.repository}`;
+    // Read before anything else, so notes that cannot be used stop init with nothing done.
+    const notes = options.notes === undefined ? null : readNotes(options.notes);
     const plan = planInit(gitRoot);
+    let notesWritten = null;
     const finish = (status, onboarding, exitCode) => {
         if (options.json) {
             const changes = plan.changes.map(({ path, action, reason, diff }) => ({ path, action, reason, diff }));
-            process.stdout.write(`${JSON.stringify(withSchema('init', { status, repository, changes, notices: plan.notices, onboarding }), null, 2)}\n`);
+            process.stdout.write(`${JSON.stringify(withSchema('init', { status, repository, changes, notices: plan.notices, notes: notesWritten,
+                onboarding }), null, 2)}\n`);
         }
         process.exitCode = exitCode;
     };
@@ -347,6 +357,10 @@ export async function initCommand(options) {
     }
     for (const notice of plan.notices)
         say(`\n${chalk.yellow(notice)}`);
+    if (notes) {
+        const given = NOTE_FIELDS.filter(field => notes[field] !== '');
+        say(`\nOnboarding notes from ${options.notes === '-' ? 'stdin' : options.notes}: ${given.join(', ')}`);
+    }
     say('');
     let token;
     try {
@@ -373,6 +387,11 @@ export async function initCommand(options) {
     }
     let onboarding;
     try {
+        // The notes first: onboarding plans from the facts it starts with, and new notes are new facts (a new onboarding).
+        if (notes) {
+            notesWritten = await writeOnboardingNotes(repository, notes, token);
+            say(notesWritten === 'written' ? chalk.green('✓ Wrote the onboarding notes') : chalk.dim('The onboarding notes are already these.'));
+        }
         onboarding = await startOnboarding(repository, deriveBaseSha(gitRoot, Date.now() + EXPLICIT_WALL_MS), token);
     }
     catch (error) {

package/dist/commands/onboarding-notes.js

@@ -0,0 +1,63 @@
+/**
+ * The notes a coding agent gives onboarding (APP-ONBOARDING-V1 rule 7, amendment 18): what it knows of how this repository's
+ * app runs, signs people in and is used, read from a JSON file (`haystack init --notes <file>`, `-` for stdin):
+ *
+ *   { "run": "...", "signIn": "...", "workflow": "..." }
+ *
+ * Each is plain text; any may be left out, at least one must say something. Onboarding plans from them and still proves
+ * everything it takes from them.
+ */
+import { readFileSync } from 'node:fs';
+export const NOTE_FIELDS = ['run', 'signIn', 'workflow'];
+/** ONBOARDING_NOTE_MAX_CHARS in onboarding_contracts.ts: characters (code points) per note. */
+export const NOTE_MAX_CHARS = 8_000;
+export const NOTES_FORMAT = `{ "run": "how production installs, builds and starts the app", "signIn": "how a person gets an account and signs in, and the kinds of users", "workflow": "what a signed-in user mainly does" }`;
+/** The notes in a file's text, trimmed; an error that says what to fix otherwise. */
+export function parseNotes(text, source) {
+    let value;
+    try {
+        value = JSON.parse(text);
+    }
+    catch (error) {
+        throw new Error(`${source} is not JSON (${error instanceof Error ? error.message : String(error)}). Write ${NOTES_FORMAT}.`);
+    }
+    if (value === null || typeof value !== 'object' || Array.isArray(value))
+        throw new Error(`${source} is not a JSON object. Write ${NOTES_FORMAT}.`);
+    const row = value;
+    const unknown = Object.keys(row).filter(key => !NOTE_FIELDS.includes(key));
+    if (unknown.length)
+        throw new Error(`${source} has ${unknown.map(key => JSON.stringify(key)).join(', ')}; the notes are "run", "signIn" and "workflow".`);
+    const notes = { run: '', signIn: '', workflow: '' };
+    for (const field of NOTE_FIELDS) {
+        const note = row[field];
+        if (note === undefined)
+            continue;
+        if (typeof note !== 'string')
+            throw new Error(`${source}: "${field}" is not text.`);
+        if (note.includes('\0'))
+            throw new Error(`${source}: "${field}" contains a NUL character.`);
+        const trimmed = note.trim();
+        const length = [...trimmed].length;
+        if (length > NOTE_MAX_CHARS)
+            throw new Error(`${source}: "${field}" is ${length} characters; a note is at most ${NOTE_MAX_CHARS}.`);
+        notes[field] = trimmed;
+    }
+    if (NOTE_FIELDS.every(field => notes[field] === ''))
+        throw new Error(`${source} says nothing: give at least one of "run", "signIn" and "workflow".`);
+    return notes;
+}
+/** `--notes <file>`: the notes in that file, or in stdin for `-`. */
+export function readNotes(path) {
+    const source = path === '-' ? 'The notes on stdin' : `The notes file ${path}`;
+    let text;
+    try {
+        text = readFileSync(path === '-' ? 0 : path, 'utf8');
+    }
+    catch (error) {
+        throw new Error(`${source} could not be read: ${error instanceof Error ? error.message : String(error)}`);
+    }
+    return parseNotes(text, source);
+}
+export function sameNotes(a, b) {
+    return a !== undefined && NOTE_FIELDS.every(field => a[field] === b[field]);
+}

package/dist/commands/telemetry.d.ts

@@ -0,0 +1,154 @@
+export declare function preloadBabel(): Promise<void>;
+declare const MANIFEST_SCHEMA_VERSION = "haystack-node-telemetry-instrumentation-v3";
+export interface TelemetryInstrumentOptions {
+    entry: string;
+    sourceRoot?: string;
+    json?: boolean;
+    /**
+     * Experimental exact symbol allow-list. Entries use the same structured
+     * sourcePath#qualifiedName coordinates emitted in the instrumentation
+     * manifest. No fuzzy name matching is performed.
+     */
+    includeSymbols?: Array<{
+        sourcePath: string;
+        qualifiedName: string;
+    }>;
+}
+export interface InstrumentedFileSummary {
+    output_path: string;
+    source_path: string;
+    probes: {
+        branches: number;
+        parameters: number;
+        bindings: number;
+        returns: number;
+        throws: number;
+    };
+}
+export interface TelemetryInstrumentationResult {
+    schema_version: typeof MANIFEST_SCHEMA_VERSION;
+    build_id: string;
+    dist_directory: string;
+    entry: string;
+    runtime: string;
+    instrumented_files: InstrumentedFileSummary[];
+    sites: InstrumentedSite[];
+    skipped_unmapped_files: string[];
+    already_instrumented: boolean;
+}
+export interface InstrumentedSite {
+    probe_key: string;
+    site_id: string;
+    source_path: string;
+    qualified_name: string;
+    probe_kind: 'condition' | 'parameter' | 'binding' | 'return' | 'throw';
+    owner_key: string;
+    structural_key: string;
+    label?: string;
+    allowed_fields: string[];
+    /**
+     * 1-based line and 0-based column of the probed node in `source_path`.
+     * Present only when the instrumented text IS the source (the bundler
+     * loader). Compiled-output instrumentation has no source map, so its
+     * positions would be output coordinates and are omitted rather than
+     * mislabelled; consumers join those sites by name only.
+     */
+    line?: number;
+    column?: number;
+}
+export interface FileProbeCounts {
+    branches: number;
+    parameters: number;
+    bindings: number;
+    returns: number;
+    throws: number;
+}
+/**
+ * The runtime source bound to one control channel, plus the integrity token
+ * generated probes present to it and the hash a bootstrap verifies before
+ * evaluating it. Deterministic in the channel, so a bundler loader running in
+ * several worker processes and the step that later writes the runtime agree
+ * without exchanging anything.
+ */
+export declare function authenticateTelemetryRuntime(controlChannel: string): {
+    source: string;
+    integrity: string;
+    hash: string;
+};
+export declare function instrumentNodeTelemetryBuild(distDirectory: string, options: TelemetryInstrumentOptions): TelemetryInstrumentationResult;
+export declare const BUNDLED_SOURCE_EXTENSIONS: string[];
+export type BundledModuleResult = {
+    kind: 'instrumented';
+    code: string;
+    probes: FileProbeCounts;
+    sites: InstrumentedSite[];
+}
+/** Already instrumented, or a "use client" module (its server copy is only the SSR render of browser code). */
+ | {
+    kind: 'unchanged';
+}
+/** Babel cannot parse it (syntax the bundler's own compiler accepts but Babel does not). */
+ | {
+    kind: 'unparsed';
+    reason: string;
+};
+/**
+ * Instrument one source module inside a bundler. `sourcePath` is the
+ * repository-relative path every site records, so production counts join the
+ * change's files and line ranges exactly. A module Babel cannot parse is
+ * reported, not thrown: the bundler's compiler may accept syntax Babel does
+ * not, and telemetry must never be the reason a build fails. Any other error
+ * is ours and stays loud.
+ */
+export declare function instrumentBundledModule(code: string, options: {
+    sourcePath: string;
+    controlChannel: string;
+    runtimeIntegrity: string;
+}): BundledModuleResult;
+export interface BundledTelemetryManifest {
+    schema_version: typeof MANIFEST_SCHEMA_VERSION;
+    build_id: string;
+    bundler: string;
+    source_root: string;
+    /** Bundled builds have no instrumented entry file; the preload installs. */
+    entry: null;
+    runtime: string;
+    instrumented_files: Array<{
+        source_path: string;
+        probes: FileProbeCounts;
+    }>;
+    /** Server modules shipped uninstrumented because Babel could not parse them, with the parser's reason. */
+    skipped_unparsed_files: Array<{
+        source_path: string;
+        reason: string;
+    }>;
+    sites: InstrumentedSite[];
+}
+/**
+ * Write the runtime, its manifest and the preload that installs it into
+ * `<outputDirectory>/.haystack-telemetry/`. The runtime reads the manifest
+ * beside its own real file; nothing here depends on where a bundler put the
+ * chunks. Returns the preload's absolute path for NODE_OPTIONS=--require.
+ */
+export declare function writeBundledTelemetryRuntime(options: {
+    outputDirectory: string;
+    controlChannel: string;
+    bundler: string;
+    sourceRoot: string;
+    modules: Array<{
+        sourcePath: string;
+        probes: FileProbeCounts;
+        sites: InstrumentedSite[];
+    }>;
+    unparsed: Array<{
+        sourcePath: string;
+        reason: string;
+    }>;
+}): {
+    registerPath: string;
+    manifest: BundledTelemetryManifest;
+};
+export declare function telemetryInstrumentCommand(distDirectory: string, options: TelemetryInstrumentOptions & {
+    includeSymbolsFile?: string;
+}): Promise<void>;
+export {};