@haystackeditor/cli 0.17.1 → 0.19.0

package/dist/commands/verify-precompute.js

@@ -1,22 +1,23 @@
 import { spawn, execFileSync } from 'node:child_process';
 import { createHash, randomUUID } from 'node:crypto';
-import { closeSync, constants, cpSync, existsSync, fsyncSync, lstatSync, mkdirSync, mkdtempSync, openSync, renameSync, rmSync, writeFileSync } from 'node:fs';
+import { closeSync, constants, cpSync, existsSync, fsyncSync, lstatSync, mkdirSync, mkdtempSync, openSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs';
 import { createGzip } from 'node:zlib';
 import { isAbsolute, join, resolve } from 'node:path';
 import { tmpdir } from 'node:os';
 import chalk from 'chalk';
 import { NotLoggedInError, resolveAuthContext } from '../utils/auth.js';
-import { findGitRoot, parseRemoteUrl, resolveDiffBaseRef, } from '../utils/git.js';
+import { findGitRoot, parseRemoteUrl, } from '../utils/git.js';
+import { resolveVerifyBaseSha } from '../utils/verify-base.js';
 import { HaystackApiError, haystackJson, REPO_NOT_ENTITLED_CODE, SUBSCRIPTION_REQUIRED_CODE, } from '../utils/haystack-api.js';
 import { trackPrecomputeEvent } from '../utils/telemetry.js';
 import { findSecretsInPatch, isSecretBearingPath } from '../utils/secret-paths.js';
 import { startPrecomputeDelivery } from './precompute-delivery.js';
-import { captureCombinationSearchHook, readCombinationSearchHookStatus } from './combination-search-hook.js';
+import { CRAWL_BUDGET_MAX_MS, CRAWL_BUDGET_MIN_MS, CRAWL_TITLE_MAX_CHARS } from './crawl-contract.js';
 // --hook has five seconds to capture the exact source and publish local custody.
 // Its detached sender owns the bounded network delivery and acknowledgement.
 // An explicit invocation still waits synchronously for the server response.
 const HOOK_WALL_MS = 5_000;
-const EXPLICIT_WALL_MS = 60_000;
+export const EXPLICIT_WALL_MS = 60_000;
 const REMOTE_COMMAND_MS = 750;
 // The published CLI cannot import verifier/contracts outside its package root.
 const hasOnlyCharacters = (value, allowed) => value.length > 0
@@ -79,50 +80,29 @@ function resolveOriginRepository(gitRoot) {
         throw new PrecomputeFailure('No supported origin repository was found.');
     }
 }
-function resolveOriginDefaultBranch(gitRoot, deadline) {
+/** The shared base rule (utils/verify-base.ts), bounded by the hook's deadline. */
+function deriveBaseSha(gitRoot, deadline) {
     try {
-        const remoteHead = gitOutput(['symbolic-ref', '--short', 'refs/remotes/origin/HEAD'], gitRoot, deadline);
-        if (remoteHead.startsWith('origin/') && remoteHead.length > 'origin/'.length) {
-            return remoteHead.slice('origin/'.length);
-        }
-    }
-    catch {
-        // A clone does not have to carry origin/HEAD; ask the remote below.
+        return resolveVerifyBaseSha('HEAD', {
+            run: (args, options) => gitOutput(args, gitRoot, deadline, options?.remote
+                ? { env: { ...process.env, GIT_TERMINAL_PROMPT: '0' }, timeoutCapMs: REMOTE_COMMAND_MS } : {}),
+            fetchOptions: () => ({ cwd: gitRoot, timeoutMs: Math.max(1, Math.min(remainingMs(deadline), REMOTE_COMMAND_MS)) }),
+        });
     }
-    const remoteRefs = gitOutput(['ls-remote', '--symref', 'origin', 'HEAD'], gitRoot, deadline, {
-        env: { ...process.env, GIT_TERMINAL_PROMPT: '0' },
-        timeoutCapMs: REMOTE_COMMAND_MS,
-    });
-    const prefix = 'ref: refs/heads/';
-    const heads = remoteRefs.split('\n').map(line => line.split('\t'))
-        .filter(fields => fields.length === 2 && fields[1] === 'HEAD' && fields[0].startsWith(prefix))
-        .map(fields => fields[0].slice(prefix.length));
-    if (heads.length !== 1 || heads[0].length === 0) {
-        throw new PrecomputeFailure('Could not resolve the origin default branch.');
-    }
-    gitOutput(['check-ref-format', `refs/heads/${heads[0]}`], gitRoot, deadline);
-    return heads[0];
-}
-function deriveBaseSha(gitRoot, deadline) {
-    const defaultBranch = resolveOriginDefaultBranch(gitRoot, deadline);
-    const baseRef = resolveDiffBaseRef(defaultBranch, {
-        cwd: gitRoot,
-        timeoutMs: Math.max(1, Math.min(remainingMs(deadline), REMOTE_COMMAND_MS)),
-    });
-    const baseSha = gitOutput(['merge-base', 'HEAD', baseRef], gitRoot, deadline);
-    if (!isCommitSha(baseSha)) {
-        throw new PrecomputeFailure('Could not derive the base commit.');
+    catch (error) {
+        if (error instanceof PrecomputeFailure)
+            throw error;
+        throw new PrecomputeFailure(error instanceof Error ? error.message : 'Could not derive the base commit.');
     }
-    return baseSha;
 }
 // The -U0 scan text omits binary content and context, so it stays far below
 // the uploaded patch; this only guards against git's 1 MiB default.
 const SCAN_PATCH_MAX_BYTES = 256 * 1024 * 1024;
 /**
  * Withholds secret-bearing changes from the snapshot index before the tree is
- * written, so the tree, work commit, uploaded patch and combination-search
- * patches all agree. A withheld path keeps its base version (or is absent when
- * base has none). Every withheld path is named on stderr; secret values never are.
+ * written, so the tree, work commit and uploaded patch all agree. A withheld
+ * path keeps its base version (or is absent when base has none). Every
+ * withheld path is named on stderr; secret values never are.
  */
 function withholdSecretBearingChanges(gitRoot, baseSha, indexEnv, deadline) {
     const changed = gitOutput(['diff', '--cached', '--no-renames', '--diff-filter=d', '--name-only', '-z', baseSha, '--'], gitRoot, deadline, { env: indexEnv, maxBuffer: SCAN_PATCH_MAX_BYTES, raw: true }).split('\0').filter(path => path.length > 0);
@@ -186,17 +166,40 @@ function deriveWorkCommit(gitRoot, baseSha, treeSha, deadline) {
     }
     // A clean checkout of the base already has the exact required identity.
     if (baseSha === headSha && treeSha === headTreeSha)
-        return headSha;
+        return { workCommit: headSha, headSha, headTreeSha };
     const workCommit = gitOutput(['commit-tree', treeSha, '-p', baseSha, '-m', 'front-half-snapshot'], gitRoot, deadline, { env: { ...process.env, ...FIXED_COMMIT_ENV } });
     if (!isCommitSha(workCommit)) {
         throw new PrecomputeFailure('Could not derive the working commit.');
     }
-    return workCommit;
+    return { workCommit, headSha, headTreeSha };
+}
+/**
+ * CrawlSubmission.changeTitle (CRAWL-V1): HEAD's subject when HEAD differs
+ * from base, then " (with uncommitted changes)" when the captured tree differs
+ * from HEAD's; "Uncommitted changes" when HEAD is base. A subject too long for
+ * 200 characters is cut at a character boundary, never the suffix.
+ */
+function deriveChangeTitle(gitRoot, baseSha, treeSha, commits, deadline) {
+    if (commits.headSha === baseSha)
+        return 'Uncommitted changes';
+    // A commit with an empty message is named by its id, so the title is never empty.
+    const subject = gitOutput(['log', '-1', '--no-show-signature', '--encoding=UTF-8', '--format=%s', commits.headSha, '--'], gitRoot, deadline, { raw: true }).replace(/\n$/, '') || `Commit ${commits.headSha.slice(0, 12)}`;
+    const suffix = treeSha === commits.headTreeSha ? '' : ' (with uncommitted changes)';
+    let title = '';
+    for (const character of subject) {
+        if (title.length + character.length + suffix.length > CRAWL_TITLE_MAX_CHARS)
+            break;
+        title += character;
+    }
+    return title + suffix;
 }
 function slimInputHash() {
-    // Mirrors printf '%s\0%s\0%s\0%s\0' slim agent-deferred none none.
+    // Mirrors front_half_cache_derive in blast-cache.sh for a slim bundle:
+    // printf '%s\0%s\0%s\0%s\0%s\0' slim agent-deferred none none crawl-blast-v1.
+    // The fifth field (CRAWL-V1 rule 4) keeps bundles written before
+    // crawl-blast.json existed from being restored for a crawl.
     return createHash('sha256')
-        .update(Buffer.from('slim\0agent-deferred\0none\0none\0'))
+        .update(Buffer.from('slim\0agent-deferred\0none\0none\0crawl-blast-v1\0'))
         .digest('hex');
 }
 async function buildCompressedPatch(gitRoot, baseSha, workCommit, deadline) {
@@ -256,13 +259,17 @@ async function buildCompressedPatch(gitRoot, baseSha, workCommit, deadline) {
         child.stdout.pipe(gzip);
     });
 }
-export async function derivePrecomputeRequest(gitRoot, owner, repository, deadline = Date.now() + HOOK_WALL_MS) {
+export async function derivePrecomputeRequest(gitRoot, owner, repository, deadline = Date.now() + HOOK_WALL_MS, 
+/** How long the crawl may take (CRAWL-V1 amendment 8); undefined: the service's default. */
+budgetMs) {
     // Capture this before git and compression work so delivery latency cannot
     // make an older stop look newer than a later one.
     const observedAt = Date.now();
     const baseSha = deriveBaseSha(gitRoot, deadline);
     const treeSha = deriveWorktreeTreeSha(gitRoot, baseSha, deadline);
-    const workCommit = deriveWorkCommit(gitRoot, baseSha, treeSha, deadline);
+    const commits = deriveWorkCommit(gitRoot, baseSha, treeSha, deadline);
+    const { workCommit } = commits;
+    const changeTitle = deriveChangeTitle(gitRoot, baseSha, treeSha, commits, deadline);
     const patchGzip = await buildCompressedPatch(gitRoot, baseSha, workCommit, deadline);
     const cacheKey = [
         baseSha.slice(0, 12),
@@ -271,7 +278,6 @@ export async function derivePrecomputeRequest(gitRoot, owner, repository, deadli
         'slim',
         slimInputHash(),
     ].join('-');
-    const combinationSearch = captureCombinationSearchHook(gitRoot, `${owner}/${repository}`, baseSha, workCommit, deadline);
     return {
         treeSha,
         request: {
@@ -283,7 +289,7 @@ export async function derivePrecomputeRequest(gitRoot, owner, repository, deadli
             cacheKey,
             patchGzBase64: patchGzip.toString('base64'),
             patchSha256: createHash('sha256').update(patchGzip).digest('hex'),
-            ...(combinationSearch ? { combinationSearch } : {}),
+            ...(workCommit === baseSha ? {} : { crawl: { treeSha, changeTitle, ...(budgetMs === undefined ? {} : { budgetMs }) } }),
         },
     };
 }
@@ -291,16 +297,17 @@ function precomputeResult(response) {
     if (!response || typeof response !== 'object' || Array.isArray(response)) {
         throw new PrecomputeFailure('The precompute service returned an invalid response.');
     }
-    const { status, runId, walker: rawWalker, search: rawSearch } = response;
+    const { status, runId, walker: rawWalker, crawl: rawCrawl } = response;
     const walker = rawWalker === undefined ? undefined : readWalkerStatus(rawWalker);
-    const search = rawSearch === undefined ? undefined : readCombinationSearchHookStatus(rawSearch);
+    const crawl = rawCrawl === undefined ? undefined : readCrawlHookStatus(rawCrawl);
+    const reported = { ...(walker === undefined ? {} : { walker }), ...(crawl === undefined ? {} : { crawl }) };
     if (status === 'queued' && typeof runId === 'string' && isRunId(runId)) {
-        return { outcome: 'enqueued', runId, ...(walker === undefined ? {} : { walker }), ...(search === undefined ? {} : { search }) };
+        return { outcome: 'enqueued', runId, ...reported };
     }
     if (status === 'cached')
-        return { outcome: 'cached', ...(walker === undefined ? {} : { walker }), ...(search === undefined ? {} : { search }) };
+        return { outcome: 'cached', ...reported };
     if (status === 'existing')
-        return { outcome: 'existing', ...(walker === undefined ? {} : { walker }), ...(search === undefined ? {} : { search }) };
+        return { outcome: 'existing', ...reported };
     throw new PrecomputeFailure('The precompute service returned an invalid response.');
 }
 /** One authenticated submission of the already captured request. No source reads or retries. */
@@ -325,6 +332,31 @@ function readWalkerStatus(value) {
     }
     throw new PrecomputeFailure('The precompute service returned an invalid walker status.');
 }
+const CRAWL_UNAVAILABLE_REASONS = ['not-configured', 'runner-unavailable', 'profile-ambiguous', 'superseded', 'enqueue-failed'];
+function readCrawlHookStatus(value) {
+    if (value && typeof value === 'object' && !Array.isArray(value)) {
+        const record = value;
+        if (Object.keys(record).length === 2) {
+            if ((record.status === 'queued' || record.status === 'existing')
+                && typeof record.runId === 'string' && isRunId(record.runId))
+                return { status: record.status, runId: record.runId };
+            const reason = CRAWL_UNAVAILABLE_REASONS.find(known => known === record.reason);
+            if (record.status === 'unavailable' && reason !== undefined)
+                return { status: record.status, reason };
+        }
+    }
+    throw new PrecomputeFailure('The precompute service returned an invalid crawl status.');
+}
+/** Why the service could not start a crawl, in plain words. */
+export function crawlUnavailableText(reason) {
+    switch (reason) {
+        case 'not-configured': return 'this repository is not set up for crawling yet';
+        case 'runner-unavailable': return 'no machine that can crawl this repository is available right now';
+        case 'profile-ambiguous': return 'the machines that crawl this repository disagree about how to run it';
+        case 'superseded': return 'a newer stop in this repository came first';
+        case 'enqueue-failed': return 'the service could not queue it';
+    }
+}
 function walkerMetadataDirectory(gitRoot, deadline) {
     const common = gitOutput(['rev-parse', '--git-common-dir'], gitRoot, deadline);
     return join(isAbsolute(common) ? common : resolve(gitRoot, common), 'haystack');
@@ -361,7 +393,8 @@ function saveWalkerStatus(directory, request, walker) {
         rmSync(temporary, { force: true });
     }
 }
-function normalModeFailure(error) {
+/** The one-line error an explicit (non-hook) submission reports. */
+export function normalModeFailure(error) {
     if (error instanceof PrecomputeFailure)
         return error;
     if (error instanceof NotLoggedInError) {
@@ -389,7 +422,8 @@ function normalModeFailure(error) {
     }
     return new PrecomputeFailure('The precompute request failed.');
 }
-function printResult(outcome, runId, walker, search) {
+function printResult(result) {
+    const { outcome, runId, walker, crawl } = result;
     if (outcome === 'enqueued') {
         console.log(chalk.green(`Precompute request queued${runId ? ` (${runId})` : ''}.`));
     }
@@ -409,10 +443,72 @@ function printResult(outcome, runId, walker, search) {
             : 'Walker unavailable: the enqueue request failed.');
     else
         console.log('Walker status was not reported by the service.');
-    if (search?.status === 'queued' || search?.status === 'existing')
-        console.log(`Browser search ${search.status} (${search.runId}).`);
-    else if (search?.status === 'unavailable' && search.reason !== 'not-configured')
-        console.log(`Browser search unavailable: ${search.reason}.`);
+    if (crawl?.status === 'queued')
+        console.log(`Crawl queued (${crawl.runId}).`);
+    else if (crawl?.status === 'existing')
+        console.log(`A matching crawl already exists (${crawl.runId}).`);
+    else if (crawl?.status === 'unavailable')
+        console.log(`Crawl unavailable: ${crawlUnavailableText(crawl.reason)}.`);
+    else
+        console.log('Crawl status was not reported by the service.');
+}
+/** The hook, an explicit precompute and `haystack verify` all capture here.
+ * The repository defaults to origin's, as the hook's does. */
+/** The repository's crawl time: .haystack.json crawl.minutes at the git root, whole minutes from 1 to 30 (CRAWL-V1 amendment 8),
+ * or undefined when the file or the field is absent. A value out of range fails the capture, never silently becomes the default. */
+function configuredCrawlMinutes(gitRoot) {
+    const path = join(gitRoot, '.haystack.json');
+    if (!existsSync(path))
+        return undefined;
+    const minutes = JSON.parse(readFileSync(path, 'utf8')).crawl?.minutes;
+    if (minutes === undefined)
+        return undefined;
+    if (typeof minutes !== 'number' || !Number.isInteger(minutes) || minutes * 60_000 < CRAWL_BUDGET_MIN_MS || minutes * 60_000 > CRAWL_BUDGET_MAX_MS) {
+        throw new PrecomputeFailure(`.haystack.json crawl.minutes must be a whole number from ${CRAWL_BUDGET_MIN_MS / 60_000} to ${CRAWL_BUDGET_MAX_MS / 60_000}.`);
+    }
+    return minutes;
+}
+export async function captureCheckout(deadline, repository, 
+/** `haystack verify --minutes`; undefined: the repository's .haystack.json crawl.minutes, else the service's default. */
+minutes) {
+    const gitRoot = findGitRoot();
+    if (!gitRoot) {
+        throw new PrecomputeFailure('Run this command from a git repository.');
+    }
+    const metadataDirectory = walkerMetadataDirectory(gitRoot, deadline);
+    const target = repository ?? resolveOriginRepository(gitRoot);
+    const chosen = minutes ?? configuredCrawlMinutes(gitRoot);
+    const derivation = await derivePrecomputeRequest(gitRoot, target.owner, target.repository, deadline, chosen === undefined ? undefined : chosen * 60_000);
+    return { metadataDirectory, derivation };
+}
+/** The explicit (non-hook) submission of a capture: one synchronous
+ * authenticated POST, bounded by the deadline, and its walker status saved.
+ * `haystack verify precompute` and `haystack verify` both submit here. */
+export async function postPrecomputeCapture(capture, token, deadline) {
+    const controller = new AbortController();
+    const fetchTimer = setTimeout(() => controller.abort(), remainingMs(deadline));
+    let response;
+    try {
+        response = await haystackJson('/api/agent/cloud-verifier/front-half', token, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify(capture.derivation.request),
+            signal: controller.signal,
+        });
+    }
+    finally {
+        clearTimeout(fetchTimer);
+    }
+    const result = precomputeResult(response);
+    if (result.walker) {
+        try {
+            saveWalkerStatus(capture.metadataDirectory, capture.derivation.request, result.walker);
+        }
+        catch {
+            throw new PrecomputeFailure('Precompute response received, but its walker status could not be saved.');
+        }
+    }
+    return result;
 }
 export async function verifyPrecomputeCommand(options) {
     const startedAt = Date.now();
@@ -420,61 +516,29 @@ export async function verifyPrecomputeCommand(options) {
     const deadline = startedAt + totalWallMs;
     let outcome = 'error';
     try {
-        const gitRoot = findGitRoot();
-        if (!gitRoot) {
-            throw new PrecomputeFailure('Run this command from a git repository.');
-        }
-        const metadataDirectory = walkerMetadataDirectory(gitRoot, deadline);
-        const repository = resolveOriginRepository(gitRoot);
-        const derivation = await derivePrecomputeRequest(gitRoot, repository.owner, repository.repository, deadline);
+        const capture = await captureCheckout(deadline);
         if (options.hook) {
             remainingMs(deadline);
-            await startPrecomputeDelivery(metadataDirectory, derivation.request);
+            await startPrecomputeDelivery(capture.metadataDirectory, capture.derivation.request);
             outcome = 'pending';
             return;
         }
         let token;
         try {
             const auth = await resolveAuthContext({
-                owner: repository.owner,
-                repo: repository.repository,
+                owner: capture.derivation.request.owner,
+                repo: capture.derivation.request.repository,
             });
             token = auth.token;
         }
         catch (error) {
-            if (error instanceof NotLoggedInError) {
+            if (error instanceof NotLoggedInError)
                 outcome = 'skipped_no_auth';
-                if (options.hook)
-                    return;
-            }
             throw error;
         }
-        const controller = new AbortController();
-        const fetchTimer = setTimeout(() => controller.abort(), remainingMs(deadline));
-        let response;
-        try {
-            response = await haystackJson('/api/agent/cloud-verifier/front-half', token, {
-                method: 'POST',
-                headers: { 'Content-Type': 'application/json' },
-                body: JSON.stringify(derivation.request),
-                signal: controller.signal,
-            });
-        }
-        finally {
-            clearTimeout(fetchTimer);
-        }
-        const result = precomputeResult(response);
+        const result = await postPrecomputeCapture(capture, token, deadline);
         outcome = result.outcome;
-        if (result.walker) {
-            try {
-                saveWalkerStatus(metadataDirectory, derivation.request, result.walker);
-            }
-            catch {
-                throw new PrecomputeFailure('Precompute response received, but its walker status could not be saved.');
-            }
-        }
-        if (!options.hook)
-            printResult(outcome, result.runId, result.walker, result.search);
+        printResult(result);
     }
     catch (error) {
         if (!options.hook)