@haystackeditor/cli 0.24.1 → 0.25.0

package/dist/commands/init-capture.js

@@ -0,0 +1,409 @@
+/**
+ * Browser capture's part of `haystack init` (CAPTURE-V1 rules 8a, 8b with 1, 2, 4 and 13a): the tag, its consent wiring,
+ * the build step that publishes the route manifest, the app's registration, and the CSP report.
+ *
+ * It is planned inside the telemetry plan (init-telemetry.ts), which chooses the app (rule 8a), asks for answers that are
+ * missing, chains capture's package.json build step with the server part's edits, and pins the CLI once, as a production
+ * dependency, for every part that runs it. Every change here is an InitChange, shown first and made with the rest only
+ * with --yes, all or nothing.
+ *
+ * Init never guesses (rule 8): the production origins and the consent answer are the user's (--origin, --consent), kept in
+ * .haystack/capture.json so a rerun (a newer script version) needs no flags; a framework, version, layout or config init
+ * cannot read is a manual step with the exact tag, manifest command and CSP lines and where init looked. Registration
+ * (POST /api/agent/cloud-verifier/capture/key) is an action the preview announces: until it runs (after --yes, before any
+ * write) the diffs show the key as a placeholder, and init plans again with the key it issues.
+ *
+ * Do no harm (rule 13a) in what init writes: the script is self-hosted, async (Next.js: lazyOnload) and pinned by
+ * integrity; consent integrations are static files that never throw; the build step is the pinned CLI's
+ * `haystack-capture-step`, joined only to a build script of plain `&&`-chained commands, which exits 0 whatever happens to
+ * the manifest or the CLI; init never edits a Content-Security-Policy. A newer script version
+ * replaces the old file (deleted) and tag.
+ */
+import chalk from 'chalk';
+import { existsSync, readdirSync, readFileSync } from 'node:fs';
+import { isAbsolute, join, posix, relative, sep } from 'node:path';
+import { detectApp } from '../capture/adapters/index.js';
+import { RAKE_FILE, RAKE_FILE_CONTENT } from '../capture/adapters/rails.js';
+import { isManual } from '../capture/adapters/types.js';
+import { captureConfigFile, formatCaptureConfig, parseOrigins, readCaptureConfig, readCaptureRegistration, registerCaptureApp } from '../capture/app-config.js';
+import { consentLabel, consentScript, formatConsentChoice, manualConsentStep, parseConsentChoice } from '../capture/consent.js';
+import { findCsp } from '../capture/csp.js';
+import { applyEdits, keyName, loadBabel, parseJsonLike } from '../capture/js-ast.js';
+import { buildManifest, exactRoutes, manifestDocument, MANIFEST_DIR } from '../capture/manifest.js';
+import { appContext } from '../capture/project.js';
+import { loadCaptureScript } from '../capture/script-release.js';
+import { htmlTags, STATIC_SUBDIR } from '../capture/tag.js';
+import { CAPTURE_ORIGIN, CAPTURE_VERSION } from './capture-contract.js';
+import { HaystackApiError } from '../utils/haystack-api.js';
+import { lineDiff } from './init.js';
+/** Shown in place of the key until registration issues it (with --yes). */
+const KEY_PLACEHOLDER = 'hc_(issued-when-init-runs-with---yes)';
+const ID_PLACEHOLDER = '(issued-when-init-runs-with---yes)';
+/** The build step: the pinned CLI's dependency-free step bin (src/capture-step.ts), which runs the manifest command and
+ * always exits 0, so neither the manifest nor the CLI failing to start can fail the build (rule 13). */
+const BUILD_COMMAND = 'haystack-capture-step';
+const EARLIER_BUILD_COMMAND = 'haystack capture manifest --app .';
+/** A build script init composes with: commands joined by `&&` and nothing else, of plain words. Any other shell syntax (`;`,
+ * `||`, `|`, `&`, quotes, redirects, variables, subshells, comments) could change what running a command after it means,
+ * so it is a manual step. */
+const COMPOSABLE_SCRIPT = /^[A-Za-z0-9_./=:@\s&-]+$/;
+const STAGED_DIR = `.haystack/${STATIC_SUBDIR}`;
+/** The answers in effect: the flags, else the ones init recorded in .haystack/capture.json (rule 8b: the user's answers,
+ * never inferred, so a rerun that only updates the script needs none). */
+export function captureAnswers(appDir, origins, consent) {
+    const recorded = readCaptureConfig(appDir);
+    return { origins: origins.length > 0 ? origins : recorded?.origins ?? [], consent: consent ?? recorded?.consent };
+}
+function readText(path) {
+    try {
+        return readFileSync(path, 'utf8');
+    }
+    catch (error) {
+        if (error.code === 'ENOENT')
+            return null;
+        throw error;
+    }
+}
+class Planner {
+    gitRoot;
+    changes = [];
+    constructor(gitRoot) {
+        this.gitRoot = gitRoot;
+    }
+    text(target, content, reason) {
+        const before = readText(target);
+        if (before === content)
+            return;
+        this.changes.push({ path: this.display(target), action: before === null ? 'create' : 'update', reason,
+            diff: lineDiff(before ?? '', content), target, content });
+    }
+    /** A file whose content is not worth reading in a diff (the minified tag): its size and integrity instead. */
+    summarized(target, content, reason, summary) {
+        const before = readText(target);
+        if (before === content)
+            return;
+        this.changes.push({ path: this.display(target), action: before === null ? 'create' : 'update', reason, diff: [`+${summary}`], target, content });
+    }
+    remove(target, reason) {
+        this.changes.push({ path: this.display(target), action: 'delete', reason, diff: ['-(the whole file)'], target, content: '' });
+    }
+    display(target) {
+        const inside = relative(this.gitRoot, target);
+        return inside.startsWith('..') || isAbsolute(inside) ? target : inside.split(sep).join('/');
+    }
+}
+/** package.json with the manifest command joined to `script`: after it or before it, once. Edited at the script's string
+ * node, so the rest of the file keeps its formatting. */
+function withBuildStep(text, file, script, phase) {
+    const ast = parseJsonLike(text, file);
+    const scripts = ast?.type === 'ObjectExpression'
+        ? ast.properties.find((property) => property.type === 'ObjectProperty' && keyName(property) === 'scripts')?.value : null;
+    const entry = scripts?.type === 'ObjectExpression'
+        ? scripts.properties.find((property) => property.type === 'ObjectProperty' && keyName(property) === script) : null;
+    if (!entry || entry.value.type !== 'StringLiteral')
+        return { manual: `package.json has no ${script} script init can read` };
+    const command = entry.value.value;
+    if (command.split('&&').some(segment => segment.trim() === BUILD_COMMAND))
+        return text;
+    // An earlier build step (the manifest command itself, which a CLI that cannot start would let fail the build) is
+    // replaced by the step in place.
+    if (command.split('&&').some(segment => segment.trim() === EARLIER_BUILD_COMMAND)) {
+        const replaced = command.split('&&').map(segment => (segment.trim() === EARLIER_BUILD_COMMAND ? segment.replace(EARLIER_BUILD_COMMAND, BUILD_COMMAND) : segment)).join('&&');
+        return applyEdits(text, [{ start: entry.value.start, end: entry.value.end, text: JSON.stringify(replaced) }]);
+    }
+    const segments = command.split('&&').map(segment => segment.trim());
+    if (!COMPOSABLE_SCRIPT.test(command) || segments.some(segment => segment === '' || segment.includes('&'))) {
+        return { manual: `its ${script} script (\`${command}\`) uses shell syntax init does not compose with; add \`${BUILD_COMMAND}\` to run ${phase} it` };
+    }
+    const next = phase === 'after' ? `${command} && ${BUILD_COMMAND}` : `${BUILD_COMMAND} && ${command}`;
+    return applyEdits(text, [{ start: entry.value.start, end: entry.value.end, text: JSON.stringify(next) }]);
+}
+function scriptFiles(dir) {
+    try {
+        return readdirSync(dir).filter(file => /^(capture\.[0-9a-z]+|consent-[a-z]+\.[0-9a-f]+)\.js$/.test(file));
+    }
+    catch {
+        return [];
+    }
+}
+function manifestCommand(appPath) {
+    return `haystack capture manifest --app ${appPath}`;
+}
+function cspFallbackLine() {
+    return `connect-src 'self' ${CAPTURE_ORIGIN}; script-src 'self'`;
+}
+function part(steps) {
+    return steps.length > 0 ? { status: 'manual', steps, received: null } : { status: 'installed', received: null };
+}
+/** Everything capture would change for this app with these answers; reads only. Throws on answers that are not ones
+ * (an origin that is not https, an unknown consent form), so init stops before anything changes. `registration` is the
+ * registration's outcome when init plans again after it. */
+export async function planCapture(input) {
+    const { gitRoot, appDir, repository } = input;
+    const origins = parseOrigins(input.answers.origins);
+    if (input.answers.consent === undefined)
+        throw new Error('planCapture needs the consent answer.');
+    const consent = parseConsentChoice(input.answers.consent);
+    const plan = { changes: [], packageText: null, needsCli: false, registration: null, notices: [], part: part([]) };
+    if (input.registration && 'failed' in input.registration) {
+        plan.part = part([`Run \`haystack init\` again: registering the app for capture failed (${input.registration.failed}).`]);
+        return plan;
+    }
+    const script = loadCaptureScript();
+    if (!script) {
+        plan.part = { status: 'unsupported', why: 'This version of the Haystack CLI carries no capture script yet; update @haystackeditor/cli and run `haystack init` again.' };
+        return plan;
+    }
+    await loadBabel();
+    const ctx = appContext(gitRoot, appDir);
+    const existing = readCaptureConfig(appDir);
+    const detected = await detectApp(ctx);
+    const registered = input.registration && 'registered' in input.registration ? input.registration.registered : null;
+    const recorded = existing !== null && existing.app === ctx.appPath
+        ? { key: existing.key, repositoryId: existing.repositoryId, applicationId: existing.applicationId } : null;
+    plan.registration = { repository, app: ctx.appPath, origins, recorded };
+    // The key the files carry: the one registration just issued, else the recorded one (prepareCapture checks it against
+    // Haystack's before anything is written, and plans again when it differs), else a placeholder until registration.
+    const identity = registered ?? recorded;
+    if (!registered && (existing === null || existing.origins.join(',') !== origins.join(','))) {
+        plan.notices.push(registrationNotice(plan.registration));
+    }
+    const consentFile = consentScript(consent);
+    const tags = {
+        script: { file: script.file, integrity: script.integrity },
+        key: identity?.key ?? KEY_PLACEHOLDER,
+        consentNotRequired: consent.kind === 'not-required',
+        consent: consentFile ? { file: consentFile.file, integrity: consentFile.integrity } : null,
+    };
+    const steps = [];
+    const planner = new Planner(gitRoot);
+    const config = {
+        version: CAPTURE_VERSION,
+        key: tags.key,
+        repositoryId: identity?.repositoryId ?? ID_PLACEHOLDER,
+        applicationId: identity?.applicationId ?? ID_PLACEHOLDER,
+        adapter: detected && !detected.blocked ? detected.id : null,
+        app: ctx.appPath,
+        origins,
+        consent: formatConsentChoice(consent),
+        // Where a build step publishes (none for a static site, whose manifest init writes and the repository keeps).
+        publish: detected && !detected.blocked && !isManual(detected.publishDirs) && !isManual(detected.build)
+            && detected.build.kind !== 'init-writes' ? detected.publishDirs : [],
+    };
+    planner.text(captureConfigFile(appDir), formatCaptureConfig(config), 'The app\'s capture identity, committed for the build step that publishes its route manifest (CAPTURE-V1 rule 8b).');
+    const consentStep = manualConsentStep(consent);
+    const automatic = detected !== null && detected.blocked === null;
+    const staticDir = automatic && typeof detected.staticDir === 'string' ? detected.staticDir : null;
+    // The files: in the app's static output, or staged for a manual step.
+    const filesDir = staticDir === null ? join(appDir, STAGED_DIR) : join(appDir, staticDir, STATIC_SUBDIR);
+    const wanted = new Set([script.file, ...(consentFile ? [consentFile.file] : [])]);
+    planner.summarized(join(filesDir, script.file), script.text, 'The capture script, self-hosted so Haystack\'s availability never affects how the app loads (CAPTURE-V1 rules 1, 13).', `(the capture script, ${script.bytes} bytes, ${script.integrity})`);
+    if (consentFile) {
+        planner.text(join(filesDir, consentFile.file), consentFile.bytes.toString('utf8'), `Applies the visitor's consent as ${consentLabel(consent)} records it to the tag (rule 2).`);
+    }
+    for (const old of scriptFiles(filesDir))
+        if (!wanted.has(old))
+            planner.remove(join(filesDir, old), 'An earlier capture file, replaced by this version.');
+    const servedAt = `/${STATIC_SUBDIR}/`;
+    if (!automatic) {
+        const why = detected ? `${detected.label}: ${detected.blocked}` : `init recognized no framework at ${ctx.appPath}`;
+        const looked = detected?.looked.join(', ') ?? `${ctx.appPath} (package.json, Gemfile, manage.py, index.html)`;
+        steps.push(`Add the tag yourself (${why}; init looked at ${looked}):\n${(detected ? detected.manualTag(tags) : ['Inside <head> of every page:', ...htmlTags(tags)]).map(line => `  ${line}`).join('\n')}`);
+        steps.push(`Serve the files init staged in ${posix.join(ctx.appPath, STAGED_DIR)}/ at ${servedAt} on each production origin.`);
+        steps.push(`Publish the route manifest (CAPTURE-V1 rule 4) after each build at /${MANIFEST_DIR}/current.json and`
+            + ` /${MANIFEST_DIR}/<release>.json on each production origin; until one is published the tag records nothing.`
+            + ` \`${manifestCommand(ctx.appPath)}\` generates it only for an app init sets up itself: once the reason above is gone,`
+            + ' run `haystack init` again and it adds that build step.');
+        steps.push(`If production sends a Content-Security-Policy, it needs: ${cspFallbackLine()}.`);
+        if (consentStep)
+            steps.push(consentStep);
+        plan.changes = planner.changes;
+        plan.part = part(steps);
+        return plan;
+    }
+    // The tag.
+    const integration = await detected.integrate(tags);
+    if (isManual(integration)) {
+        steps.push(`Add the tag yourself (${integration.manual}; init looked at ${detected.looked.join(', ')}):\n${detected.manualTag(tags).map(line => `  ${line}`).join('\n')}`);
+    }
+    else {
+        for (const edit of integration)
+            planner.text(edit.target, edit.content, 'Adds the capture tag, async and pinned by its integrity, exactly once (CAPTURE-V1 rules 1, 13a).');
+    }
+    if (typeof detected.staticDir !== 'string')
+        steps.push(`Serve ${posix.join(ctx.appPath, STAGED_DIR)} at ${servedAt}: ${detected.staticDir.manual}.`);
+    // The build step and where the manifest goes.
+    await planBuild(plan, planner, steps, ctx, detected, config);
+    // Routes now, so a router init cannot read is a step today rather than a manifest missing after the next build.
+    if (!isManual(detected.build) && detected.build.kind !== 'rake') {
+        const routes = exactRoutes(await detected.routes({}));
+        if (isManual(routes))
+            steps.push(`Make the routes readable or publish the manifest yourself: ${routes.manual}.`);
+        else {
+            const count = new Set(routes.map(route => route.match)).size;
+            plan.notices.push(`Capture: ${count} ${count === 1 ? 'route' : 'routes'} for the route manifest (${detected.label}).`);
+        }
+    }
+    // CSP: report, never edit.
+    const csp = findCsp(gitRoot, appDir, detected.id === 'next', origins);
+    if (csp.findings.length === 0) {
+        plan.notices.push(`Capture: no Content-Security-Policy found in ${csp.looked}. If production sets one elsewhere (a CDN or proxy), it needs: ${cspFallbackLine()}.`);
+    }
+    for (const finding of csp.findings) {
+        for (const note of finding.notes)
+            plan.notices.push(`Capture: ${finding.where}: ${note}`);
+        if (finding.additions.length > 0) {
+            steps.push(`Update the Content-Security-Policy at ${finding.where} (init never edits a policy):\n${finding.additions.map(line => `  ${line}`).join('\n')}`);
+        }
+    }
+    if (consentStep)
+        steps.push(consentStep);
+    plan.changes = planner.changes;
+    plan.part = part(steps);
+    return plan;
+}
+async function planBuild(plan, planner, steps, ctx, detected, config) {
+    const build = detected.build;
+    const publishDirs = detected.publishDirs;
+    if (isManual(publishDirs)) {
+        steps.push(publishDirs.manual);
+        return;
+    }
+    if (isManual(build)) {
+        steps.push(`Run \`${manifestCommand(ctx.appPath)}\` after each build (${build.manual}).`);
+        return;
+    }
+    const manifestFile = join(ctx.appDir, 'package.json');
+    if (build.kind === 'package-script') {
+        const text = readFileSync(manifestFile, 'utf8');
+        const edited = withBuildStep(text, manifestFile, build.script, build.phase);
+        if (typeof edited !== 'string') {
+            steps.push(`Publish the route manifest with each build: ${edited.manual}, with @haystackeditor/cli as an exact-version production dependency.`);
+            return;
+        }
+        plan.packageText = edited === text ? null : edited;
+        plan.needsCli = true;
+    }
+    else if (build.kind === 'rake') {
+        if (!existsSync(manifestFile)) {
+            steps.push('Add @haystackeditor/cli to the app as an exact-version dependency (the app has no package.json, so its build has no pinned'
+                + ' CLI to publish the route manifest with), then run `haystack init` again.');
+            return;
+        }
+        plan.needsCli = true;
+        planner.text(join(ctx.appDir, RAKE_FILE), RAKE_FILE_CONTENT, 'Publishes the route manifest after assets:precompile from the routes Rails lists, with the CLI the app pins, never failing the build (rules 4, 13).');
+    }
+    else {
+        // A static site: init writes the manifest itself (no build).
+        plan.notices.push('Capture: this site has no build, so init writes its route manifest, and `haystack init` writes it again after its pages change.');
+        await planStaticManifest(planner, ctx, detected, config, publishDirs);
+        return;
+    }
+    // A manifest written into a source directory is a build product: keep it out of commits (once: init's own line).
+    if (typeof detected.staticDir === 'string' && publishDirs.includes(detected.staticDir)) {
+        const line = `/${posix.join(detected.staticDir === '.' ? '' : detected.staticDir, MANIFEST_DIR)}/`;
+        const file = join(ctx.appDir, '.gitignore');
+        const before = readText(file) ?? '';
+        if (!before.split('\n').includes(line)) {
+            planner.text(file, `${before}${before === '' || before.endsWith('\n') ? '' : '\n'}# Haystack's route manifest, written by each build\n${line}\n`, 'Keeps the route manifest each build writes out of commits.');
+        }
+    }
+}
+async function planStaticManifest(planner, ctx, detected, config, publishDirs) {
+    const routes = exactRoutes(await detected.routes({}));
+    if (isManual(routes))
+        return;
+    const manifest = buildManifest(config, detected.manifestAdapter, routes, detected.chunks());
+    const body = manifestDocument(manifest);
+    for (const dir of publishDirs) {
+        for (const name of [`${manifest.release}.json`, 'current.json']) {
+            planner.text(join(ctx.appDir, dir, MANIFEST_DIR, name), body, 'The route manifest (CAPTURE-V1 rule 4), published with the site.');
+        }
+    }
+}
+function appName(app) {
+    return app === '.' ? 'the app at the repository root' : app;
+}
+/** The preview's line for the registration init checks with --yes. */
+function registrationNotice(request) {
+    return `Capture: with --yes, init checks the registration of ${appName(request.app)} of ${request.repository} with Haystack (production`
+        + ` origins ${request.origins.join(', ')}): ${request.recorded ? 'the origins are updated after every change below is written'
+            : 'the app is registered before anything is written, and the key it issues replaces the placeholder in the diffs'}.`;
+}
+const errorText = (error) => (error instanceof Error ? error.message : String(error));
+const nothing = async () => null;
+/**
+ * The registration side of capture with --yes (rule 8b), so Haystack never accepts origins the checkout does not carry:
+ *   - the recorded key is Haystack's (stable for the app): nothing is sent before the writes; the origins are updated
+ *     after every write succeeded (a failed update is a step, and the next run compares with Haystack again);
+ *   - no recorded key, or another one: the app is registered before anything is written (the files need its key), and a
+ *     write failing afterwards puts the previous origins back (a new app's registration stays: it accepts nothing the
+ *     checkout has not deployed).
+ * `approved`: the user said yes to changes; without it nothing is sent, and a needed update is a step.
+ */
+export async function prepareCapture(request, token, say, approved) {
+    const app = appName(request.app);
+    let current;
+    try {
+        current = await readCaptureRegistration(token, request.repository, request.app);
+    }
+    catch (error) {
+        // Unreachable (no answer, a server error, a rate limit): the recorded key still names the app, so the local updates go
+        // ahead with it and only the origins check waits. Refused (any other answer), or no key to go on: capture waits.
+        const unreachable = !(error instanceof HaystackApiError) || error.status >= 500 || error.status === 429;
+        if (unreachable && request.recorded) {
+            say(chalk.yellow(`Haystack could not be reached to check ${app}'s capture registration (${errorText(error)}); the recorded key stays.`));
+            return { replan: null, rollback: async () => { }, finish: async () => `Run \`haystack init --yes\` again: Haystack could not be reached to check ${app}'s production origins for capture (${errorText(error)}).` };
+        }
+        say(chalk.yellow(`Browser capture is not set up this time: reading ${app}'s registration failed (${errorText(error)}).`));
+        return { replan: { failed: `reading the app's registration failed (${errorText(error)})` }, finish: nothing, rollback: async () => { } };
+    }
+    const sameOrigins = current !== null && current.origins.join(',') === request.origins.join(',');
+    const register = async (origins) => registerCaptureApp(token, request.repository, request.app, origins);
+    if (request.recorded && current && current.key === request.recorded.key) {
+        return { replan: null, rollback: async () => { }, finish: async (finished) => {
+                if (sameOrigins)
+                    return null;
+                if (!approved)
+                    return `Run \`haystack init --yes\`: Haystack accepts capture from ${current.origins.join(', ') || 'no origin'}, not ${request.origins.join(', ')}.`;
+                try {
+                    await register(request.origins);
+                    finished(chalk.green(`✓ Updated ${app}'s production origins for browser capture`));
+                    return null;
+                }
+                catch (error) {
+                    return `Run \`haystack init --yes\` again: updating ${app}'s production origins for capture failed (${errorText(error)}).`;
+                }
+            } };
+    }
+    if (!approved) {
+        return { replan: { failed: 'registering the app needs your yes (`haystack init --yes`)' }, finish: nothing, rollback: async () => { } };
+    }
+    let registered;
+    try {
+        registered = await register(request.origins);
+    }
+    catch (error) {
+        say(chalk.yellow(`Browser capture is not set up this time: registering ${app} failed (${errorText(error)}).`));
+        return { replan: { failed: errorText(error) }, finish: nothing, rollback: async () => { } };
+    }
+    say(chalk.green(`✓ Registered ${app} for browser capture`));
+    const previous = current !== null && !sameOrigins ? current.origins : null;
+    return { replan: { registered }, finish: nothing, rollback: async (undo) => {
+            if (previous === null)
+                return;
+            try {
+                await register(previous);
+                undo(chalk.yellow(`Restored ${app}'s previous production origins for browser capture.`));
+            }
+            catch (error) {
+                undo(chalk.red(`Could not restore ${app}'s previous production origins for browser capture (${errorText(error)}); run \`haystack init --yes\` again.`));
+            }
+        } };
+}
+/** The capture part with one more step (an update that could not be made after the writes). */
+export function withCaptureStep(part, step) {
+    if (part.status === 'unsupported')
+        return part;
+    return { status: 'manual', steps: [...(part.status === 'manual' ? part.steps : []), step], received: part.received };
+}