devharness 0.9.15 → 0.9.17

package/build/tools/replay-tools.js

@@ -3,6 +3,8 @@
  */
 import { z } from 'zod';
 import { selectSuiteFiles, sequenceFolders } from '../helpers/sequence-tree.js';
+import { promises as fs } from 'fs';
+import { join } from 'path';
 import { createTool } from '../validation-helpers.js';
 import { createSuccessResponse, createErrorResponse } from '../messages.js';
 import { showReplayOverlay } from '../interaction-recorder.js';
@@ -117,6 +119,10 @@ import { readHistoryLines, getHistoryFilePath } from '../debug-logger.js';
 import { startRecording, eventsToCommands, generateCondensedTimeline, isCommentEvent, } from '../interaction-recorder.js';
 import { addIssue, initializeTracker, saveIssueSequence, } from '../issue-tracker.js';
 import { configManager } from '../config.js';
+import { hasTemplateToken } from './interpolation.js';
+import { parseEnvFile } from '../helpers/env-file.js';
+import { getProjectDir } from '../helpers/paths.js';
+import { isAbsolute } from 'path';
 // =============================================================================
 // Schema Definition
 // =============================================================================
@@ -133,7 +139,8 @@ const replaySchema = z.object({
     description: z.string().optional(),
     expectedOutcome: z.string().optional(),
     startUrl: z.string().optional().describe('create: sequence start URL. run: replace the stored startUrl for this run only (e.g. a freshly minted link)'),
-    baseUrl: z.string().optional().describe('run: retarget the sequence at another deployment — every absolute URL (startUrl + command params) keeps its path/query but takes this origin. Not preserved across a mid-run pause/step resume'),
+    envFile: z.string().optional().describe("run/runAll: a KEY=value file supplying the {{env:NAME}} tokens this run resolves. A relative path resolves against the project directory (the one holding .devharness); absolute is used as-is. Its values WIN over the server's own environment, and the server's environment is never modified - two runs may name different files. A missing file, or a line that is neither blank, a # comment, nor NAME=value, fails before anything runs. Keeps a credential out of the sequence file and out of this call, and changing it needs no client restart"),
+    baseUrl: z.string().optional().describe('run/runAll: retarget at another deployment — every absolute URL (startUrl, command params, a declared connection\u2019s launch url) keeps its path/query but takes this origin, in the sequence itself and in every sequence it reaches through a conditional or forEach. On runAll it applies to every sequence in the suite. Not preserved across a mid-run pause/step resume'),
     indices: z.array(z.number()).optional().describe('Command indices'),
     lines: z.array(z.number()).optional().describe('Log line numbers'),
     sequenceId: z.string().optional(),
@@ -155,7 +162,7 @@ const replaySchema = z.object({
     requiredSockets: z.array(z.string()).optional().describe("declare: URL substrings of the WebSockets this sequence's assertions ride on, e.g. ['/api/sync/socket']. Match the app's own path, not the origin, so it survives baseUrl. Replaces the whole list; [] clears it"),
     connections: z.record(z.string()).optional().describe("run: rebind a multi-connection sequence's recorded references onto this session - { \"<recorded reference>\": \"<reference here>\" }. Only needed when steps carry their own connectionReason (replay({action:'get', outputFormat:'commands'}) shows which)"),
     record: z.boolean().optional(),
-    variables: z.record(z.string()).optional(),
+    variables: z.record(z.string()).optional().describe("run/runAll: replace the text of recorded input-type steps. Keys are BUILT from the selector - var_<0-based step index>_<selector, non-alphanumerics replaced by _> - so '#password' at step 3 is 'var_3__password', two underscores; read them off `get` or off the prompt a run returns rather than composing them. A key naming no typed-text step is rejected (runAll checks against the whole suite's union). Reaches sequences a conditional or forEach nests into. For a credential prefer {{env:NAME}} in the step itself, which keeps the value out of the sequence file and out of this call; an explicit value here still wins over the environment"),
     stepTimeout: z.number().optional().describe('Per-step ms (default 30000). A step exceeding min(stepTimeout, remaining totalTimeout) fails the run at that step. wait steps are exempt (own timeoutMs) but still capped by totalTimeout'),
     totalTimeout: z.number().optional().describe('Total ms'),
     startFrom: z.number().optional().describe('Start step (1-indexed)'),
@@ -184,7 +191,7 @@ const replaySchema = z.object({
     strict: z.enum(['errors', 'warnings']).optional().describe("run/runAll: fail the run when it PRODUCES console output - 'errors' fails on new console errors, 'warnings' also fails on new warnings. Counted per connection and diffed against the start of the run, so pre-existing noise is not blamed on this sequence. A sequence can be functionally correct and still be logging; strict is how you separate those questions"),
     folder: z.string().optional().describe("runAll: sequences subfolder to run, relative to the sequences dir (e.g. 'spine'). Omit to run every sequence outside folders whose name starts with '_'. The whole tree is always LOADED first so name references (a conditional's then, a forEach's do) resolve wherever the helper lives"),
     continueOnFailure: z.boolean().optional().describe('runAll: keep going after a sequence fails and report every result (default true). false stops at the first failure'),
-    killChromeOnFinish: z.boolean().optional().describe("run: after finishing (skipped on pause/abort), kill the browsers this run owns - its own connection plus any a launchChrome step actually created. A step that reached an already-bound reference only borrowed that browser and it is left running, so an instance you launched yourself survives. Also skipped for any browser whose port another live connection shares (a launchChrome step usually opens a tab in the same instance), and the run reports which connection kept it alive."),
+    killChromeOnFinish: z.boolean().optional().describe("run/runAll: after finishing (skipped on pause/abort), kill the browsers this run owns - its own connection plus any a launchChrome step actually created. A step that reached an already-bound reference only borrowed that browser and it is left running, so an instance you launched yourself survives. Also skipped for any browser whose port another live connection shares (a launchChrome step usually opens a tab in the same instance), and the run reports which connection kept it alive. On runAll only the LAST sequence carries it, so a preamble's browser survives between sequences and a suite that stops early leaves the browsers up."),
 }).strict();
 async function handleHistory(args, recorder) {
     const limit = args.limit || 50;
@@ -765,6 +772,92 @@ function collectNestedRebindableReferences(commands, recorder, depth = 0, seen =
     }
     return { references, complete };
 }
+/**
+ * Read the run's `envFile` into the values {{env:NAME}} resolves against.
+ *
+ * Loaded here, before any side effects, so a missing file or a malformed line
+ * fails the run as a parameter error rather than as a step failure halfway
+ * through a flow that has already logged in. The values stay on the run's
+ * context: writing them into process.env would leak one run's credentials into
+ * every concurrent background run, which name their own files.
+ */
+async function loadRunEnv(envFile) {
+    const path = isAbsolute(envFile) ? envFile : join(getProjectDir(), envFile);
+    let text;
+    try {
+        text = await fs.readFile(path, 'utf-8');
+    }
+    catch (err) {
+        return { error: `Could not read envFile "${envFile}" (resolved to ${path}): ${err?.code === 'ENOENT' ? 'no such file' : err?.message || String(err)}` };
+    }
+    const { values, problems } = parseEnvFile(text);
+    if (problems.length > 0) {
+        const shown = problems.slice(0, 3).map(p => `line ${p.line}: ${p.text}`).join('; ');
+        return { error: `envFile "${envFile}" (resolved to ${path}) has ${problems.length} line(s) that are neither blank, a # comment, nor NAME=value with a name matching [A-Za-z_][A-Za-z0-9_]* - ${shown}${problems.length > 3 ? ', ...' : ''}. A skipped line reads as a set variable, so the run stops here.` };
+    }
+    return { values };
+}
+/**
+ * Every `variables` key a run could substitute on: this sequence's typed-text
+ * steps plus those of every sequence it reaches through a `conditional`'s
+ * `then` or a `forEach`'s `do`.
+ *
+ * Resolution is memory-only and best-effort, the same rule
+ * collectNestedRebindableReferences follows: a helper that lives on disk and
+ * has not been loaded is not loaded here (that would register it as a side
+ * effect of validation), so `complete: false` says "this list may be short"
+ * and the caller must not call a missing key a typo. `runAll` loads the whole
+ * tree before it runs anything, so a suite run always gets a complete list.
+ */
+function collectVariableKeys(commands, recorder, depth = 0, seen = new Set()) {
+    const keys = new Set(Object.keys(extractTextVariables(commands)));
+    // Must track the executor's own cap: with a raised maxConditionalDepth, keys
+    // at runtime-reachable depths would be omitted while `complete` still claimed
+    // the list was exhaustive, and a valid key would be rejected as a typo.
+    if (depth >= configManager.getReplayConfig().maxConditionalDepth) {
+        return { keys, complete: false };
+    }
+    let complete = true;
+    for (const cmd of commands) {
+        const named = cmd.tool === 'conditional' ? cmd.params?.then
+            : cmd.tool === 'forEach' ? cmd.params?.do
+                : undefined;
+        if (typeof named !== 'string' || !named || seen.has(named))
+            continue;
+        seen.add(named);
+        const nested = recorder.listSequences().find(sq => sq.name === named);
+        if (!nested) {
+            complete = false;
+            continue;
+        }
+        const deeper = collectVariableKeys(nested.commands, recorder, depth + 1, seen);
+        for (const key of deeper.keys)
+            keys.add(key);
+        complete = complete && deeper.complete;
+    }
+    return { keys, complete };
+}
+/**
+ * Keys the caller supplied that name no typed-text step anywhere the run can
+ * reach. Empty when the key list could not be resolved in full, because a key
+ * valid for an unloaded helper is not a typo.
+ *
+ * A `variables` key is matched exactly and nothing else looks at it, so an
+ * unmatched key used to be dropped in silence: the step ran on its RECORDED
+ * text while the call read as an override. For a recorded credential that
+ * means the old password reaching the live app with the run reporting success.
+ * Same rule the `connections` rebinding already applies to a reference that
+ * names no recorded step.
+ */
+function unmatchedVariableKeys(supplied, commands, recorder) {
+    const { keys, complete } = collectVariableKeys(commands, recorder);
+    if (!complete)
+        return { unmatched: [], known: [...keys].sort() };
+    return {
+        unmatched: Object.keys(supplied).filter(k => !keys.has(k)),
+        known: [...keys].sort(),
+    };
+}
 /**
  * Run every sequence in a folder, in filename order, and report one line each.
  *
@@ -838,9 +931,44 @@ async function handleRunAll(args, recorder, executeToolCall, getPageForConnectio
                 (folders.length ? `Available folders: ${folders.join(', ')}.` : 'No subfolders exist yet — sequences are all at the top level.')
         });
     }
+    // One `variables` map covers the whole suite, so a key is a typo only when it
+    // matches NO member. Checked once, before anything runs: a supplied key that
+    // lands nowhere is dropped in silence by the executor and every sequence runs
+    // on its recorded text, which for a credential means the recorded one reaching
+    // the live app with the suite reporting green. The whole tree is in memory by
+    // now, so the union is exhaustive.
+    if (args.variables && Object.keys(args.variables).length > 0) {
+        const suiteKeys = new Set();
+        for (const entry of selected) {
+            const seq = recorder.listSequences().find(sq => sq.name === entry.name);
+            if (!seq)
+                continue;
+            for (const key of collectVariableKeys(seq.commands, recorder).keys)
+                suiteKeys.add(key);
+        }
+        const unmatched = Object.keys(args.variables).filter(k => !suiteKeys.has(k));
+        if (unmatched.length > 0) {
+            const known = [...suiteKeys].sort();
+            return createErrorResponse('INVALID_PARAMETER', {
+                parameter: 'variables',
+                value: unmatched.join(', '),
+                message: `${unmatched.length > 1 ? 'Those keys name' : `"${unmatched[0]}" names`} no typed-text step in any sequence this run selects. ` +
+                    `The key is BUILT from the selector - var_<0-based step index>_<selector, non-alphanumerics replaced by _> - so "#password" at step 3 is "var_3__password", with two underscores. ` +
+                    (known.length
+                        ? `Substitutable across this suite: ${known.join(', ')}.`
+                        : `No sequence here has typed text to substitute.`),
+            });
+        }
+    }
     const keepGoing = args.continueOnFailure !== false;
     const results = [];
-    for (const entry of selected) {
+    for (const [index, entry] of selected.entries()) {
+        // killChromeOnFinish means the SUITE's finish here, not each sequence's: a
+        // teardown between sequences destroys the state a _helpers preamble just
+        // established. Only the last sequence carries it, so a suite that stops
+        // early (continueOnFailure: false, a cancel) leaves the browsers up for
+        // the failure to be read in.
+        const isLast = index === selected.length - 1;
         if (abortSignal?.aborted) {
             results.push({ filename: entry.filename, name: entry.name, ok: false, detail: 'cancelled before it ran' });
             continue;
@@ -863,16 +991,16 @@ async function handleRunAll(args, recorder, executeToolCall, getPageForConnectio
                 // no-op that a caller then has to notice.
                 variables: args.variables ?? {},
                 // Per-run args that are actively wrong when fanned across a suite:
-                // killChromeOnFinish would tear down the browser between sequences and
-                // destroy the state a _helpers preamble just established, and
                 // startFrom/stepTo/stepCount/startUrl mean something only for one
-                // specific sequence.
-                killChromeOnFinish: undefined,
+                // specific sequence. baseUrl does carry - it retargets every
+                // sequence at the same deployment, which is what a suite run of a
+                // recorded set against another environment needs.
+                killChromeOnFinish: isLast ? args.killChromeOnFinish : undefined,
                 startFrom: undefined,
                 stepTo: undefined,
                 stepCount: undefined,
                 startUrl: undefined,
-            }, recorder, executeToolCall, getPageForConnection, abortSignal, getConnectionPort);
+            }, recorder, executeToolCall, getPageForConnection, abortSignal, getConnectionPort, { validateVariableKeys: false });
             const text = (res?.content || []).map((c) => c?.text || '').join('\n');
             // performRun stamps _meta.replay on every terminal response, so trust that
             // over the prose. Regexing for a line starting with "Error:" both misses
@@ -1275,7 +1403,13 @@ function socketFailures(before, after, required, requiredOn) {
     }
     return { settled: out, absent };
 }
-async function handleRun(args, recorder, executeToolCall, getPageForConnection, abortSignal, getConnectionPort) {
+async function handleRun(args, recorder, executeToolCall, getPageForConnection, abortSignal, getConnectionPort, 
+/**
+ * `validateVariableKeys: false` for a sequence running as part of a suite:
+ * `runAll` validates the one map it holds against the whole suite's keys and
+ * a per-sequence check would reject a key meant for a different member.
+ */
+opts) {
     // Load sequence
     const loadResult = await loadSequence({ name: args.name, sequenceId: args.sequenceId }, recorder);
     if (!loadResult.success) {
@@ -1297,9 +1431,15 @@ async function handleRun(args, recorder, executeToolCall, getPageForConnection,
     if (!connectionReason && !analysis.hasLaunchBeforeConnection && needsConnection) {
         connectionReason = deriveConnectionReference(sequence.name);
     }
-    // Handle variable extraction and prompting
+    // Handle variable extraction and prompting. A step whose recorded text
+    // carries an interpolation token ({{env:NAME}}, {{var:...}}) is already
+    // parameterised - its value arrives at run time by definition - so it does
+    // not hold the run open for an answer. It stays in the extracted list, so a
+    // caller can still override it and the key validation still accepts it.
     const extractedVariables = extractTextVariables(commands);
-    if (Object.keys(extractedVariables).length > 0 && args.variables === undefined) {
+    const needsAnswer = Object.values(extractedVariables)
+        .some(v => !hasTemplateToken(v.value));
+    if (needsAnswer && args.variables === undefined) {
         const idParam = args.sequenceId || args.name;
         // Tag it: this response is a PROMPT, not a run. runAll has to be able to
         // tell "asked you a question" from "executed and passed", or a suite goes
@@ -1309,6 +1449,43 @@ async function handleRun(args, recorder, executeToolCall, getPageForConnection,
             _meta: { tool: 'replay', action: 'run', timestamp: Date.now(), replay: { success: false, prompted: true } }
         };
     }
+    // The envFile is read before any side effects: a missing file or a malformed
+    // line is a parameter error, not a step failure halfway through a flow that
+    // has already logged in.
+    let runEnv;
+    if (args.envFile) {
+        const loaded = await loadRunEnv(args.envFile);
+        if ('error' in loaded) {
+            return createErrorResponse('INVALID_PARAMETER', {
+                parameter: 'envFile',
+                value: args.envFile,
+                message: loaded.error,
+            });
+        }
+        runEnv = loaded.values;
+    }
+    // Same rule for the typed-text substitutions, before any side effects. A key
+    // matching no step is dropped in silence by the executor and the step runs on
+    // its RECORDED text, so a mistyped key reads as an override while the recorded
+    // value - a password among them - reaches the live app.
+    //
+    // Skipped for a suite member: `runAll` hands ONE map to every sequence and
+    // checks it against the union of the whole suite, so a key valid for another
+    // sequence must not fail this one.
+    if (args.variables && opts?.validateVariableKeys !== false) {
+        const { unmatched, known } = unmatchedVariableKeys(args.variables, commands, recorder);
+        if (unmatched.length > 0) {
+            return createErrorResponse('INVALID_PARAMETER', {
+                parameter: 'variables',
+                value: unmatched.join(', '),
+                message: `${unmatched.length > 1 ? 'Those keys name' : `"${unmatched[0]}" names`} no typed-text step in "${sequence.name}" or any sequence it reaches. ` +
+                    `The key is BUILT from the selector - var_<0-based step index>_<selector, non-alphanumerics replaced by _> - so "#password" at step 3 is "var_3__password", with two underscores. ` +
+                    (known.length
+                        ? `Substitutable here: ${known.join(', ')}.`
+                        : `This sequence has no typed text to substitute.`),
+            });
+        }
+    }
     // Validate the connection rebinding before any side effects. A key that names
     // no recorded reference is a typo the user needs to hear about now: silently
     // ignoring it would leave the step on its recorded reference and, in the worst
@@ -1398,6 +1575,7 @@ async function handleRun(args, recorder, executeToolCall, getPageForConnection,
         sequence, analysis, connectionReason, needsConnection,
         launchedConnections: new Set(),
         ...(connectionMap && { connectionMap }),
+        ...(runEnv && { runEnv }),
     };
     // wait: true - pre-0.7 blocking behaviour, driven by the MCP request signal.
     // Also what nested `replay run` STEPS use (the executor injects wait: true),
@@ -1609,7 +1787,7 @@ async function handleRun(args, recorder, executeToolCall, getPageForConnection,
  * record's terminal status - never derived by parsing the response.
  */
 async function performRun(deps, abortSignal, runId, onProgress) {
-    const { args, recorder, executeToolCall, getPageForConnection, getConnectionPort, sequence, analysis, connectionReason, needsConnection, connectionMap, launchedConnections } = deps;
+    const { args, recorder, executeToolCall, getPageForConnection, getConnectionPort, sequence, analysis, connectionReason, needsConnection, connectionMap, launchedConnections, runEnv } = deps;
     // Build execution context
     const ctx = {
         executeToolCall,
@@ -1618,7 +1796,17 @@ async function performRun(deps, abortSignal, runId, onProgress) {
         logPrefix: 'run',
         variableStore: {},
         launchedConnections,
-        ...(connectionMap && { connectionMap })
+        ...(connectionMap && { connectionMap }),
+        // Carried on the context so nested sequences inherit the retarget; the
+        // top-level sequence was already rebased in handleRun.
+        ...(args.baseUrl && { rebaseOrigin: args.baseUrl }),
+        // Same reason: a shared login helper reached by a conditional is exactly
+        // where a supplied credential has to land.
+        ...(args.variables && { variables: args.variables }),
+        // Held on the context rather than written into process.env: concurrent
+        // background runs may name different files, and a global write would let
+        // one run's credentials resolve inside the other.
+        ...(runEnv && { runEnv })
     };
     // Ensure connection is ready
     let didAutoLaunch = false;