@codyswann/lisa 4.70.4 → 4.71.1

package/all/copy-overwrite/scripts/lib/process-tree-runner.mjs

@@ -3,7 +3,7 @@
 // Do not edit directly — durable changes belong upstream in Lisa.
 
 /**
- * Run one shell command in a process boundary and reap every descendant.
+ * Run one shell command or explicit literal argv in a process boundary and reap every descendant.
  *
  * Node's synchronous timeout signals only the direct child. A gate command is
  * a tree (shell, package manager, test workers), so killing only the shell
@@ -33,15 +33,87 @@
  */
 
 import { spawn, spawnSync } from "node:child_process";
+import { writeSync } from "node:fs";
 
 import { invokedAsScript } from "./invoked-as-script.mjs";
 import { startWindowsProcessJob } from "./windows-process-job.mjs";
 
+/** Diagnostic memory stays bounded independently of the complete live stream. */
+const CAPTURE_TAIL_BYTES = 512 * 1024;
 const KILL_GRACE_MS = 750;
 const REAP_POLL_MS = 25;
 const WINDOWS_TIMEOUT_EXIT_CODE = 255;
 const TERMINATING_SIGNALS = ["SIGINT", "SIGTERM", "SIGHUP"];
 
+/** Same literal-dispatch envelope as the launcher; control overhead is not free. */
+function snapshotArguments(argv) {
+  if (!Array.isArray(argv)) throw new Error("direct argv requires an array");
+  const copied = [...argv];
+  if (
+    copied.length > 512 ||
+    copied.some(
+      value =>
+        typeof value !== "string" ||
+        value.includes("\0") ||
+        Buffer.byteLength(value) > 65_536
+    ) ||
+    Buffer.byteLength(JSON.stringify(copied)) > 262_144
+  )
+    throw new Error("invalid or unbounded direct argument vector");
+  return copied;
+}
+
+/** The direct API fails before any handlers or child backend acquire ownership. */
+function directInvocation(command, args, timeoutMs) {
+  if (!["linux", "darwin"].includes(process.platform))
+    throw new Error("direct argv requires Linux or macOS");
+  if (!Number.isInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 1_800_000)
+    throw new Error("invalid direct command deadline");
+  if (
+    typeof command !== "string" ||
+    command.length === 0 ||
+    !Array.isArray(args)
+  )
+    throw new Error(
+      "direct argv requires a nonempty executable and argument array"
+    );
+  return snapshotArguments([command, ...args]).slice(1);
+}
+
+/** Only the explicit new-mode prefix is closed; legacy parsing remains tolerant. */
+function parseDirectArguments(argv, separator) {
+  snapshotArguments([...process.execArgv, process.argv[1], ...argv]);
+  const vector = snapshotArguments(argv);
+  if (separator < 0) throw new Error("direct argv requires a separator");
+  const flags = vector.slice(0, separator);
+  const timeouts = flags.filter(value => /^--timeout-ms=\d+$/.test(value));
+  const capture = flags.filter(value => value === "--capture-fd=3");
+  const watches = flags.filter(value => /^--watch-pid=\d+$/.test(value));
+  if (
+    flags.filter(value => value === "--direct-argv").length !== 1 ||
+    timeouts.length !== 1 ||
+    capture.length > 1 ||
+    watches.some(
+      value => !isWatchablePid(Number(value.slice("--watch-pid=".length)))
+    ) ||
+    flags.length !== 1 + timeouts.length + capture.length + watches.length
+  )
+    throw new Error("invalid direct supervisor control prefix");
+  const command = vector[separator + 1];
+  const timeoutMs = Number(timeouts[0].slice("--timeout-ms=".length));
+  return {
+    command,
+    timeoutMs,
+    directArgs: directInvocation(
+      command,
+      vector.slice(separator + 2),
+      timeoutMs
+    ),
+    watchPids: parseWatchPids(watches),
+    captureFd: capture.length === 1 ? 3 : undefined,
+  };
+}
+
 /**
  * How often the supervisor asks whether the run it serves is still there.
  *
@@ -103,6 +175,8 @@ function parseArguments(argv) {
   const separator = argv.indexOf("--");
   const timeoutMs = Number(timeoutArg?.slice("--timeout-ms=".length));
   const flags = separator >= 0 ? argv.slice(0, separator) : argv;
+  if (flags.includes("--direct-argv"))
+    return parseDirectArguments(argv, separator);
   const command = separator >= 0 ? argv.slice(separator + 1).join(" ") : "";
   if (!Number.isFinite(timeoutMs) || timeoutMs <= 0 || !command) {
     throw new Error(
@@ -110,7 +184,20 @@ function parseArguments(argv) {
         "[--watch-pid=<pid>]... -- <command>"
     );
   }
-  return { command, timeoutMs, watchPids: parseWatchPids(flags) };
+  const captureFlags = flags.filter(value => value.startsWith("--capture-fd="));
+  if (
+    captureFlags.length > 1 ||
+    captureFlags.some(value => value !== "--capture-fd=3")
+  ) {
+    throw new Error("capture requires the fixed caller descriptor 3");
+  }
+  return {
+    command,
+    timeoutMs,
+    watchPids: parseWatchPids(flags),
+    captureFd: captureFlags.length === 1 ? 3 : undefined,
+    directArgs: undefined,
+  };
 }
 
 /**
@@ -403,6 +490,79 @@ export async function reapTree(pid, controls = DEFAULT_REAP_CONTROLS) {
   throw new Error(`gate process tree ${pid} survived SIGKILL`);
 }
 
+/**
+ * Bound pipe drainage after tree reap by the existing two-phase cleanup budget.
+ * A missing close notification cannot postpone a verdict indefinitely; it is
+ * cleanup failure, and retained pipes are detached before the failure returns.
+ * @param {import("node:child_process").ChildProcess} child Owned command shell.
+ * @param {Promise<unknown>} closed Already-armed output completion observation.
+ * @returns {Promise<void>} Confirmed drainage, or a terminal cleanup failure.
+ */
+async function waitForOutputClose(child, closed) {
+  let timer;
+  try {
+    await Promise.race([
+      closed,
+      new Promise((_, reject) => {
+        timer = setTimeout(() => {
+          child.stdout?.destroy();
+          child.stderr?.destroy();
+          reject(
+            new Error("gate diagnostic output did not close after tree reap")
+          );
+        }, KILL_GRACE_MS * 2);
+      }),
+    ]);
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+/**
+ * Forward command output immediately, retaining only a bounded diagnostic tail.
+ * The fixed caller descriptor is deliberately absent from the child's stdio.
+ * Waiting for close drains the pipes AFTER exit has already started tree reap.
+ * A broken diagnostic sink cannot replace the actual process verdict.
+ * @param {import("node:child_process").ChildProcess} child Owned command shell.
+ * @param {number|undefined} descriptor Optional caller-only descriptor.
+ * @returns {() => Promise<void>} Drain and publish diagnostics once.
+ */
+function captureChildOutput(child, descriptor) {
+  if (descriptor === undefined) return async () => {};
+  let tail = Buffer.alloc(0);
+  let publication;
+  const closed = new Promise(resolve => child.once("close", resolve));
+  for (const stream of [child.stdout, child.stderr]) {
+    stream.on("data", chunk => {
+      tail =
+        chunk.length >= CAPTURE_TAIL_BYTES
+          ? Buffer.from(chunk.subarray(-CAPTURE_TAIL_BYTES))
+          : Buffer.concat([
+              tail.subarray(-Math.max(0, CAPTURE_TAIL_BYTES - chunk.length)),
+              chunk,
+            ]);
+    });
+    stream.pipe(process.stdout, { end: false });
+  }
+  const publish = async () => {
+    await waitForOutputClose(child, closed);
+    try {
+      let offset = 0;
+      while (offset < tail.length) {
+        offset += writeSync(descriptor, tail, offset, tail.length - offset);
+      }
+    } catch {
+      process.stderr.write(
+        "gate diagnostic capture unavailable; OS verdict retained\n"
+      );
+    }
+  };
+  return () => {
+    publication ??= publish();
+    return publication;
+  };
+}
+
 /**
  * Run one command as a supervised process tree.
  *
@@ -410,7 +570,7 @@ export async function reapTree(pid, controls = DEFAULT_REAP_CONTROLS) {
  * terminating signal, and now the run it serves going away — and all three
  * share one reap, because a tree reaped twice is a tree whose second kill
  * lands on a recycled pid.
- * @param {string} command Shell source to supervise.
+ * @param {string} command Shell source, or literal executable when directArgs is present.
  * @param {number} timeoutMs Deadline for the whole tree.
  * @param {typeof reapTree} [reap] Injectable POSIX group reaper. Windows retains its native job owner.
  * @param {object} [options] Interrupt-watch seams, injectable for tests.
@@ -419,16 +579,32 @@ export async function reapTree(pid, controls = DEFAULT_REAP_CONTROLS) {
  * @param {number} [options.pollMs] How often to ask.
  * @param {(line: string) => void} [options.report] Where the report goes.
  * @param {number} [options.launchParentPid] Parent pid captured at start.
+ * @param {3} [options.captureFd] Fixed caller-only diagnostic descriptor.
+ * @param {readonly string[]} [options.directArgs] Literal arguments; even [] selects POSIX direct spawning.
  * @returns {Promise<{code: number|null, signal: string|null}>} The verdict.
  */
 export function supervise(command, timeoutMs, reap = reapTree, options = {}) {
+  // Snapshot before reading other option getters or arming handlers. The caller
+  // cannot change the selected argv while lifetime ownership is being set up.
+  const suppliedArgs = options.directArgs;
+  const directArgs =
+    suppliedArgs === undefined
+      ? undefined
+      : directInvocation(command, suppliedArgs, timeoutMs);
   const {
     watchPids = [],
     detect = interruptionReason,
     pollMs = ORPHAN_POLL_MS,
     report = line => process.stderr.write(`${line}\n`),
     launchParentPid = process.ppid,
+    captureFd,
   } = options;
+  if (
+    captureFd !== undefined &&
+    (captureFd !== 3 || process.platform === "win32")
+  ) {
+    throw new Error("diagnostic capture requires POSIX caller descriptor 3");
+  }
   return new Promise((resolve, reject) => {
     let windowsJob;
     let pendingSignal;
@@ -457,11 +633,16 @@ export function supervise(command, timeoutMs, reap = reapTree, options = {}) {
           windowsJob = startWindowsProcessJob(command);
           return windowsJob.child;
         }
-        return spawn("/bin/sh", ["-c", command], {
-          detached: true,
-          env: process.env,
-          stdio: "inherit",
-        });
+        return spawn(
+          directArgs === undefined ? "/bin/sh" : command,
+          directArgs === undefined ? ["-c", command] : directArgs,
+          {
+            detached: true,
+            env: process.env,
+            stdio:
+              captureFd === undefined ? "inherit" : ["inherit", "pipe", "pipe"],
+          }
+        );
       } catch (error) {
         clearSignalHandlers();
         reject(error);
@@ -486,6 +667,7 @@ export function supervise(command, timeoutMs, reap = reapTree, options = {}) {
       return;
     }
 
+    const publishCapture = captureChildOutput(child, captureFd);
     let timedOut = false;
     let settled = false;
     let settling = false;
@@ -500,7 +682,7 @@ export function supervise(command, timeoutMs, reap = reapTree, options = {}) {
       clearSignalHandlers();
     };
     // Every exit path shares one reap. A signal that arrives while the direct
-    // shell's close handler is reaping must join that operation instead of
+    // shell's exit handler is reaping must join that operation instead of
     // starting a second kill sequence or taking the default signal action.
     const reapOnce = () => {
       reapPromise ??= windowsJob ? windowsJob.reap() : reap(pid);
@@ -514,6 +696,7 @@ export function supervise(command, timeoutMs, reap = reapTree, options = {}) {
       // exits. Reap that residue fully before returning a verdict too.
       try {
         await reapOnce();
+        await publishCapture();
         settled = true;
         cleanup();
         resolve({ code, signal });
@@ -530,6 +713,8 @@ export function supervise(command, timeoutMs, reap = reapTree, options = {}) {
       // the handle and rejecting makes the supervisor terminal instead of
       // leaving its promise pending behind a live event-loop reference.
       child.unref();
+      child.stdout?.destroy();
+      child.stderr?.destroy();
       reject(
         new Error(
           `gate process-tree supervisor failed: ${error instanceof Error ? error.message : String(error)}`
@@ -547,12 +732,15 @@ export function supervise(command, timeoutMs, reap = reapTree, options = {}) {
       settling = true;
       clearDeadline();
       clearInterval(interruptWatch);
-      reapOnce().then(() => {
-        cleanup();
-        report(interruptionReport(reason, pid));
-        // Restore the signal-shaped exit after the detached tree is gone.
-        process.kill(process.pid, signal);
-      }, failReap);
+      reapOnce()
+        .then(async () => {
+          await publishCapture();
+          cleanup();
+          report(interruptionReport(reason, pid));
+          // Restore the signal-shaped exit after the detached tree is gone.
+          process.kill(process.pid, signal);
+        })
+        .catch(failReap);
     };
     const relaySignal = signal => {
       stopWith(`it received ${signal}`, signal);
@@ -579,21 +767,24 @@ export function supervise(command, timeoutMs, reap = reapTree, options = {}) {
       terminating = true;
       settling = true;
       clearDeadline();
-      reapOnce().then(() => {
-        cleanup();
-        const verdict = timeoutVerdictForPlatform(process.platform);
-        // A signal-shaped result is the existing gate runner vocabulary for
-        // "no verdict". It also prevents an ordinary exit code such as 124
-        // from being confused with a user command that returned that code.
-        if (verdict.signal === null) {
-          // Windows has no signal-shaped process result. 255 is a dedicated
-          // supervisor timeout code with a documented (but unavoidable)
-          // collision risk, well outside ordinary command exit conventions.
-          process.exit(verdict.code);
-        } else {
-          process.kill(process.pid, verdict.signal);
-        }
-      }, failReap);
+      reapOnce()
+        .then(async () => {
+          await publishCapture();
+          cleanup();
+          const verdict = timeoutVerdictForPlatform(process.platform);
+          // A signal-shaped result is the existing gate runner vocabulary for
+          // "no verdict". It also prevents an ordinary exit code such as 124
+          // from being confused with a user command that returned that code.
+          if (verdict.signal === null) {
+            // Windows has no signal-shaped process result. 255 is a dedicated
+            // supervisor timeout code with a documented (but unavoidable)
+            // collision risk, well outside ordinary command exit conventions.
+            process.exit(verdict.code);
+          } else {
+            process.kill(process.pid, verdict.signal);
+          }
+        })
+        .catch(failReap);
     }, timeoutMs);
 
     dispatchSignal = relaySignal;
@@ -607,9 +798,9 @@ export function supervise(command, timeoutMs, reap = reapTree, options = {}) {
         reject(error);
       }, failReap);
     });
-    child.once("close", (code, signal) => {
+    child.once("exit", (code, signal) => {
       // On timeout, wait for the forced group reap above before this supervisor
-      // ends. Exiting on the direct shell's close is the race that used to leave
+      // ends. Exiting before the complete group reap is the race that used to leave
       // its descendants alive.
       if (!timedOut) void finish(code, signal);
     });
@@ -619,10 +810,13 @@ export function supervise(command, timeoutMs, reap = reapTree, options = {}) {
 }
 
 async function main() {
-  const { command, timeoutMs, watchPids } = parseArguments(
-    process.argv.slice(2)
-  );
-  const result = await supervise(command, timeoutMs, reapTree, { watchPids });
+  const { command, timeoutMs, watchPids, captureFd, directArgs } =
+    parseArguments(process.argv.slice(2));
+  const result = await supervise(command, timeoutMs, reapTree, {
+    watchPids,
+    captureFd,
+    directArgs,
+  });
   if (result.signal) {
     process.kill(process.pid, result.signal);
   } else {

package/all/copy-overwrite/scripts/lisa-automation-provenance.mjs

@@ -0,0 +1,161 @@
+// This file is managed by Lisa and IS replaced on each `lisa` run.
+// Do not edit directly — durable changes belong upstream in Lisa.
+
+/** Authenticated non-AI attribution supplements ordinary tracking and lint gates. */
+import {
+  predictedCommit,
+  verifyRecoveryOrigin,
+} from "./lib/github-attestation-recovery.mjs";
+import { invokedAsScript } from "./lib/invoked-as-script.mjs";
+import {
+  assertPinnedVerifier,
+  requireProof,
+  sha256,
+  verifyCurrentProvider,
+  verifyDescriptorAttestation,
+} from "./lib/github-attestation-verifier.mjs";
+import {
+  canonicalJson,
+  trustedConfiguration,
+  npmProposal,
+} from "./lib/automation-provenance-contract.mjs";
+import {
+  git,
+  privateBytes,
+  localDescriptor,
+  proofSnapshot,
+  canonicalContext,
+} from "./lib/automation-provenance-local.mjs";
+export { canonicalJson } from "./lib/automation-provenance-contract.mjs";
+
+/** A present but malformed reference is a failure, even with ordinary AI trailers. */
+export function automationReference(message) {
+  const lines = message
+    .split(/\r?\n/)
+    .filter(line => /^\s*Automation-Provenance\s*:/i.test(line));
+  if (lines.length === 0) return undefined;
+  requireProof(lines.length === 1, "ambiguous automation reference");
+  const match =
+    /^Automation-Provenance: actions\/([1-9]\d*)\/attempts\/([1-9]\d*)$/.exec(
+      lines[0]
+    );
+  requireProof(Boolean(match), "malformed automation reference");
+  return { runId: match[1], runAttempt: match[2] };
+}
+
+/** Fresh recovery is a fixed separate role; original proof never acquires current authority. */
+function recoveryPermission(
+  policy,
+  descriptor,
+  context,
+  config,
+  bytes,
+  paths,
+  recoveryBytes,
+  recoveryBundle,
+  messageBytes
+) {
+  const recovery = JSON.parse(recoveryBytes.toString("utf8"));
+  requireProof(
+    recoveryBytes.toString("utf8") === `${canonicalJson(recovery)}\n`,
+    "noncanonical/duplicate recovery fields"
+  );
+  const npmPolicySha256 = sha256(canonicalJson(config.npmUpdater));
+  verifyRecoveryOrigin(policy, descriptor, recovery, sha256(bytes), paths, {
+    npmPolicySha256,
+    proposalKey: sha256(
+      canonicalJson({
+        repository: policy.repository,
+        target: "main",
+        ecosystem: "npm",
+        directory: ".",
+        parent: descriptor.parent,
+        policySha256: npmPolicySha256,
+        updates: descriptor.updates,
+      })
+    ),
+    leafBodySha256: sha256(context.issue.body),
+    commit: predictedCommit(descriptor, messageBytes),
+    maintainer: config.npmUpdater.maintainer,
+    recoverySha256: sha256(recoveryBytes),
+  });
+  requireProof(
+    privateBytes(paths.recovery, 65_536).equals(recoveryBytes) &&
+      privateBytes(paths.recoveryBundle, 1_048_576).equals(recoveryBundle),
+    "recovery proof changed during verification"
+  );
+}
+
+/** Verify fixed private proof against the final message, tree, binding and provider. */
+export function verifyAutomationProvenance(messageFile) {
+  const messageBytes = privateBytes(messageFile, 65_536, false);
+  const message = messageBytes.toString("utf8");
+  const reference = automationReference(message);
+  if (!reference) return false;
+  const { config, policy } = trustedConfiguration(git);
+  const { paths, optional, bytes, bundleBytes, descriptor } = proofSnapshot();
+  localDescriptor(descriptor, messageBytes, reference, policy);
+  npmProposal(descriptor, git);
+  requireProof(
+    descriptor.proposalKey ===
+      sha256(
+        canonicalJson({
+          repository: policy.repository,
+          parent: descriptor.parent,
+          updates: descriptor.updates,
+        })
+      ),
+    "deterministic proposal key differs"
+  );
+  assertPinnedVerifier(policy);
+  const context = canonicalContext(message, config, policy, descriptor);
+  const recoveryBytes = optional[0]
+    ? privateBytes(paths.recovery, 65_536)
+    : null;
+  const recoveryBundle = optional[0]
+    ? privateBytes(paths.recoveryBundle, 1_048_576)
+    : null;
+  if (recoveryBytes) {
+    recoveryPermission(
+      policy,
+      descriptor,
+      context,
+      config,
+      bytes,
+      paths,
+      recoveryBytes,
+      recoveryBundle,
+      messageBytes
+    );
+  } else {
+    verifyDescriptorAttestation(policy, descriptor, paths, sha256(bytes));
+    verifyCurrentProvider(policy, descriptor);
+  }
+  requireProof(
+    privateBytes(paths.bundle, 1_048_576).equals(bundleBytes),
+    "origin bundle changed during verification"
+  );
+  requireProof(
+    privateBytes(paths.descriptor, 65_536).equals(bytes),
+    "descriptor changed during verification"
+  );
+  localDescriptor(
+    descriptor,
+    privateBytes(messageFile, 65_536, false),
+    reference,
+    policy
+  );
+  return true;
+}
+
+if (invokedAsScript(import.meta.url)) {
+  try {
+    const verified = verifyAutomationProvenance(process.argv[2]);
+    process.exitCode = verified ? 0 : 10;
+  } catch {
+    console.error(
+      "Invalid automation provenance: required proof, policy or provider evidence failed"
+    );
+    process.exitCode = 1;
+  }
+}

package/all/copy-overwrite/scripts/lisa-hooks/block-no-verify.sh

@@ -152,6 +152,7 @@ fi
 
 if ! BLOCK_NO_VERIFY_COMMAND="$command_str" python3 - <<'PY'
 import os
+import io
 import re
 import shlex
 import sys
@@ -1284,6 +1285,57 @@ def git_skips_verification(text, depth=0):
     return False
 
 
+def git_config_reads_key(tokens, index):
+    """Whether this key is a read operand of this exact Git config invocation.
+
+    Keep the command boundary: a query cannot authorize the next command or
+    turn a later setting into a read. Unknown options remain conservative.
+    """
+    start = index
+    while start > 0 and tokens[start - 1] not in COMMAND_SEPARATORS:
+        start -= 1
+    end = index + 1
+    while end < len(tokens) and tokens[end] not in COMMAND_SEPARATORS:
+        end += 1
+    program, args, opaque = command_word(tokens[start:end])
+    if opaque or program != "git":
+        return False
+    found = subcommand_after_git(args, 0)
+    if not found or found[0] != "config":
+        return False
+    argv = found[1]
+    key_offset = index - (end - len(argv))
+    read = bool(argv and argv[0] == "get")
+    cursor = 1 if read else 0
+    operands = []
+    read_actions = {"--get", "--get-all", "--get-regexp", "--get-urlmatch"}
+    modifiers = {
+        "--local", "--global", "--system", "--worktree", "--includes",
+        "--no-includes", "--show-origin", "--show-scope", "--null", "-z",
+        "--name-only", "--all", "--regexp",
+    }
+    value_options = {"--file", "-f", "--blob", "--type", "--default", "--value"}
+    while cursor < len(argv):
+        token = argv[cursor]
+        if token in read_actions:
+            if read:
+                return False
+            read = True
+        elif token in value_options:
+            if cursor + 1 >= len(argv):
+                return False
+            cursor += 1
+        elif token in modifiers:
+            pass
+        elif token.startswith("-"):
+            if not any(token.startswith(option + "=") for option in value_options):
+                return False
+        else:
+            operands.append(cursor)
+        cursor += 1
+    return bool(operands and operands[0] == key_offset and (read or len(operands) == 1))
+
+
 def token_bypass(tokens):
     """Whether any token disables hooks by environment or by git config.
 
@@ -1295,7 +1347,7 @@ def token_bypass(tokens):
     option, and an option belongs to an argv.
 
     Args:
-        tokens: Punctuation-stripped tokens of one command or script.
+        tokens: Operator-aware tokens of one command or script.
 
     Returns:
         True when a token disables verification.
@@ -1321,7 +1373,9 @@ def token_bypass(tokens):
         if lowered.startswith("core.hookspath="):
             if not is_permitted_hooks_path(token.split("=", 1)[1]):
                 return True
-        if lowered == "core.hookspath" and i + 1 < len(tokens):
+        if (lowered == "core.hookspath" and i + 1 < len(tokens)
+                and tokens[i + 1] not in COMMAND_SEPARATORS
+                and not git_config_reads_key(tokens, i)):
             if not is_permitted_hooks_path(tokens[i + 1]):
                 return True
         # `git --config-env=core.hooksPath=SOMEVAR` sets the same config, reading
@@ -1388,8 +1442,22 @@ def token_bypass(tokens):
     return False
 
 
+class OperatorStream(io.StringIO):
+    """Remember punctuation read while shlex is inside a quote or escape."""
+
+    lexer = None
+    literal_operator = False
+
+    def read(self, size=-1):
+        value = super().read(size)
+        if (self.lexer is not None and self.lexer.state in {"'", '"', "\\"}
+                and any(char in "();|&<>" for char in value)):
+            self.literal_operator = True
+        return value
+
+
 def flat_tokens(text):
-    """Punctuation-stripped tokens, or None when the text does not lex.
+    """Operator-aware tokens without printed data, or None on a lex failure.
 
     Args:
         text: A command line or a script's contents.
@@ -1398,9 +1466,25 @@ def flat_tokens(text):
         The token list, or None.
     """
     try:
-        tokens = shell_tokens(text)
+        stream = OperatorStream(line_boundaries_as_separators(text))
+        lexer = shlex.shlex(stream, posix=True, punctuation_chars=True)
+        lexer.whitespace_split = True
+        lexer.commenters = ""
+        stream.lexer = lexer
+        tokens = []
+        while True:
+            stream.literal_operator = False
+            token = lexer.get_token()
+            if token is None:
+                break
+            # shlex otherwise makes quoted ";" indistinguishable from a real
+            # command boundary. Keep literal destinations as data, so a setter
+            # cannot masquerade as a one-operand configuration read.
+            if token in COMMAND_SEPARATORS and stream.literal_operator:
+                token = "\x00" + token
+            tokens.append(token)
         return [
-            token.strip("();|&")
+            token
             for index, token in enumerate(tokens)
             if not printed_argument(tokens, index)
         ]