@sema-agent/core 7.27.0 → 7.28.0

package/dist/shell/lex.js

@@ -0,0 +1,1253 @@
+export const SHELL_LEX_DIALECT = "bash";
+export const QUOTE_KINDS = ["'", '"', "$'", '$"', "\\"];
+export const EXPANSION_KINDS = ["parameter", "command", "process", "arithmetic", "tilde", "brace", "glob"];
+export const UNRESOLVED_REASONS = ["ansi_c_escape", "locale_string", "parameter_text", "subscript_text", "compound_array_value"];
+export const UNRESOLVED_VIEW = {
+    ansi_c_escape: "refuse",
+    locale_string: "fold",
+    parameter_text: "refuse",
+    subscript_text: "refuse",
+    compound_array_value: "refuse",
+};
+export const UNRESOLVED_SENTENCE = {
+    ansi_c_escape: "an ANSI-C quoted word carries an escape this lexer does not fold",
+    locale_string: "a locale-translated string carries text this lexer does not read",
+    parameter_text: "a parameter expansion carries text this lexer does not read",
+    subscript_text: "a subscript carries text this lexer does not read",
+    compound_array_value: "a compound assignment value is a list this lexer does not read",
+};
+export const PARSE_FAILURE_REASONS = [
+    "unterminated_single_quote",
+    "unterminated_double_quote",
+    "unterminated_ansi_c",
+    "unterminated_subscript",
+    "unterminated_compound_value",
+    "unterminated_command_substitution",
+    "unterminated_process_substitution",
+    "unterminated_parameter_expansion",
+    "unterminated_heredoc",
+    "undelimited_substitution",
+    "redirect_without_target",
+    "heredoc_without_delimiter",
+    "heredoc_delimiter_expands",
+    "background_operator",
+];
+export const PARSE_FAILURE_SENTENCE = {
+    unterminated_single_quote: "a single-quoted word never closes",
+    unterminated_double_quote: "a double-quoted word never closes",
+    unterminated_ansi_c: "an ANSI-C quoted word never closes",
+    unterminated_subscript: "a subscript never closes",
+    unterminated_compound_value: "a compound assignment value never closes",
+    unterminated_command_substitution: "a command substitution never terminates",
+    unterminated_process_substitution: "a process substitution never terminates",
+    unterminated_parameter_expansion: "a parameter expansion never terminates",
+    unterminated_heredoc: "a here-document body never terminates",
+    undelimited_substitution: "a command substitution carries a construct this lexer cannot delimit",
+    redirect_without_target: "a redirection names no target",
+    heredoc_without_delimiter: "a here-document operator names no delimiter",
+    heredoc_delimiter_expands: "a here-document delimiter is not a plain word",
+    background_operator: "`&` backgrounding is not allowed — a backgrounded process outlives the command",
+};
+export const BASH_ONLY_CONSTRUCTS = [
+    "ansi_c_quoting",
+    "locale_string",
+    "append_assignment",
+    "subscript_assignment",
+    "compound_assignment",
+    "here_string",
+    "process_substitution",
+    "redirect_both",
+    "brace_expansion",
+];
+export const ANSI_C_ESCAPES = {
+    a: "\x07",
+    b: "\b",
+    e: "\x1b",
+    E: "\x1b",
+    f: "\f",
+    n: "\n",
+    r: "\r",
+    t: "\t",
+    v: "\v",
+    "\\": "\\",
+    "'": "'",
+    '"': '"',
+    "?": "?",
+};
+export const SUBSTITUTION_PLACEHOLDER = " subst";
+export const SHELL_CONTROL_PREFIXES = new Set(["!", "if", "then", "elif", "else", "while", "until", "do", "time"]);
+const COMPOUND_OPERAND_COMMANDS = new Set(["alias", "declare", "eval", "export", "let", "local", "readonly", "typeset"]);
+const GROUP_WORDS = new Set(["{", "}"]);
+function quoteStep(open, ch) {
+    if (open === undefined)
+        return ch === '"' || ch === "'" ? ch : undefined;
+    return open === ch ? undefined : open;
+}
+export function dollarOpensQuote(text, at) {
+    const next = text[at + 1];
+    if (next !== "'" && next !== '"')
+        return false;
+    let run = 0;
+    for (let k = at - 1; k >= 0 && text[k] === "\\"; k--)
+        run += 1;
+    return run % 2 === 0;
+}
+export function ansiCQuoteClose(source, open) {
+    for (let i = open + 1; i < source.length; i++) {
+        if (source[i] === "\\") {
+            i += 1;
+            continue;
+        }
+        if (source[i] === "'")
+            return i;
+    }
+    return undefined;
+}
+export function quoteMask(s) {
+    const openedBy = new Array(s.length).fill(undefined);
+    const quoted = new Array(s.length).fill(false);
+    let open;
+    for (let i = 0; i < s.length; i++) {
+        if (open === undefined && s[i] === "$" && s[i + 1] === "'") {
+            const close = ansiCQuoteClose(s, i + 1);
+            for (let k = i; k <= (close ?? s.length - 1); k++) {
+                openedBy[k] = "'";
+                quoted[k] = true;
+            }
+            if (close === undefined) {
+                open = "'";
+                break;
+            }
+            i = close;
+            continue;
+        }
+        const before = open;
+        open = quoteStep(open, s[i]);
+        const holds = before ?? open;
+        openedBy[i] = holds;
+        quoted[i] = holds !== undefined;
+    }
+    return { openedBy, quoted, balanced: open === undefined };
+}
+export function literalTildeSpelling(word) {
+    return word.startsWith("~") ? `./${word}` : word;
+}
+function decodeAnsiC(body) {
+    let text = "";
+    let unresolved = false;
+    for (let i = 0; i < body.length; i++) {
+        const c = body[i];
+        if (c !== "\\" || i + 1 >= body.length) {
+            text += c;
+            continue;
+        }
+        const next = body[i + 1];
+        const folded = ANSI_C_ESCAPES[next];
+        if (folded === undefined) {
+            unresolved = true;
+            text += c + next;
+        }
+        else
+            text += folded;
+        i++;
+    }
+    return { text, unresolved };
+}
+export function wordsOf(segment) {
+    const folded = [];
+    const raw = [];
+    for (const t of segment.tokens) {
+        if (t.kind !== "word")
+            continue;
+        folded.push(t.folded);
+        raw.push(t.raw);
+    }
+    return { folded, raw };
+}
+function past(text, j) {
+    let k = j;
+    while (text[k] === "\\" && text[k + 1] === "\n")
+        k += 2;
+    return k;
+}
+function parenBodyEnd(text, open) {
+    let depth = 0;
+    let quoted;
+    let wordStart = true;
+    let word = "";
+    for (let j = open; j < text.length; j++) {
+        const c = text[j];
+        if (quoted !== undefined) {
+            if (c === "\\" && quoted === '"')
+                j++;
+            else if (c === quoted)
+                quoted = undefined;
+            wordStart = false;
+            continue;
+        }
+        if (c === "\\") {
+            if (text[j + 1] === "\n") {
+                j++;
+                continue;
+            }
+            j++;
+            wordStart = false;
+            word = "";
+            continue;
+        }
+        if (c === "#" && wordStart) {
+            const nl = text.indexOf("\n", j);
+            if (nl < 0)
+                return undefined;
+            j = nl;
+            wordStart = true;
+            word = "";
+            continue;
+        }
+        if (c === "'" || c === '"') {
+            quoted = c;
+            wordStart = false;
+            word = "";
+            continue;
+        }
+        if (c === "$" && text[past(text, j + 1)] === "{") {
+            const open2 = past(text, j + 1);
+            const close = braceExpansionEnd(text, open2);
+            if (close === undefined)
+                return undefined;
+            if (UNREADABLE_EXPANSION_TEXT.test(text.slice(open2 + 1, close - 1)))
+                return "unreadable";
+            j = close - 1;
+            wordStart = false;
+            word = "";
+            continue;
+        }
+        if (c === "<" && text[past(text, j + 1)] === "<" && text[past(text, past(text, j + 1) + 1)] !== "<")
+            return "unreadable";
+        if (c === "(")
+            depth++;
+        else if (c === ")") {
+            depth--;
+            if (depth === 0)
+                return j + 1;
+        }
+        const boundary = /[\s;|&()]/.test(c);
+        if (boundary && word === "case")
+            return "unreadable";
+        word = boundary ? "" : word + c;
+        wordStart = boundary;
+    }
+    return undefined;
+}
+function braceExpansionEnd(text, open) {
+    let quoted;
+    for (let j = open + 1; j < text.length; j++) {
+        const c = text[j];
+        if (quoted !== undefined) {
+            if (c === "\\" && quoted === '"')
+                j++;
+            else if (c === quoted)
+                quoted = undefined;
+            continue;
+        }
+        if (c === "\\") {
+            j++;
+            continue;
+        }
+        if (c === "'" || c === '"') {
+            quoted = c;
+            continue;
+        }
+        if (c === "}")
+            return j + 1;
+    }
+    return undefined;
+}
+function matchedParen(text, open) {
+    let depth = 0;
+    let quoted;
+    for (let j = open; j < text.length; j++) {
+        const c = text[j];
+        if (quoted !== undefined) {
+            if (c === "\\" && quoted === '"')
+                j++;
+            else if (c === quoted)
+                quoted = undefined;
+            continue;
+        }
+        if (c === "\\") {
+            j++;
+            continue;
+        }
+        if (c === "'" || c === '"') {
+            quoted = c;
+            continue;
+        }
+        if (c === "$" && text[j + 1] === "'") {
+            const close = ansiCQuoteClose(text, j + 1);
+            if (close === undefined)
+                return undefined;
+            j = close;
+            continue;
+        }
+        if (c === "$" && text[j + 1] === "{") {
+            const close = braceExpansionEnd(text, j + 1);
+            if (close === undefined)
+                return undefined;
+            j = close - 1;
+            continue;
+        }
+        if (c === "(")
+            depth++;
+        else if (c === ")" && --depth === 0)
+            return j + 1;
+    }
+    return undefined;
+}
+const UNREADABLE_EXPANSION_TEXT = /[$`()<>{\\]/;
+const EXPANSION_NAME = /^[#!]?([A-Za-z_][A-Za-z0-9_]*|\d+|[@*#?$!0-])/;
+const BARE_PARAMETER = /^(?:[A-Za-z_][A-Za-z0-9_]*|\d|[@*#?$!0-])/;
+const IDENTIFIER = /^[A-Za-z_][A-Za-z0-9_]*$/;
+const NAME_HEAD = /^[A-Za-z_][A-Za-z0-9_]*/;
+function backtickBody(source, open) {
+    let body = "";
+    for (let j = open + 1; j < source.length; j++) {
+        const c = source[j];
+        if (c === "\\" && j + 1 < source.length) {
+            const next = source[j + 1];
+            if (next === "\n") {
+                j++;
+                continue;
+            }
+            body += next === "$" || next === "`" || next === "\\" ? next : c + next;
+            j++;
+            continue;
+        }
+        if (c === "`")
+            return { body, close: j };
+        body += c;
+    }
+    return undefined;
+}
+export function substitutionsInHeredocBody(body) {
+    const out = [];
+    for (let i = 0; i < body.length; i++) {
+        const ch = body[i];
+        if (ch === "\\") {
+            i++;
+            continue;
+        }
+        if (ch === "`") {
+            const tick = backtickBody(body, i);
+            if (tick === undefined)
+                return undefined;
+            out.push(tick.body);
+            i = tick.close;
+            continue;
+        }
+        if (ch === "$" && body[i + 1] === "(") {
+            const end = parenBodyEnd(body, i + 1);
+            if (typeof end !== "number")
+                return undefined;
+            out.push(body.slice(i + 2, end - 1));
+            i = end - 1;
+        }
+    }
+    return out;
+}
+const WHITESPACE = /\s/;
+const BLANK = /[ \t]/;
+const fail = (reason, at) => ({ parseFailure: { reason, at } });
+const publicPiece = (p) => ({
+    span: [p.start, p.end],
+    quotedBy: p.quotedBy,
+    ...(p.literal !== undefined ? { literal: p.literal } : {}),
+    ...(p.expansion !== undefined ? { expansion: p.expansion } : {}),
+});
+class WordBuilder {
+    raw = "";
+    folded = "";
+    started = false;
+    at = 0;
+    pieces = [];
+    rawQuoted = [];
+    foldedAt = [];
+    srcOfRaw = [];
+    rawOfFolded = [];
+    marks = [];
+    subscript;
+    compound = false;
+    begin(src) { if (!this.started)
+        this.at = src; }
+    push(w, f, by, src) {
+        const rawStart = this.raw.length;
+        for (let k = 0; k < w.length; k++) {
+            this.rawQuoted.push(by);
+            this.foldedAt.push(this.folded.length);
+            this.srcOfRaw.push(src + k);
+        }
+        for (let k = 0; k < f.length; k++)
+            this.rawOfFolded.push(rawStart);
+        this.raw += w;
+        this.folded += f;
+        this.started = true;
+    }
+    emit(w, f, by, src, srcEnd) {
+        this.begin(src);
+        const last = this.pieces[this.pieces.length - 1];
+        if (last !== undefined && last.literal !== undefined && last.quotedBy === by && last.end === src && last.rawEnd === this.raw.length) {
+            last.literal += f;
+            last.end = srcEnd;
+            last.rawEnd += w.length;
+            last.foldedEnd += f.length;
+        }
+        else {
+            this.pieces.push({ start: src, end: srcEnd, rawStart: this.raw.length, rawEnd: this.raw.length + w.length, foldedStart: this.folded.length, foldedEnd: this.folded.length + f.length, quotedBy: by, literal: f, expansion: undefined });
+        }
+        this.push(w, f, by, src);
+    }
+    opaque(w, f, by, src, srcEnd, exp) {
+        this.begin(src);
+        this.pieces.push({ start: src, end: srcEnd, rawStart: this.raw.length, rawEnd: this.raw.length + w.length, foldedStart: this.folded.length, foldedEnd: this.folded.length + f.length, quotedBy: by, literal: undefined, expansion: exp });
+        this.push(w, f, by, src);
+    }
+    quoteChar(ch, by, src) {
+        this.begin(src);
+        this.rawQuoted.push(by);
+        this.foldedAt.push(this.folded.length);
+        this.srcOfRaw.push(src);
+        this.raw += ch;
+        this.started = true;
+    }
+    mark(reason, rawAt) { this.marks.push({ at: rawAt, reason }); }
+    inSubscript() { return this.subscript !== undefined && this.subscript.rawEnd === undefined; }
+    foldedOf(r) { return r >= this.foldedAt.length ? this.folded.length : this.foldedAt[r]; }
+    slice(from, to) {
+        const out = [];
+        for (const p of this.pieces) {
+            if (p.rawEnd <= from || p.rawStart >= to)
+                continue;
+            if (p.rawStart >= from && p.rawEnd <= to) {
+                out.push(p);
+                continue;
+            }
+            const rs = Math.max(p.rawStart, from);
+            const re = Math.min(p.rawEnd, to);
+            const fs = this.foldedOf(rs);
+            const fe = this.foldedOf(re);
+            out.push({
+                ...p,
+                start: this.srcOfRaw[rs],
+                end: this.srcOfRaw[re - 1] + 1,
+                rawStart: rs,
+                rawEnd: re,
+                foldedStart: fs,
+                foldedEnd: fe,
+                literal: p.literal === undefined ? undefined : p.literal.slice(fs - p.foldedStart, fe - p.foldedStart),
+            });
+        }
+        return out;
+    }
+    expansionPiece(from, to, exp) {
+        return { start: this.srcOfRaw[from], end: this.srcOfRaw[to - 1] + 1, rawStart: from, rawEnd: to, foldedStart: this.foldedOf(from), foldedEnd: this.foldedOf(to), quotedBy: undefined, literal: undefined, expansion: exp };
+    }
+    carve(from, to, exp, inExp) {
+        this.pieces = [...this.slice(0, from), this.expansionPiece(from, to, exp), ...this.slice(to, this.raw.length)];
+        if (inExp !== undefined)
+            for (let k = from; k < to; k++)
+                inExp[k] = true;
+    }
+    carveAll(ranges) {
+        if (ranges.length === 0)
+            return;
+        const cut = (p, rs, re) => {
+            if (rs === p.rawStart && re === p.rawEnd)
+                return p;
+            const fs = this.foldedOf(rs);
+            const fe = this.foldedOf(re);
+            return { ...p, start: this.srcOfRaw[rs], end: this.srcOfRaw[re - 1] + 1, rawStart: rs, rawEnd: re, foldedStart: fs, foldedEnd: fe, literal: p.literal === undefined ? undefined : p.literal.slice(fs - p.foldedStart, fe - p.foldedStart) };
+        };
+        const out = [];
+        let r = 0;
+        for (const p of this.pieces) {
+            let cur = p.rawStart;
+            while (r < ranges.length && ranges[r].from < p.rawEnd) {
+                const rg = ranges[r];
+                if (rg.from >= cur) {
+                    if (rg.from > cur)
+                        out.push(cut(p, cur, rg.from));
+                    out.push(this.expansionPiece(rg.from, rg.to, rg.exp));
+                }
+                if (rg.to >= p.rawEnd) {
+                    cur = p.rawEnd;
+                    break;
+                }
+                cur = rg.to;
+                r++;
+            }
+            if (cur < p.rawEnd)
+                out.push(cut(p, cur, p.rawEnd));
+            if (r < ranges.length && ranges[r].to === p.rawEnd)
+                r++;
+        }
+        this.pieces = out;
+    }
+    inExpansion() {
+        const out = new Array(this.raw.length).fill(false);
+        for (const p of this.pieces)
+            if (p.expansion !== undefined)
+                for (let k = p.rawStart; k < p.rawEnd; k++)
+                    out[k] = true;
+        return out;
+    }
+    tildePrefixEnd(at, inValue) {
+        if (this.raw[at] !== "~" || this.rawQuoted[at] !== undefined)
+            return undefined;
+        let k = at + 1;
+        for (; k < this.raw.length; k++) {
+            if (this.rawQuoted[k] !== undefined)
+                return undefined;
+            const c = this.raw[k];
+            if (c === "/" || (inValue && c === ":"))
+                break;
+        }
+        return k;
+    }
+    carveTilde(from, to) {
+        const covering = this.pieces.find((p) => p.literal !== undefined && p.rawStart <= from && p.rawEnd >= to);
+        if (covering === undefined)
+            return;
+        this.carve(from, to, { kind: "tilde", yields: "one", body: this.raw.slice(from, to) });
+    }
+    carveBraceAndGlob() {
+        const inExp = this.inExpansion();
+        const n = this.raw.length;
+        const readable = (j) => this.rawQuoted[j] === undefined && !inExp[j];
+        const braceClose = new Map();
+        const stack = [];
+        for (let j = 0; j < n; j++) {
+            if (!readable(j))
+                continue;
+            const d = this.raw[j];
+            if (d === "{")
+                stack.push({ open: j, listed: false });
+            else if (d === "}") {
+                const top = stack.pop();
+                if (top !== undefined)
+                    braceClose.set(top.open, { close: j, listed: top.listed });
+            }
+            else if (stack.length > 0 && (d === "," || (d === "." && this.raw[j + 1] === ".")))
+                stack[stack.length - 1].listed = true;
+        }
+        const nextBracketClose = new Array(n + 1).fill(-1);
+        for (let j = n - 1; j >= 0; j--)
+            nextBracketClose[j] = readable(j) && this.raw[j] === "]" ? j : nextBracketClose[j + 1];
+        const ranges = [];
+        for (let k = 0; k < n; k++) {
+            if (!readable(k))
+                continue;
+            const c = this.raw[k];
+            if (c === "{") {
+                const m = braceClose.get(k);
+                if (m === undefined || !m.listed)
+                    continue;
+                ranges.push({ from: k, to: m.close + 1, exp: { kind: "brace", yields: "many", body: this.raw.slice(k + 1, m.close) } });
+                k = m.close;
+                continue;
+            }
+            if (c === "*" || c === "?") {
+                ranges.push({ from: k, to: k + 1, exp: { kind: "glob", yields: "many" } });
+                continue;
+            }
+            if (c === "[") {
+                let j = k + 1;
+                if (this.raw[j] === "!" || this.raw[j] === "^")
+                    j++;
+                if (this.raw[j] === "]")
+                    j++;
+                const close = j < n ? nextBracketClose[j] : -1;
+                if (close < 0)
+                    continue;
+                ranges.push({ from: k, to: close + 1, exp: { kind: "glob", yields: "many", body: this.raw.slice(k + 1, close) } });
+                k = close;
+            }
+        }
+        this.carveAll(ranges);
+    }
+    readAssignment() {
+        const m = NAME_HEAD.exec(this.folded);
+        if (m === null)
+            return undefined;
+        const name = m[0];
+        const inExp = this.inExpansion();
+        const quotedF = (f) => { const r = this.rawOfFolded[f]; return r === undefined || this.rawQuoted[r] !== undefined; };
+        let at = name.length;
+        let sub;
+        if (this.folded[at] === "[") {
+            let depth = 0;
+            let close = -1;
+            for (let k = at; k < this.folded.length; k++) {
+                const r = this.rawOfFolded[k];
+                if (this.rawQuoted[r] !== undefined || inExp[r])
+                    continue;
+                if (this.folded[k] === "[")
+                    depth++;
+                else if (this.folded[k] === "]" && --depth === 0) {
+                    close = k;
+                    break;
+                }
+            }
+            if (close < 0)
+                return undefined;
+            sub = { from: at, to: close + 1 };
+            at = close + 1;
+        }
+        const appends = this.folded.startsWith("+=", at);
+        if (!appends && this.folded[at] !== "=")
+            return undefined;
+        const opLen = appends ? 2 : 1;
+        let bare = true;
+        for (let f = 0; f < name.length; f++)
+            if (quotedF(f))
+                bare = false;
+        if (sub !== undefined && (quotedF(sub.from) || quotedF(sub.to - 1)))
+            bare = false;
+        for (let f = at; f < at + opLen; f++)
+            if (quotedF(f))
+                bare = false;
+        const opRaw = this.rawOfFolded[at];
+        const valueRawFrom = opRaw + opLen;
+        if (!this.compound) {
+            for (let k = valueRawFrom; k < this.raw.length; k++) {
+                if (k !== valueRawFrom && !(this.raw[k - 1] === ":" && this.rawQuoted[k - 1] === undefined && !inExp[k - 1]))
+                    continue;
+                const end = this.tildePrefixEnd(k, true);
+                if (end !== undefined)
+                    this.carveTilde(k, end);
+            }
+        }
+        const valuePieces = this.slice(valueRawFrom, this.raw.length).map(publicPiece);
+        const value = {
+            raw: this.raw.slice(valueRawFrom),
+            folded: this.folded.slice(at + opLen),
+            pieces: valuePieces,
+            expansions: valuePieces.filter((p) => p.expansion !== undefined),
+            ...(this.compound ? { unresolved: "compound_array_value" } : {}),
+        };
+        let subscript;
+        if (sub !== undefined) {
+            const rawFrom = this.rawOfFolded[sub.from] + 1;
+            const rawTo = this.rawOfFolded[sub.to - 1];
+            const inner = this.slice(rawFrom, rawTo).map(publicPiece);
+            subscript = { raw: this.raw.slice(rawFrom, rawTo), folded: this.folded.slice(sub.from + 1, sub.to - 1), expansions: inner.filter((p) => p.expansion !== undefined) };
+        }
+        return {
+            name,
+            op: appends ? "+=" : "=",
+            ...(subscript !== undefined ? { subscript } : {}),
+            value,
+            bare,
+            form: this.compound ? "compound" : "scalar",
+            settlesWholeName: !appends && sub === undefined && !this.compound,
+        };
+    }
+    isAssignmentHead() {
+        if (this.compound || this.inSubscript())
+            return false;
+        if (!/^[A-Za-z_][A-Za-z0-9_]*(?:\[[\s\S]*\])?\+?=$/.test(this.raw))
+            return false;
+        const sub = this.subscript;
+        for (let k = 0; k < this.raw.length; k++) {
+            if (sub !== undefined && k > sub.rawStart && k < sub.rawEnd - 1)
+                continue;
+            if (this.rawQuoted[k] !== undefined)
+                return false;
+        }
+        return true;
+    }
+    finish(atCommandPosition, tildeSpelling = true) {
+        const rawText = this.raw;
+        let foldedText = this.folded;
+        let assignment;
+        const tildeEnd = this.tildePrefixEnd(0, false);
+        if (tildeEnd !== undefined)
+            this.carveTilde(0, tildeEnd);
+        else {
+            if (tildeSpelling)
+                foldedText = literalTildeSpelling(foldedText);
+            assignment = this.readAssignment();
+        }
+        if (!(assignment !== undefined && assignment.bare && atCommandPosition))
+            this.carveBraceAndGlob();
+        if (assignment !== undefined && this.subscript?.text)
+            this.mark("subscript_text", this.subscript.rawStart);
+        const marks = this.marks.filter((m) => m.reason !== "subscript_text" || assignment?.subscript !== undefined)
+            .sort((a, b) => (UNRESOLVED_VIEW[a.reason] === UNRESOLVED_VIEW[b.reason] ? a.at - b.at : UNRESOLVED_VIEW[a.reason] === "refuse" ? -1 : 1));
+        const unresolved = marks[0]?.reason ?? (assignment?.value.unresolved);
+        const spans = [];
+        for (let i = 0; i < this.rawQuoted.length; i++) {
+            const by = this.rawQuoted[i];
+            if (by === undefined)
+                continue;
+            const last = spans[spans.length - 1];
+            if (last !== undefined && last.by === by && last.end === i)
+                spans[spans.length - 1] = { ...last, end: i + 1 };
+            else
+                spans.push({ start: i, end: i + 1, by });
+        }
+        const pieces = this.pieces.map(publicPiece);
+        return {
+            kind: "word",
+            raw: rawText,
+            folded: foldedText,
+            pieces,
+            quotedSpans: spans,
+            expansions: pieces.filter((p) => p.expansion !== undefined),
+            ...(assignment !== undefined ? { assignment } : {}),
+            ...(unresolved !== undefined ? { unresolved } : {}),
+            atCommandPosition,
+        };
+    }
+}
+function scan(source, opts) {
+    const lenient = opts.mode === "lenient";
+    const split = opts.connectors !== "text";
+    const segments = [{ start: 0, end: source.length, connector: "start", tokens: [], substitutions: [] }];
+    const current = () => segments[segments.length - 1];
+    const pendingHeredocs = [];
+    let pendingRedirect;
+    let pendingHeredoc;
+    let inSingle = false;
+    let inDouble = false;
+    let regionBy;
+    let regionAt = 0;
+    let pendingLocale = false;
+    let acceptable = true;
+    let afterLeadingRedirect = false;
+    let wordsSoFar = 0;
+    let commandWord;
+    let w = new WordBuilder();
+    const next = (j) => (lenient ? past(source, j + 1) : j + 1);
+    const quotedBy = () => (inSingle || inDouble ? regionBy : undefined);
+    const take = (ch, src) => w.emit(ch, ch, quotedBy(), src, src + 1);
+    const substitution = (from, to, body, kind) => {
+        const by = quotedBy();
+        w.opaque(source.slice(from, to), SUBSTITUTION_PLACEHOLDER, by, from, to, { kind, yields: kind === "command" && by === undefined ? "many" : "one", body });
+        current().substitutions.push(body);
+    };
+    const endWord = () => {
+        if (!w.started)
+            return undefined;
+        if (w.inSubscript())
+            return fail("unterminated_subscript", w.srcOfRaw[w.subscript.rawStart]);
+        const word = w.finish(acceptable, pendingHeredoc === undefined);
+        const at = w.at;
+        w = new WordBuilder();
+        if (pendingHeredoc !== undefined) {
+            const header = pendingHeredoc;
+            pendingHeredoc = undefined;
+            if (word.folded === "" || /[$`]/.test(word.raw))
+                return fail("heredoc_delimiter_expands", at);
+            const token = { kind: "heredoc", delimiter: word.folded, quoted: /["'\\]/.test(word.raw), body: "" };
+            current().tokens.push(token);
+            pendingHeredocs.push({ delimiter: word.folded, quoted: token.quoted, dash: header.dash, token });
+            if (wordsSoFar > 0)
+                acceptable = false;
+            else
+                afterLeadingRedirect = true;
+            return undefined;
+        }
+        if (pendingRedirect !== undefined) {
+            const op = pendingRedirect;
+            pendingRedirect = undefined;
+            const target = { ...word, atCommandPosition: false };
+            if ((op === ">&" || op === "<&") && word.raw === word.folded && /^(?:\d+-?|-)$/.test(word.folded))
+                current().tokens.push({ kind: "redirect", op });
+            else
+                current().tokens.push({ kind: "redirect", op: op === ">&" ? "&>" : op === "<&" ? "<" : op, target });
+            if (wordsSoFar > 0)
+                acceptable = false;
+            else
+                afterLeadingRedirect = true;
+            return undefined;
+        }
+        current().tokens.push(word);
+        if (acceptable && word.raw === word.folded && (SHELL_CONTROL_PREFIXES.has(word.folded) || GROUP_WORDS.has(word.folded)))
+            return undefined;
+        wordsSoFar++;
+        if (word.assignment !== undefined && word.assignment.bare && acceptable)
+            return undefined;
+        if (acceptable)
+            commandWord = word.raw === word.folded ? word.folded : undefined;
+        acceptable = false;
+        return undefined;
+    };
+    const operandOwed = (at) => pendingRedirect !== undefined ? fail("redirect_without_target", at) : pendingHeredoc !== undefined ? fail("heredoc_without_delimiter", at) : undefined;
+    const closeSegment = (connector, at, from) => {
+        const refusal = endWord() ?? operandOwed(at);
+        if (refusal !== undefined)
+            return refusal;
+        current().end = at;
+        segments.push({ start: from, end: source.length, connector, tokens: [], substitutions: [] });
+        acceptable = true;
+        wordsSoFar = 0;
+        commandWord = undefined;
+        afterLeadingRedirect = false;
+        return undefined;
+    };
+    const readHeredocBodies = (from) => {
+        let p = from;
+        for (const h of pendingHeredocs) {
+            const lines = [];
+            for (;;) {
+                if (p > source.length)
+                    return fail("unterminated_heredoc", from);
+                let nl = source.indexOf("\n", p);
+                let line = nl < 0 ? source.slice(p) : source.slice(p, nl);
+                if (!h.quoted) {
+                    while (nl >= 0 && /(?<!\\)(?:\\\\)*\\$/.test(line)) {
+                        const after = source.indexOf("\n", nl + 1);
+                        line = line.slice(0, -1) + (after < 0 ? source.slice(nl + 1) : source.slice(nl + 1, after));
+                        nl = after;
+                    }
+                }
+                if ((h.dash ? line.replace(/^\t+/, "") : line) === h.delimiter) {
+                    p = nl < 0 ? source.length + 1 : nl + 1;
+                    break;
+                }
+                if (nl < 0)
+                    return fail("unterminated_heredoc", from);
+                lines.push(line);
+                p = nl + 1;
+            }
+            h.token.body = lines.join("\n");
+        }
+        pendingHeredocs.length = 0;
+        return Math.min(p, source.length);
+    };
+    for (let i = 0; i < source.length; i++) {
+        if (lenient && !inSingle) {
+            const j = past(source, i);
+            if (j >= source.length)
+                break;
+            i = j;
+        }
+        const ch = source[i];
+        if (ch === "$" && !inSingle && !inDouble && dollarOpensQuote(source, i)) {
+            const q = next(i);
+            if (source[q] === '"') {
+                w.quoteChar("$", '$"', i);
+                pendingLocale = true;
+                i = q - 1;
+                continue;
+            }
+            if (source[q] === "'") {
+                const close = ansiCQuoteClose(source, q);
+                if (close === undefined)
+                    return fail("unterminated_ansi_c", i);
+                const decoded = decodeAnsiC(source.slice(q + 1, close));
+                const written = source.slice(i, close + 1);
+                if (decoded.unresolved) {
+                    w.mark("ansi_c_escape", w.raw.length);
+                    w.opaque(written, decoded.text, "$'", i, close + 1, undefined);
+                }
+                else
+                    w.emit(written, decoded.text, "$'", i, close + 1);
+                if (w.inSubscript() && decoded.text.includes("]"))
+                    w.subscript.text = true;
+                i = close;
+                continue;
+            }
+        }
+        if (ch === "'" && !inDouble) {
+            if (inSingle) {
+                w.quoteChar(ch, "'", i);
+                inSingle = false;
+                regionBy = undefined;
+            }
+            else {
+                w.quoteChar(ch, "'", i);
+                inSingle = true;
+                regionBy = "'";
+                regionAt = i;
+            }
+            continue;
+        }
+        if (ch === '"' && !inSingle) {
+            if (inDouble) {
+                w.quoteChar(ch, regionBy ?? '"', i);
+                inDouble = false;
+                regionBy = undefined;
+            }
+            else {
+                const by = pendingLocale ? '$"' : '"';
+                pendingLocale = false;
+                if (by === '$"')
+                    w.mark("locale_string", w.raw.length - 1);
+                w.quoteChar(ch, by, i);
+                inDouble = true;
+                regionBy = by;
+                regionAt = i;
+            }
+            continue;
+        }
+        if (inSingle) {
+            if (w.inSubscript() && ch === "]")
+                w.subscript.text = true;
+            take(ch, i);
+            continue;
+        }
+        if (!inDouble && ch === "[" && w.started && w.subscript === undefined && !w.compound && acceptable && IDENTIFIER.test(w.raw)) {
+            w.subscript = { rawStart: w.raw.length, depth: 1, text: false };
+            take(ch, i);
+            continue;
+        }
+        if (w.inSubscript() && !inDouble) {
+            const sub = w.subscript;
+            if (ch === "[") {
+                sub.depth++;
+                take(ch, i);
+                continue;
+            }
+            if (ch === "]") {
+                take(ch, i);
+                if (--sub.depth === 0)
+                    sub.rawEnd = w.raw.length;
+                continue;
+            }
+            if (/[\n;|&<>]/.test(ch)) {
+                sub.text = true;
+                take(ch, i);
+                continue;
+            }
+            if (BLANK.test(ch)) {
+                if (afterLeadingRedirect)
+                    sub.text = true;
+                take(ch, i);
+                continue;
+            }
+            if (!lenient) {
+                take(ch, i);
+                continue;
+            }
+            if (ch === "\\" && source[i + 1] === "]")
+                sub.text = true;
+        }
+        if (w.inSubscript() && inDouble && ch === "]")
+            w.subscript.text = true;
+        if (lenient) {
+            const n1 = next(i);
+            if (ch === "\\") {
+                const escaped = source[i + 1];
+                if (escaped === undefined) {
+                    take(ch, i);
+                    continue;
+                }
+                if (inDouble && !"$`\"\\".includes(escaped))
+                    w.emit(ch + escaped, ch + escaped, quotedBy(), i, i + 2);
+                else
+                    w.emit(ch + escaped, escaped, inDouble ? quotedBy() : "\\", i, i + 2);
+                i++;
+                continue;
+            }
+            if (ch === "`") {
+                const tick = backtickBody(source, i);
+                if (tick === undefined)
+                    return fail("unterminated_command_substitution", i);
+                substitution(i, tick.close + 1, tick.body, "command");
+                i = tick.close;
+                continue;
+            }
+            if (ch === "$" && source[n1] === "(") {
+                const arithmetic = source[next(n1)] === "(";
+                const end = parenBodyEnd(source, n1);
+                if (end === undefined)
+                    return fail("unterminated_command_substitution", i);
+                if (end === "unreadable")
+                    return fail("undelimited_substitution", i);
+                substitution(i, end, source.slice(n1 + 1, end - 1), arithmetic ? "arithmetic" : "command");
+                i = end - 1;
+                continue;
+            }
+            if (ch === "$" && source[n1] === "{") {
+                const end = braceExpansionEnd(source, n1);
+                if (end === undefined)
+                    return fail("unterminated_parameter_expansion", i);
+                const body = source.slice(n1 + 1, end - 1);
+                const text = source.slice(i, end);
+                const name = EXPANSION_NAME.exec(body)?.[1];
+                if (UNREADABLE_EXPANSION_TEXT.test(body))
+                    w.mark("parameter_text", w.raw.length);
+                w.opaque(text, text, quotedBy(), i, end, { kind: "parameter", yields: inDouble ? "one" : "many", body, ...(name !== undefined ? { name } : {}) });
+                i = end - 1;
+                continue;
+            }
+            if (ch === "$") {
+                const head = BARE_PARAMETER.exec(source.slice(n1, n1 + 64))?.[0];
+                if (head !== undefined) {
+                    const text = source.slice(i, n1 + head.length);
+                    w.opaque(text, text, quotedBy(), i, n1 + head.length, { kind: "parameter", yields: inDouble ? "one" : "many", name: head });
+                    i = n1 + head.length - 1;
+                    continue;
+                }
+            }
+            if (inDouble) {
+                take(ch, i);
+                continue;
+            }
+            if ((ch === "<" || ch === ">") && source[n1] === "(") {
+                const end = parenBodyEnd(source, n1);
+                if (end === undefined)
+                    return fail("unterminated_process_substitution", i);
+                if (end === "unreadable")
+                    return fail("undelimited_substitution", i);
+                substitution(i, end, source.slice(n1 + 1, end - 1), "process");
+                i = end - 1;
+                continue;
+            }
+            if (ch === "(" && w.started && w.isAssignmentHead() && (acceptable || (commandWord !== undefined && COMPOUND_OPERAND_COMMANDS.has(commandWord)))) {
+                const end = matchedParen(source, i);
+                if (end === undefined)
+                    return fail("unterminated_compound_value", i);
+                const inner = scan(source.slice(i + 1, end - 1), { mode: "lenient", connectors: "text" });
+                if ("parseFailure" in inner)
+                    return fail(inner.parseFailure.reason, i + 1 + inner.parseFailure.at);
+                const text = source.slice(i, end);
+                w.mark("compound_array_value", w.raw.length);
+                w.opaque(text, text, undefined, i, end, undefined);
+                w.compound = true;
+                current().substitutions.push(...inner.command.substitutions);
+                i = end - 1;
+                continue;
+            }
+            if (w.inSubscript()) {
+                take(ch, i);
+                continue;
+            }
+            if (ch === "#" && !w.started) {
+                const nl = source.indexOf("\n", i);
+                i = nl < 0 ? source.length : nl - 1;
+                continue;
+            }
+            if (ch === "\n") {
+                if (!split) {
+                    const r = endWord();
+                    if (r !== undefined)
+                        return r;
+                    continue;
+                }
+                const refusal = closeSegment("\n", i, i + 1);
+                if (refusal !== undefined)
+                    return refusal;
+                if (pendingHeredocs.length > 0) {
+                    const resume = readHeredocBodies(i + 1);
+                    if (typeof resume !== "number")
+                        return resume;
+                    current().start = resume;
+                    i = resume - 1;
+                }
+                continue;
+            }
+            if (ch === "(" || ch === ")") {
+                const r = endWord() ?? operandOwed(i);
+                if (r !== undefined)
+                    return r;
+                current().tokens.push({ kind: "group", op: ch });
+                acceptable = true;
+                wordsSoFar = 0;
+                commandWord = undefined;
+                afterLeadingRedirect = false;
+                continue;
+            }
+            if (ch === "<" || ch === ">" || (ch === "&" && source[n1] === ">")) {
+                if (ch !== "&" && w.started && w.raw === w.folded && /^\d+$/.test(w.raw))
+                    w = new WordBuilder();
+                else {
+                    const r = endWord() ?? operandOwed(i);
+                    if (r !== undefined)
+                        return r;
+                }
+                const n2 = next(n1);
+                if (ch === "&") {
+                    if (source[n2] === ">") {
+                        pendingRedirect = "&>>";
+                        i = n2;
+                    }
+                    else {
+                        pendingRedirect = "&>";
+                        i = n1;
+                    }
+                    continue;
+                }
+                if (ch === "<" && source[n1] === "<") {
+                    if (source[n2] === "<") {
+                        pendingRedirect = "<<<";
+                        i = n2;
+                        continue;
+                    }
+                    const dash = source[n2] === "-";
+                    pendingHeredoc = { dash };
+                    i = dash ? n2 : n1;
+                    continue;
+                }
+                if (source[n1] === "&") {
+                    pendingRedirect = ch === ">" ? ">&" : "<&";
+                    i = n1;
+                    continue;
+                }
+                if (ch === ">" && (source[n1] === ">" || source[n1] === "|")) {
+                    pendingRedirect = source[n1] === ">" ? ">>" : ">";
+                    i = n1;
+                    continue;
+                }
+                pendingRedirect = ch;
+                continue;
+            }
+            if (split) {
+                let refusal;
+                if (ch === ";")
+                    refusal = closeSegment(";", i, i + 1);
+                else if (ch === "&") {
+                    if (source[n1] === "&") {
+                        refusal = closeSegment("&&", i, n1 + 1);
+                        i = n1;
+                    }
+                    else
+                        refusal = closeSegment("&", i, i + 1);
+                }
+                else if (ch === "|") {
+                    if (source[n1] === "|" || source[n1] === "&") {
+                        refusal = closeSegment(source[n1] === "|" ? "||" : "|", i, n1 + 1);
+                        i = n1;
+                    }
+                    else
+                        refusal = closeSegment("|", i, i + 1);
+                }
+                else if (BLANK.test(ch)) {
+                    refusal = endWord();
+                }
+                else {
+                    take(ch, i);
+                    continue;
+                }
+                if (refusal !== undefined)
+                    return refusal;
+                continue;
+            }
+            if (BLANK.test(ch)) {
+                const r = endWord();
+                if (r !== undefined)
+                    return r;
+                continue;
+            }
+            take(ch, i);
+            continue;
+        }
+        if (!inDouble && split) {
+            if (ch === "&") {
+                if (source[i + 1] !== "&")
+                    return fail("background_operator", i);
+                const r = closeSegment("&&", i, i + 2);
+                if (r !== undefined)
+                    return r;
+                i++;
+                continue;
+            }
+            if (ch === "|") {
+                const double = source[i + 1] === "|";
+                const r = closeSegment(double ? "||" : "|", i, double ? i + 2 : i + 1);
+                if (r !== undefined)
+                    return r;
+                if (double)
+                    i++;
+                continue;
+            }
+            if (ch === ";") {
+                const r = closeSegment(";", i, i + 1);
+                if (r !== undefined)
+                    return r;
+                continue;
+            }
+        }
+        if (!inDouble && WHITESPACE.test(ch)) {
+            const r = endWord();
+            if (r !== undefined)
+                return r;
+            continue;
+        }
+        take(ch, i);
+    }
+    if (inSingle)
+        return fail("unterminated_single_quote", regionAt);
+    if (inDouble)
+        return fail("unterminated_double_quote", regionAt);
+    const tail = endWord() ?? operandOwed(source.length);
+    if (tail !== undefined)
+        return tail;
+    if (pendingHeredocs.length > 0)
+        return fail("unterminated_heredoc", source.length);
+    const out = segments.map((s) => ({
+        text: source.slice(s.start, s.end),
+        connector: s.connector,
+        tokens: s.tokens,
+        substitutions: s.substitutions,
+    }));
+    return { command: { segments: out, substitutions: out.flatMap((s) => s.substitutions) } };
+}
+export function lexShell(source, opts) {
+    const pass = scan(source, opts);
+    return "parseFailure" in pass ? pass : pass.command;
+}
+export function lexShellText(source, opts) {
+    const lexed = lexShell(source, opts);
+    if ("parseFailure" in lexed)
+        return { reject: PARSE_FAILURE_SENTENCE[lexed.parseFailure.reason] };
+    for (const segment of lexed.segments) {
+        for (const token of segment.tokens) {
+            const word = token.kind === "word" ? token : token.kind === "redirect" ? token.target : undefined;
+            if (word?.unresolved !== undefined && UNRESOLVED_VIEW[word.unresolved] === "refuse")
+                return { reject: UNRESOLVED_SENTENCE[word.unresolved] };
+        }
+    }
+    return lexed;
+}
+export function bashOnlyConstructsOf(command) {
+    const found = new Set();
+    const word = (w) => {
+        for (const s of w.quotedSpans) {
+            if (s.by === "$'")
+                found.add("ansi_c_quoting");
+            if (s.by === '$"')
+                found.add("locale_string");
+        }
+        for (const p of w.expansions) {
+            if (p.expansion?.kind === "process")
+                found.add("process_substitution");
+            if (p.expansion?.kind === "brace")
+                found.add("brace_expansion");
+        }
+        if (w.assignment !== undefined) {
+            if (w.assignment.op === "+=")
+                found.add("append_assignment");
+            if (w.assignment.subscript !== undefined)
+                found.add("subscript_assignment");
+            if (w.assignment.form === "compound")
+                found.add("compound_assignment");
+        }
+    };
+    for (const segment of command.segments) {
+        for (const t of segment.tokens) {
+            if (t.kind === "word")
+                word(t);
+            else if (t.kind === "redirect") {
+                if (t.op === "<<<")
+                    found.add("here_string");
+                if (t.op === "&>" || t.op === "&>>")
+                    found.add("redirect_both");
+                if (t.target !== undefined)
+                    word(t.target);
+            }
+        }
+    }
+    return BASH_ONLY_CONSTRUCTS.filter((c) => found.has(c));
+}