@poe-platform/safe-bash 0.1.73 → 0.1.75

package/dist/safe-bash/shell/runtime.js

@@ -7,6 +7,7 @@ import { HereDocumentSyntaxError, hereDocumentWords, parseShellInputUnit, parseS
 import { ShellLimitError, ShellSyntaxError } from "./types.js";
 import { fileInput, ShellInput } from "./input.js";
 import { evaluateArithmetic, prepareArithmetic } from "./arithmetic.js";
+import { defaultMaxParseUnits, ParseBudget } from "./parse-budget.js";
 import { evaluatePositionalArithmetic } from "./arithmetic-parameters.js";
 import { compilePattern, matchesPattern } from "./pattern.js";
 import { byteLocale } from "./locale.js";
@@ -28,6 +29,7 @@ import { EreLedger } from "../commands/regex-execution/ere/limits.js";
 import { compileEre } from "../commands/regex-execution/ere/syntax.js";
 import { matchEre } from "../commands/regex-execution/ere/matcher.js";
 export const defaultLimits = {
+    maxParseUnits: defaultMaxParseUnits,
     maxInputBytes: 32 * 1024 * 1024,
     maxOutputBytes: 16 * 1024 * 1024,
     maxCommands: 10_000,
@@ -65,6 +67,7 @@ export function resolveLimits(...limits) {
 const budgetedSinks = new WeakMap();
 export class Budget {
     limits;
+    parsing;
     values;
     commands = 0;
     iterations = 0;
@@ -79,6 +82,7 @@ export class Budget {
     constructor(limits, signal) {
         this.limits = limits;
         this.signal = signal ? AbortSignal.any([signal, this.controller.signal]) : this.controller.signal;
+        this.parsing = new ParseBudget(limits.maxParseUnits, this.signal, error => this.controller.abort(error));
         this.values = new ValueArena(limits.maxExpansionBytes, limits.maxExpansionFields, () => this.signal.throwIfAborted(), limit => this.fail(limit));
         this.#wallClockDeadline = Date.now() + limits.maxWallClockMs;
         this.#armWallClock();
@@ -1278,7 +1282,7 @@ export class Runtime {
             this.budget.fail("maxExpansionBytes");
         if (name === "OPTIND" && state.getopts?.integer && origin !== "arithmetic") {
             try {
-                value = String(evaluateArithmetic(prepareArithmetic(shellValueText(value) || "0"), this.arithmeticVariables(state)));
+                value = String(evaluateArithmetic(prepareArithmetic(shellValueText(value) || "0", this.budget.parsing), this.arithmeticVariables(state), this.budget.parsing));
             }
             catch (error) {
                 this.rethrowArithmeticControl(error);
@@ -2074,12 +2078,13 @@ export class Runtime {
                 if (command.kind === "arithmetic") {
                     try {
                         return Number(evaluatePositionalArithmetic(command.expression, {
+                            parseBudget: this.budget.parsing,
                             positional: state.positional, arg0: state.arg0 ?? "virtual-bash", owner: arrayStore(state)?.owner,
                             maximumBytes: this.budget.limits.maxExpansionBytes,
                             checkpoint: () => this.signal.throwIfAborted(),
                             requireParameter: (name, value) => this.requireParameter(value, name, state, io),
                             limit: () => this.budget.fail("maxExpansionBytes"),
-                        }, (prepared) => evaluateArithmetic(prepared, this.arithmeticVariables(state, io.diagnosticLine))) === 0n);
+                        }, (prepared) => evaluateArithmetic(prepared, this.arithmeticVariables(state, io.diagnosticLine), this.budget.parsing)) === 0n);
                     }
                     catch (error) {
                         this.rethrowArithmeticControl(error);
@@ -2264,7 +2269,7 @@ export class Runtime {
         let words = 0;
         const warnings = [];
         try {
-            for (const word of hereDocumentWords(document, line, byteLocale(state.variables), warnings)) {
+            for (const word of hereDocumentWords(document, line, byteLocale(state.variables), warnings, this.budget.parsing)) {
                 this.signal.throwIfAborted();
                 for (const warning of warnings.splice(0))
                     await writeText(io.stderr, `shell: warning: ${warning}\n`);
@@ -3274,7 +3279,7 @@ export class Runtime {
         try {
             do {
                 this.signal.throwIfAborted();
-                const unit = parseShellUnit(source, position, byteLocale(state.variables));
+                const unit = parseShellUnit(source, position, byteLocale(state.variables), this.budget.parsing);
                 for (const warning of unit.script.warnings ?? [])
                     await writeText(io.stderr, `${io.scriptName}: warning: ${warning}\n`);
                 if (unit.script.lists.length) {
@@ -3318,7 +3323,7 @@ export class Runtime {
                 source += this.sourceText(bytes, io.scriptName ?? "shell");
             const unitIO = { ...io, diagnosticOffset: offset };
             try {
-                const unit = eof ? parseShellUnit(source, 0, byteLocale(state.variables)) : parseShellInputUnit(source, byteLocale(state.variables));
+                const unit = eof ? parseShellUnit(source, 0, byteLocale(state.variables), this.budget.parsing) : parseShellInputUnit(source, byteLocale(state.variables), this.budget.parsing);
                 if (unit) {
                     for (const warning of unit.script.warnings ?? [])
                         await writeText(io.stderr, `${io.scriptName}: warning: ${warning}\n`);
@@ -3585,7 +3590,7 @@ export class Runtime {
             let position = 0;
             do {
                 this.signal.throwIfAborted();
-                const unit = parseShellUnit(source, position, byteLocale(context.env));
+                const unit = parseShellUnit(source, position, byteLocale(context.env), this.budget.parsing);
                 units.push(unit.script);
                 position = unit.next;
             } while (position < source.length);
@@ -3622,7 +3627,7 @@ export class Runtime {
         try {
             do {
                 this.signal.throwIfAborted();
-                const unit = parseShellUnit(source, position, byteLocale(state.variables));
+                const unit = parseShellUnit(source, position, byteLocale(state.variables), this.budget.parsing);
                 for (const warning of unit.script.warnings ?? [])
                     await writeText(io.stderr, `${io.scriptName ?? "shell"}: warning: ${warning}\n`);
                 if (unit.script.lists.length) {
@@ -3992,7 +3997,7 @@ export class Runtime {
         for (let index = offset; index < args.length; index++) {
             this.signal.throwIfAborted();
             try {
-                value = evaluateArithmetic(prepareArithmetic(args[index]), variables);
+                value = evaluateArithmetic(prepareArithmetic(args[index], this.budget.parsing), variables, this.budget.parsing);
             }
             catch (error) {
                 this.rethrowArithmeticControl(error);
@@ -4725,9 +4730,12 @@ export class Runtime {
                     else {
                         let index;
                         try {
-                            index = numericIndex(literalIndex(selector, 0));
+                            index = numericIndex(literalIndex(selector, 0, this.budget.parsing));
                         }
-                        catch {
+                        catch (error) {
+                            this.signal.throwIfAborted();
+                            if (error instanceof ShellLimitError)
+                                throw error;
                             await this.diagnostic(context, "indexed array: unsupported subscript");
                             status = 2;
                             continue;
@@ -4942,12 +4950,13 @@ export class Runtime {
         if (part.kind === "arithmetic") {
             try {
                 return String(evaluatePositionalArithmetic(part.expression, {
+                    parseBudget: this.budget.parsing,
                     positional: state.positional, arg0: state.arg0 ?? "virtual-bash", owner: arrayStore(state)?.owner,
                     maximumBytes: this.budget.limits.maxExpansionBytes,
                     checkpoint: () => this.signal.throwIfAborted(),
                     requireParameter: (name, value) => this.requireParameter(value, name, state, io, part.line),
                     limit: () => this.budget.fail("maxExpansionBytes"),
-                }, (prepared) => evaluateArithmetic(prepared, this.arithmeticVariables(state, io.diagnosticLine ?? part.line))));
+                }, (prepared) => evaluateArithmetic(prepared, this.arithmeticVariables(state, io.diagnosticLine ?? part.line), this.budget.parsing)));
             }
             catch (error) {
                 this.rethrowArithmeticControl(error);
@@ -5113,7 +5122,7 @@ export class Runtime {
             }
             this.signal.throwIfAborted();
             try {
-                return { value: evaluateArithmetic(prepareArithmetic(source), variables), source };
+                return { value: evaluateArithmetic(prepareArithmetic(source, this.budget.parsing), variables, this.budget.parsing), source };
             }
             catch (error) {
                 this.rethrowArithmeticControl(error);
@@ -5264,11 +5273,22 @@ export class Runtime {
         const arrayOwned = word.parts.some(part => part.kind === "variable" && (getArraySelector(part) !== undefined || arrayStore(state)?.get(part.name) !== undefined));
         const owner = arrayOwned ? requireArrays(state).owner : undefined;
         const holding = owner?.hold();
+        const scratch = !owner && split && state.variables.IFS !== "" && word.parts.some(part => !part.quoted && part.kind !== "text")
+            ? this.budget.values.scope() : undefined;
         try {
             if (owner)
                 await this.prepareArrayObservers(state, owner);
             owner?.reserve({ metadata: 128 + word.parts.length * 32, allocatedSlots: word.parts.length + 1, work: word.parts.length + 5 });
-            const fields = [{ value: "", fragments: [], bytes: false, pattern: "", present: false }];
+            scratch?.reserve(word.parts.length * 32, 0);
+            const fields = [];
+            const addField = () => {
+                if (fields.length >= this.budget.limits.maxExpansionFields)
+                    this.budget.fail("maxExpansionFields");
+                scratch?.reserve(32, 0);
+                owner?.reserve({ metadata: 32, allocatedSlots: 1, work: 3 });
+                fields.push({ fragments: [], bytes: false, patterns: undefined, present: false });
+            };
+            addField();
             let expansionBytes = 0;
             const append = (value, glob, present) => {
                 const text = shellValueText(value);
@@ -5276,21 +5296,59 @@ export class Runtime {
                 if (size > this.budget.limits.maxExpansionBytes - expansionBytes)
                     this.budget.fail("maxExpansionBytes");
                 expansionBytes += size;
-                regexAppend?.(text, !glob);
                 const field = fields.at(-1);
+                let escapes = 0;
+                if (!glob) {
+                    const special = conditionalPattern ? "\\*?[]-^()|+!@" : "\\*?[]-^";
+                    for (const character of text)
+                        if (special.includes(character))
+                            escapes++;
+                }
+                scratch?.reserve(32, 0);
                 if (owner)
-                    owner.reserve({ payload: exactSum(Buffer.byteLength(field.value) + size, Buffer.byteLength(field.pattern) + (glob ? size : size * 2)), metadata: 64, work: text.length + 8 });
+                    owner.reserve({ payload: size + (escapes ? size + escapes : 0), metadata: 64, work: text.length + 8 });
+                if (escapes) {
+                    scratch?.reserve((field.patterns ? 32 : 32 * (field.fragments.length + 1)) + (text.length + escapes) * 2, 0);
+                    field.patterns ??= field.fragments.map(shellValueText);
+                    field.patterns.push(text.replace(conditionalPattern ? /[\\*?[\]\-^()|+!@]/gu : /[\\*?[\]\-^]/gu, "\\$&"));
+                }
+                else if (field.patterns) {
+                    scratch?.reserve(32, 0);
+                    field.patterns.push(text);
+                }
                 if (typeof value !== "string" || field.bytes) {
-                    io[valueScope]?.reserve(32 * (field.bytes ? 1 : field.fragments.length + 1), field.bytes ? 1 : field.fragments.length + 1);
+                    if (!scratch)
+                        io[valueScope]?.reserve(32 * (field.bytes ? 1 : field.fragments.length + 1), field.bytes ? 1 : field.fragments.length + 1);
                     if (typeof value !== "string")
                         io[valueScope]?.hold(value);
                     field.bytes = true;
                 }
+                regexAppend?.(text, !glob);
                 field.fragments.push(value);
-                field.value += text;
-                field.pattern += glob ? text : text.replace(conditionalPattern ? /[\\*?[\]\-^()|+!@]/gu : /[\\*?[\]\-^]/gu, "\\$&");
                 field.present ||= present;
             };
+            const appendSplit = async (value) => {
+                const separators = state.variables.IFS ?? " \t\n";
+                let boundary = false;
+                for await (const piece of this.splitValue(value, separators, io, scratch)) {
+                    if (typeof piece === "string" && separators.includes(piece)) {
+                        if (!" \t\n".includes(piece)) {
+                            fields.at(-1).present = true;
+                            addField();
+                        }
+                        else if (fields.at(-1).present)
+                            boundary = true;
+                    }
+                    else {
+                        if (boundary)
+                            addField();
+                        boundary = false;
+                        append(piece, true, true);
+                    }
+                }
+                if (boundary)
+                    addField();
+            };
             const parts = word.parts.map((part) => ({ part, splitText: false }));
             for (let index = 0; index < parts.length; index++) {
                 const { part, splitText } = parts[index];
@@ -5299,6 +5357,7 @@ export class Runtime {
                     const value = this.variable(state, part.name);
                     const missing = value === undefined || (part.operator.startsWith(":") && value === "");
                     if (part.operator.endsWith("+") ? !missing : missing) {
+                        scratch?.reserve(part.alternate.parts.length * 32, 0);
                         const alternate = part.alternate.parts.map((entry) => ({ part: copyArraySelector(entry, { ...entry, quoted: entry.quoted || part.quoted }), splitText: true }));
                         if (!alternate.length && part.quoted)
                             append("", false, true);
@@ -5310,41 +5369,13 @@ export class Runtime {
                 if (part.kind === "variable" && selector?.kind === "members" && !part.length && split && (!part.quoted || selector.separator === "@")) {
                     const members = await this.arrayMembers(part.name, state);
                     for (let position = 0; position < members.length; position++) {
-                        if (position > 0) {
-                            owner?.reserve({ metadata: 32, allocatedSlots: 1, work: 3 });
-                            fields.push({ value: "", fragments: [], bytes: false, pattern: "", present: false });
-                        }
+                        if (position > 0)
+                            addField();
                         const value = members[position];
                         if (part.quoted || state.variables.IFS === "")
                             append(value, !part.quoted, part.quoted || value.length > 0);
-                        else {
-                            const separators = state.variables.IFS ?? " \t\n";
-                            let boundary = false;
-                            for (const character of value) {
-                                if (separators.includes(character)) {
-                                    if (!/[ \t\n]/u.test(character)) {
-                                        fields.at(-1).present = true;
-                                        owner?.reserve({ metadata: 32, allocatedSlots: 1, work: 3 });
-                                        fields.push({ value: "", fragments: [], bytes: false, pattern: "", present: false });
-                                    }
-                                    else if (fields.at(-1).present)
-                                        boundary = true;
-                                }
-                                else {
-                                    if (boundary) {
-                                        owner?.reserve({ metadata: 32, allocatedSlots: 1, work: 3 });
-                                        fields.push({ value: "", fragments: [], bytes: false, pattern: "", present: false });
-                                    }
-                                    boundary = false;
-                                    append(character, true, true);
-                                }
-                                await owner.ledger.checkpoint(this.signal);
-                            }
-                            if (boundary) {
-                                owner?.reserve({ metadata: 32, allocatedSlots: 1, work: 3 });
-                                fields.push({ value: "", fragments: [], bytes: false, pattern: "", present: false });
-                            }
-                        }
+                        else
+                            await appendSplit(value);
                     }
                 }
                 else if (part.kind === "text" && !splitText) {
@@ -5356,7 +5387,7 @@ export class Runtime {
                 else if (part.kind === "variable" && part.name === "@" && part.quoted && !part.operator && split) {
                     for (let position = 0; position < state.positional.length; position++) {
                         if (position > 0)
-                            fields.push({ value: "", fragments: [], bytes: false, pattern: "", present: false });
+                            addField();
                         append(stateMonitor(state)?.positionals.get(String(position), state.positional[position]) ?? state.positional[position], false, true);
                     }
                     if (state.positional.length === 0 && word.parts.every((entry) => (entry.kind === "text" && entry.value === "") || entry === part))
@@ -5366,30 +5397,8 @@ export class Runtime {
                     const value = part.kind === "text" ? part.byteValue ?? part.value : await this.valuePart(part, state, io, hereString);
                     if (part.quoted || !split || state.variables.IFS === "")
                         append(value, !part.quoted, quotedPresence || !split || shellValueByteLength(value) > 0);
-                    else {
-                        const separators = state.variables.IFS ?? " \t\n";
-                        let boundary = false;
-                        const pieces = this.splitValue(value, separators, io);
-                        for (const character of pieces) {
-                            const text = shellValueText(character);
-                            if (typeof character === "string" && separators.includes(character)) {
-                                if (!/[ \t\n]/u.test(text)) {
-                                    fields.at(-1).present = true;
-                                    fields.push({ value: "", fragments: [], bytes: false, pattern: "", present: false });
-                                }
-                                else if (fields.at(-1).present)
-                                    boundary = true;
-                            }
-                            else {
-                                if (boundary)
-                                    fields.push({ value: "", fragments: [], bytes: false, pattern: "", present: false });
-                                boundary = false;
-                                append(character, true, true);
-                            }
-                        }
-                        if (boundary)
-                            fields.push({ value: "", fragments: [], bytes: false, pattern: "", present: false });
-                    }
+                    else
+                        await appendSplit(value);
                 }
                 if (fields.length > this.budget.limits.maxExpansionFields)
                     this.budget.fail("maxExpansionFields");
@@ -5401,15 +5410,27 @@ export class Runtime {
             for (const field of fields) {
                 if (!field.present && split)
                     continue;
+                if (scratch && field.fragments.length > 1 && !field.bytes)
+                    scratch.reserve(field.fragments.reduce((bytes, value) => bytes + shellValueText(value).length * 2, 0), 0);
                 const assembled = concatShellValues(field.fragments, io[valueScope]);
                 const projection = shellValueText(assembled);
-                const expanded = split ? await this.glob(projection, field.pattern, state) : [pattern ? field.pattern : projection];
+                if (field.bytes && field.fragments.length > 1 && !field.patterns) {
+                    scratch?.reserve(field.fragments.length * 32, 0);
+                    field.patterns = field.fragments.map(shellValueText);
+                }
+                if (scratch && field.patterns && field.patterns.length > 1)
+                    scratch.reserve(field.patterns.reduce((bytes, text) => bytes + text.length * 2, 0), 0);
+                const fieldPattern = field.patterns ? field.patterns.join("") : projection;
+                const expanded = split ? await this.glob(projection, fieldPattern, state) : [pattern ? fieldPattern : projection];
                 for (const text of expanded) {
+                    if (result.length >= this.budget.limits.maxExpansionFields)
+                        this.budget.fail("maxExpansionFields");
                     const value = !pattern && expanded.length === 1 && text === projection ? assembled : text;
                     const size = shellValueByteLength(value);
                     if (size > this.budget.limits.maxExpansionBytes - resultBytes)
                         this.budget.fail("maxExpansionBytes");
                     resultBytes += size;
+                    scratch?.reserve(32, 0);
                     owner?.reserve({ metadata: 32, allocatedSlots: 1, work: 3 });
                     result.push(value);
                 }
@@ -5419,17 +5440,54 @@ export class Runtime {
             return result;
         }
         finally {
+            scratch?.close();
             holding?.release();
         }
     }
-    *splitValue(value, separators, io) {
+    splitWork;
+    async *splitValue(value, separators, io, scratch) {
+        this.budget.cpuCheckpoint();
+        const work = this.splitWork ??= { scanned: 0 };
         if (typeof value === "string" || Array.from(separators).some(character => character.charCodeAt(0) > 127)) {
-            yield* shellValueText(value);
+            const text = shellValueText(value);
+            const slice = (start, end) => {
+                if (start === 0 && end === text.length)
+                    return text;
+                scratch?.reserve((end - start) * 2, 0);
+                return text.slice(start, end);
+            };
+            let start = 0;
+            for (let index = 0; index < text.length;) {
+                const character = String.fromCodePoint(text.codePointAt(index));
+                const end = index + character.length;
+                if (separators.includes(character)) {
+                    if (start < index)
+                        yield slice(start, index);
+                    yield character;
+                    start = end;
+                }
+                else if (end - start >= 4096) {
+                    yield slice(start, end);
+                    start = end;
+                }
+                work.scanned += character.length;
+                index = end;
+                if (work.scanned >= 4096) {
+                    work.scanned = 0;
+                    await yieldTurn(this.signal);
+                }
+            }
+            if (start < text.length)
+                yield slice(start, text.length);
             return;
         }
         const bytes = shellValueBytes(value, io[valueScope]);
         let start = 0;
         for (let index = 0; index < bytes.length; index++) {
+            if (++work.scanned >= 4096) {
+                work.scanned = 0;
+                await yieldTurn(this.signal);
+            }
             const byte = bytes[index];
             if (byte > 127 || !separators.includes(String.fromCharCode(byte)))
                 continue;