@poe-platform/safe-bash 0.1.526 → 0.1.528

package/dist/safe-bash/shell/runtime.js

@@ -30,7 +30,8 @@ import { cloneGetoptsState, createGetoptsInput, createGetoptsState, GetoptsError
 import { activateChildCancellation, prepareChildCancellation, selectRuntimeCancellationOutcome, subscribeCancellation, } from "./cancellation.js";
 import { getArrayAssignment, getArraySelector, copyArraySelector, numericIndex, literalIndex, isQuoteMarker, prefixNameQuoteGroups } from "./arrays/syntax.js";
 import { ArrayFailure, ArrayOwner, exactSum } from "./arrays/ledger.js";
-import { controlNames, IndexedBinding, textToken } from "./arrays/bindings.js";
+import { controlNames, IndexedBinding, textToken, valueToken } from "./arrays/bindings.js";
+import { collectMapfile, mapfileOptions, MapfileUsageError } from "./mapfile.js";
 import { arrayStore, guestArrays, requireArrays, snapshotState, stateMonitor, trackState } from "./arrays/state.js";
 import { publishPipelineStatus } from "./pipestatus.js";
 import { EreProfileLimitError, EreSyntaxError, EreUnsupportedError } from "../commands/regex-execution/ere/errors.js";
@@ -58,7 +59,7 @@ export const defaultLimits = {
     pipeHighWaterMark: 64 * 1024,
 };
 const shellBuiltinNames = new Set([
-    ":", "true", "false", "pwd", "cd", "set", "shift", "export", "local", "unset", "read",
+    ":", "true", "false", "pwd", "cd", "set", "shift", "export", "local", "unset", "read", "mapfile", "readarray",
     "exit", "return", "break", "continue", "command", "builtin", "type", "readonly", "echo", "printf", "test", "[", ".", "source", "eval", "getopts", "let", "pushd", "dirs", "popd", "shopt",
 ]);
 const implementedBuiltins = new Set([...shellBuiltinNames].filter(name => !["echo", "printf", "test", "["].includes(name)));
@@ -1081,6 +1082,7 @@ class InvocationCancellationOwner {
         this.#failures.push(...result.failures);
     }
 }
+const mapfileCallbackStates = new WeakSet();
 export class Runtime {
     fs;
     commands;
@@ -1409,6 +1411,16 @@ export class Runtime {
         if (error instanceof NounsetFailure || error instanceof Flow || error instanceof ShellLimitError || error instanceof ShellSyntaxError)
             throw error;
     }
+    arithmeticValue(program, state, io) {
+        return evaluatePositionalArithmetic(program, {
+            parseBudget: this.budget.parsing,
+            positional: state.positional, arg0: state.arg0 ?? "virtual-bash", owner: arrayStore(state)?.owner,
+            maximumBytes: this.budget.limits.maxExpansionBytes,
+            checkpoint: () => this.signal.throwIfAborted(),
+            requireParameter: (name, value) => this.requireParameter(value, name, state, io),
+            limit: () => this.budget.fail("maxExpansionBytes"),
+        }, prepared => evaluateArithmetic(prepared, this.arithmeticVariables(state, io.diagnosticLine), this.budget.parsing));
+    }
     arithmeticVariables(state, line) {
         return new Proxy(state.variables, {
             get: (target, key) => {
@@ -1436,7 +1448,7 @@ export class Runtime {
             this.writeVariable(state, name, value, origin);
             return;
         }
-        await this.arrayZero(state, name, async () => shellValueText(value));
+        await this.arrayZero(state, name, async () => value);
     }
     async prepareVariable(state, name, saved, scalarLegacy = false) {
         const store = requireArrays(state);
@@ -1619,6 +1631,7 @@ export class Runtime {
         const store = requireArrays(state);
         const operation = ArrayOwner.create(store.owner.ledger, store.owner);
         const holding = store.owner.hold();
+        const valueAllocation = this.budget.values.scope();
         let staged;
         try {
             const watch = await store.watch(name, operation, this.signal);
@@ -1639,8 +1652,10 @@ export class Runtime {
             if (!current)
                 throw new ArrayFailure("stale binding");
             staged = await current.copy(this.signal);
-            const value = append ? await this.arrayJoin(operation, [current.get(0) ?? "", expanded], "") : expanded;
-            const token = await textToken(staged.owner, value, this.signal);
+            const previous = current.values.get(0)?.text.rawValue ?? current.get(0) ?? "";
+            const value = !append ? expanded : typeof previous === "string" && typeof expanded === "string"
+                ? await this.arrayJoin(operation, [previous, expanded], "") : concatShellValues([previous, expanded], valueAllocation);
+            const token = await valueToken(staged.owner, value, this.signal);
             try {
                 staged.insert(0, token);
             }
@@ -1673,6 +1688,7 @@ export class Runtime {
                 await operation.close();
             }
             finally {
+                valueAllocation.close();
                 holding.release();
             }
         }
@@ -1747,7 +1763,7 @@ export class Runtime {
             const preserve = assignment.kind === "element" || assignment.append;
             staged = preserve && current ? await current.copy(this.signal) : IndexedBinding.create(store.owner);
             if (preserve && !current && state.variables[name] !== undefined) {
-                const token = await textToken(staged.owner, state.variables[name], this.signal);
+                const token = await valueToken(staged.owner, stateMonitor(state)?.values.get(name, state.variables[name]) ?? state.variables[name], this.signal);
                 try {
                     staged.insert(0, token);
                 }
@@ -1761,7 +1777,7 @@ export class Runtime {
             const insert = async (index, value) => {
                 if (index > 2147483647)
                     throw new ArrayFailure("index outside 0..2147483647");
-                const token = await textToken(staged.owner, value, this.signal);
+                const token = await valueToken(staged.owner, value, this.signal);
                 try {
                     staged.insert(index, token);
                 }
@@ -1771,12 +1787,14 @@ export class Runtime {
                 }
                 writes++;
             };
+            const join = async (values) => values.every(value => typeof value === "string")
+                ? this.arrayJoin(operation, values, "") : concatShellValues(values, io[valueScope]);
             if (assignment.kind === "element") {
                 const index = numericIndex(assignment.index);
-                const fields = await this.word(assignment.value, state, io, false);
-                let value = await this.arrayJoin(operation, fields, "");
+                const fields = await this.valueWord(assignment.value, state, io, false);
+                let value = await join(fields);
                 if (assignment.append)
-                    value = await this.arrayJoin(operation, [staged.get(index) ?? "", value], "");
+                    value = await join([staged.values.get(index)?.text.rawValue ?? staged.get(index) ?? "", value]);
                 await insert(index, value);
             }
             else
@@ -1790,17 +1808,17 @@ export class Runtime {
                             expandedEntry = true;
                             const values = await this.valueWord(expanded, state, io, true, false, false, false, undefined, false);
                             for (const value of values) {
-                                await insert(cursor, shellValueText(value));
+                                await insert(cursor, value);
                                 cursor++;
                             }
                         }
                         if (expandedEntry)
                             continue;
                     }
-                    const fields = await this.word(entry.value, state, io, entry.index === undefined);
+                    const fields = await this.valueWord(entry.value, state, io, entry.index === undefined);
                     if (entry.index) {
                         const index = numericIndex(entry.index);
-                        await insert(index, await this.arrayJoin(operation, fields, ""));
+                        await insert(index, await join(fields));
                         cursor = index + 1;
                     }
                     else
@@ -2190,14 +2208,7 @@ export class Runtime {
                 }
                 if (command.kind === "arithmetic") {
                     try {
-                        return Number(evaluatePositionalArithmetic(command.expression, {
-                            parseBudget: this.budget.parsing,
-                            positional: state.positional, arg0: state.arg0 ?? "virtual-bash", owner: arrayStore(state)?.owner,
-                            maximumBytes: this.budget.limits.maxExpansionBytes,
-                            checkpoint: () => this.signal.throwIfAborted(),
-                            requireParameter: (name, value) => this.requireParameter(value, name, state, io),
-                            limit: () => this.budget.fail("maxExpansionBytes"),
-                        }, (prepared) => evaluateArithmetic(prepared, this.arithmeticVariables(state, io.diagnosticLine), this.budget.parsing)) === 0n);
+                        return Number(this.arithmeticValue(command.expression, state, io) === 0n);
                     }
                     catch (error) {
                         this.rethrowArithmeticControl(error);
@@ -2267,6 +2278,37 @@ export class Runtime {
                                 break;
                         }
                     }
+                    else if (command.kind === "arithmetic-for") {
+                        const evaluate = async (program) => {
+                            if (!program)
+                                return 1n;
+                            try {
+                                return this.arithmeticValue(program, state, io);
+                            }
+                            catch (error) {
+                                this.rethrowArithmeticControl(error);
+                                await this.diagnostic(io, `((: ${message(error, this.budget.onInternalError)}`);
+                                return undefined;
+                            }
+                        };
+                        if (await evaluate(command.expressions[0]) === undefined)
+                            return 1;
+                        while (true) {
+                            this.budget.loop();
+                            await yieldTurn(this.signal);
+                            const condition = await evaluate(command.expressions[1]);
+                            if (condition === undefined)
+                                return 1;
+                            if (condition === 0n)
+                                break;
+                            const result = await this.loopBody(command.body, state, io);
+                            status = result.status;
+                            if (result.stop)
+                                break;
+                            if (await evaluate(command.expressions[2]) === undefined)
+                                return 1;
+                        }
+                    }
                     else if (command.kind === "select") {
                         const values = command.words ? await this.valueWords(command.words, state, io) : this.positionalValues(state);
                         if (!values.length)
@@ -2784,8 +2826,8 @@ export class Runtime {
                         throw new ArrayFailure("indexed binding cannot be a command prefix");
                     }
                     await this.arrayZero(state, assignment.name, async () => {
-                        const fields = await this.word(assignment.value, state, io, false);
-                        return this.arrayJoin(requireArrays(state).owner, fields, "");
+                        const fields = await this.valueWord(assignment.value, state, io, false);
+                        return fields.every(value => typeof value === "string") ? this.arrayJoin(requireArrays(state).owner, fields, "") : concatShellValues(fields, io[valueScope]);
                     }, assignment.append);
                     continue;
                 }
@@ -3127,6 +3169,9 @@ export class Runtime {
                         functionRestoration?.close();
                         throw error;
                     }
+                    const callerLoopDepth = state.loopDepth;
+                    if (mapfileCallbackStates.has(state))
+                        state.loopDepth = 0;
                     try {
                         return { exitCode: await this.command(body, state, { ...io, ...context, scriptName: body.sourceName ?? io.scriptName ?? "shell" }) };
                     }
@@ -3141,6 +3186,7 @@ export class Runtime {
                             savedPositionals.close();
                             state.positionalSetVersion = positionalSetVersion;
                             state.functionDepth--;
+                            state.loopDepth = callerLoopDepth;
                             state.depth--;
                             state.locals.pop();
                         };
@@ -4759,6 +4805,135 @@ export class Runtime {
             allocation.close();
         }
     }
+    async mapfileBuiltin(context, state) {
+        const allocation = this.budget.values.scope();
+        const work = { remaining: this.budget.limits.maxExpansionBytes * 8, signal: this.signal, exhausted: () => this.budget.fail("maxExpansionBytes") };
+        let pinned;
+        let releaseHolding;
+        try {
+            const options = await mapfileOptions(context, work, allocation);
+            if (state.readonlyVariables?.has(options.name))
+                throw new MapfileUsageError(`${options.name}: readonly variable`, 1);
+            if (controlNames.has(options.name) || state.exported.has(options.name))
+                throw new ArrayFailure("control or exported binding cannot be indexed");
+            const store = requireArrays(state);
+            const holding = store.owner.hold();
+            releaseHolding = () => holding.release();
+            if (!options.preserve || !store.get(options.name))
+                await this.arrayAssignment({ kind: "compound", name: options.name, append: options.preserve, entries: [] }, state, context);
+            const entry = store.bindings.get(options.name);
+            allocation.reserve(128 + Buffer.byteLength(options.name) * 2, 0);
+            pinned = entry.binding.retain();
+            const input = context.stdin instanceof ShellInput ? context.stdin : new ShellInput(context.stdin, this.budget, this.signal);
+            let pendingFlow;
+            await collectMapfile(options, input, {
+                allocation: () => this.budget.values.scope(),
+                loop: () => this.budget.loop(),
+                write: async (index, value) => {
+                    if (pinned !== entry.binding) {
+                        const previous = pinned;
+                        pinned = entry.binding.retain();
+                        await previous.release();
+                    }
+                    const operation = ArrayOwner.create(store.owner.ledger, store.owner);
+                    let staged;
+                    try {
+                        const current = pinned;
+                        const attached = store.bindings.get(options.name) === entry;
+                        const watch = attached ? await store.watch(options.name, operation, this.signal) : undefined;
+                        const supersede = attached ? await stateMonitor(state).prepareTypedPublication(options.name, operation, this.signal) : undefined;
+                        if (attached)
+                            await this.prepareArrayObservers(state, operation);
+                        const tickets = operation.reserve({ generation: true, version: true, epoch: true, work: 8 });
+                        // Exclude the mapfile pin and the live name's reference from COW.
+                        staged = current.references > (attached ? 2 : 1) ? await current.copy(this.signal) : current.retain();
+                        const token = await valueToken(staged.owner, value, this.signal);
+                        try {
+                            this.signal.throwIfAborted();
+                            if (watch && !watch.valid())
+                                throw new ArrayFailure("stale binding");
+                            staged.insert(index, token);
+                        }
+                        catch (error) {
+                            token.release();
+                            throw error;
+                        }
+                        if (attached) {
+                            let released;
+                            stateMonitor(state).publish(tickets, options.name, () => {
+                                supersede();
+                                released = store.publish(options.name, staged, tickets);
+                            });
+                            staged = undefined;
+                            await released;
+                        }
+                        else {
+                            const previous = pinned;
+                            pinned = staged;
+                            entry.binding = staged;
+                            staged = undefined;
+                            await previous.release();
+                        }
+                    }
+                    finally {
+                        try {
+                            await staged?.release();
+                        }
+                        finally {
+                            await operation.close();
+                        }
+                    }
+                },
+                callback: async (source, index, value) => {
+                    if (pendingFlow)
+                        return;
+                    const callbackAllocation = this.budget.values.scope();
+                    const nested = mapfileCallbackStates.has(state);
+                    try {
+                        const quoted = shellValueText(await transformParameter(value, "Q", { maximumBytes: this.budget.limits.maxExpansionBytes, byteLocale: byteLocale(state.variables), work, allocation: callbackAllocation }));
+                        if (Buffer.byteLength(source) + Buffer.byteLength(quoted) + String(index).length + 2 > this.budget.limits.maxExpansionBytes)
+                            this.budget.fail("maxExpansionBytes");
+                        mapfileCallbackStates.add(state);
+                        await this.evalBuiltin({ ...context, args: [source, String(index), quoted] }, state, context, false);
+                    }
+                    catch (error) {
+                        if (error instanceof Flow && (error.kind === "break" || error.kind === "continue"))
+                            pendingFlow = error;
+                        else
+                            throw error;
+                    }
+                    finally {
+                        if (!nested)
+                            mapfileCallbackStates.delete(state);
+                        callbackAllocation.close();
+                    }
+                },
+            });
+            if (pendingFlow)
+                throw pendingFlow;
+            return 0;
+        }
+        catch (error) {
+            this.signal.throwIfAborted();
+            if (!(error instanceof MapfileUsageError))
+                throw error;
+            await this.diagnostic(context, `${context.command}: ${error.message}`);
+            return error.status;
+        }
+        finally {
+            try {
+                await pinned?.release();
+            }
+            finally {
+                try {
+                    releaseHolding?.();
+                }
+                finally {
+                    allocation.close();
+                }
+            }
+        }
+    }
     async builtin(context, state, assignments, diagnose, suppressSpecial = false) {
         const { command, args, stdout, stderr } = context;
         if (command === ":" || command === "true")
@@ -4769,6 +4944,8 @@ export class Runtime {
             return this.shoptBuiltin(context, state);
         if (command === "let")
             return this.letBuiltin(context, state);
+        if (command === "mapfile" || command === "readarray")
+            return this.mapfileBuiltin(context, state);
         if (command === "getopts")
             return this.getoptsBuiltin(context, state);
         if (command === "pushd" || command === "dirs" || command === "popd")
@@ -5389,7 +5566,7 @@ export class Runtime {
         if (part.kind === "variable" && part.transform) {
             const selector = getArraySelector(part);
             if (selector?.kind === "members" || part.name === "@" || part.name === "*") {
-                const members = selector ? await this.arrayMembers(part.name, state) : this.positionalValues(state);
+                const members = selector ? await this.arrayMembers(part.name, state, io) : this.positionalValues(state);
                 const ifs = state.variables.IFS ?? " ";
                 const separator = ifs.length ? String.fromCodePoint(ifs.codePointAt(0)) : "";
                 const fragments = [];
@@ -5439,6 +5616,8 @@ export class Runtime {
         const token = index === undefined || selector?.kind === "members" ? undefined : binding?.values.get(index)?.text;
         token?.retain();
         try {
+            if (token?.rawValue && part.kind === "variable" && !part.length && !part.operator && !part.substring)
+                return shellValueFromBytes(shellValueBytes(token.rawValue, io[valueScope]), io[valueScope]);
             const value = await this.partValue(part, state, io, hereString);
             if (binding)
                 await textToken(requireArrays(state).owner, shellValueText(value), this.signal);
@@ -5568,8 +5747,12 @@ export class Runtime {
             }
             if (part.length)
                 return String(binding?.values.size ?? (state.variables[part.name] === undefined ? 0 : 1));
-            const values = await this.arrayMembers(part.name, state);
-            return this.arrayJoin(store.owner, values, Array.from(state.variables.IFS ?? " ")[0] ?? "");
+            const values = await this.arrayMembers(part.name, state, io);
+            const separator = Array.from(state.variables.IFS ?? " ")[0] ?? "";
+            if (values.every(value => typeof value === "string"))
+                return this.arrayJoin(store.owner, values, separator);
+            io[valueScope]?.reserve(values.length * 32 + 64, 0);
+            return concatShellValues(values.flatMap((value, index) => index ? [separator, value] : [value]), io[valueScope]);
         }
         let value = part.name === "?" ? String(state.status)
             : part.name === "-" ? `${state.errexit ? "e" : ""}${state.nounset ? "u" : ""}${state.braceexpand !== false ? "B" : ""}`
@@ -6046,7 +6229,7 @@ export class Runtime {
                     }
                 }
                 else if (part.kind === "variable" && split && (selector?.kind === "members" && !part.length && (!part.quoted || selector.separator === "@") || part.transform && part.name === "@")) {
-                    const members = selector?.kind === "members" ? await this.arrayMembers(part.name, state) : this.positionalValues(state);
+                    const members = selector?.kind === "members" ? await this.arrayMembers(part.name, state, io) : this.positionalValues(state);
                     for (let position = 0; position < members.length; position++) {
                         if (position > 0)
                             addField();
@@ -6221,7 +6404,7 @@ export class Runtime {
                 allocation.hold(value);
         return createCommandArguments(values, allocation);
     }
-    async arrayMembers(name, state) {
+    async arrayMembers(name, state, io) {
         const store = requireArrays(state);
         const holding = store.owner.hold();
         try {
@@ -6241,8 +6424,10 @@ export class Runtime {
                 const values = [];
                 for (const index of indices) {
                     store.owner.reserve({ metadata: 32, allocatedSlots: 1, work: 4 });
-                    const value = binding.get(index);
-                    await textToken(store.owner, value, this.signal);
+                    const token = binding.values.get(index).text;
+                    const value = token.rawValue === undefined ? token.value : shellValueFromBytes(shellValueBytes(token.rawValue, io[valueScope]), io[valueScope]);
+                    if (typeof value === "string")
+                        await textToken(store.owner, value, this.signal);
                     values.push(value);
                     await store.owner.ledger.checkpoint(this.signal);
                 }