@poe-platform/safe-bash 0.1.793 → 0.1.794
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/safe-bash/chunks/{chunk-BENDQWMV.js → chunk-6IBTC4YL.js} +16 -2
- package/dist/safe-bash/chunks/{chunk-HEENNMEW.js → chunk-6P25QN5M.js} +16 -21
- package/dist/safe-bash/command-chunks/{chunk-JCORWQ2I.js → chunk-7FKUC34S.js} +1 -1
- package/dist/safe-bash/command-chunks/{chunk-ZJB7XJXD.js → chunk-XAFSZDY2.js} +1 -1
- package/dist/safe-bash/commands/python/index.browser.js +1 -1
- package/dist/safe-bash/commands/python/llm-capability.js +11 -2
- package/dist/safe-bash/commands/python/llm-module.js +6 -0
- package/dist/safe-bash/contracts/escaping.d.ts +2 -0
- package/dist/safe-bash/contracts/escaping.js +2 -0
- package/dist/safe-bash/full-core.browser.js +2 -2
- package/dist/safe-bash/opt-in/optional/commands/shuf/args.js +270 -168
- package/dist/safe-bash/opt-in/optional/commands/shuf/filesystem-output.js +19 -1
- package/dist/safe-bash/opt-in/optional/commands/shuf/input.js +121 -5
- package/dist/safe-bash/opt-in/optional/commands/shuf/random.d.ts +3 -0
- package/dist/safe-bash/opt-in/optional/commands/shuf/random.js +31 -23
- package/dist/safe-bash/opt-in/optional/commands/shuf/shuf.js +39 -37
- package/dist/safe-bash/shell/runtime.d.ts +1 -1
- package/dist/safe-bash/shell/runtime.js +10 -16
- package/dist/safe-bash/shell-entry.browser.js +1 -1
- package/dist/safe-bash-command-shuf/args.d.ts +6 -4
- package/dist/safe-bash-command-shuf/args.js +270 -168
- package/dist/safe-bash-command-shuf/filesystem-output.js +19 -1
- package/dist/safe-bash-command-shuf/input.d.ts +19 -1
- package/dist/safe-bash-command-shuf/input.js +121 -5
- package/dist/safe-bash-command-shuf/random.d.ts +3 -0
- package/dist/safe-bash-command-shuf/random.js +31 -23
- package/dist/safe-bash-command-shuf/shuf.js +39 -37
- package/dist/safe-bash-command-sqlite3/btree.d.ts +1 -1
- package/dist/safe-bash-command-sqlite3/engine.d.ts +1 -1
- package/dist/safe-bash-command-sqlite3/index.js +61 -63
- package/dist/safe-bash-io-engine/commands/search/requirements.js +2 -2
- package/dist/safe-bash-io-engine/commands/text-programs/shared.js +2 -2
- package/dist/safe-bash-io-engine/internal.js +1 -1
- package/package.json +3 -3
|
@@ -4,13 +4,116 @@ import { Diagnostic } from "./args.js";
|
|
|
4
4
|
export function virtualPath(cwd, name) {
|
|
5
5
|
if (name === "")
|
|
6
6
|
throw new FsError("ENOENT");
|
|
7
|
-
|
|
7
|
+
let decoded;
|
|
8
|
+
try {
|
|
9
|
+
decoded = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(Uint8Array.from(name, character => character.charCodeAt(0)));
|
|
10
|
+
}
|
|
11
|
+
catch {
|
|
12
|
+
throw new FsError("ENOENT");
|
|
13
|
+
}
|
|
14
|
+
return decoded.startsWith("/") ? decoded : `${cwd.endsWith("/") ? cwd : `${cwd}/`}${decoded}`;
|
|
15
|
+
}
|
|
16
|
+
/** Own only acquisition and reads; a capability or pathname query owns no reader. */
|
|
17
|
+
export class FileInput {
|
|
18
|
+
context;
|
|
19
|
+
maxBytes;
|
|
20
|
+
stat;
|
|
21
|
+
handle;
|
|
22
|
+
source;
|
|
23
|
+
opening;
|
|
24
|
+
closing;
|
|
25
|
+
closed = false;
|
|
26
|
+
controller = new AbortController();
|
|
27
|
+
signal;
|
|
28
|
+
constructor(context, maxBytes) {
|
|
29
|
+
this.context = context;
|
|
30
|
+
this.maxBytes = maxBytes;
|
|
31
|
+
this.signal = AbortSignal.any([context.signal, this.controller.signal]);
|
|
32
|
+
}
|
|
33
|
+
async open(name) {
|
|
34
|
+
const { fs } = this.context;
|
|
35
|
+
const signal = this.signal;
|
|
36
|
+
signal.throwIfAborted();
|
|
37
|
+
const path = virtualPath(this.context.cwd, name);
|
|
38
|
+
const capabilities = await fs.capabilitiesFor?.(path, { signal }) ?? fs.capabilities;
|
|
39
|
+
signal.throwIfAborted();
|
|
40
|
+
const retained = fs.openReadFile !== undefined && capabilities.retainedRead !== false;
|
|
41
|
+
if (!retained) {
|
|
42
|
+
if (capabilities.stat !== false)
|
|
43
|
+
this.stat = await fs.stat(path, { signal });
|
|
44
|
+
signal.throwIfAborted();
|
|
45
|
+
}
|
|
46
|
+
if (this.closed)
|
|
47
|
+
throw new FsError("EBADF");
|
|
48
|
+
this.opening = Promise.resolve().then(async () => {
|
|
49
|
+
signal.throwIfAborted();
|
|
50
|
+
let input;
|
|
51
|
+
if (retained) {
|
|
52
|
+
this.handle = await fs.openReadFile(path, { signal });
|
|
53
|
+
signal.throwIfAborted();
|
|
54
|
+
this.stat = await this.handle.stat({ signal });
|
|
55
|
+
signal.throwIfAborted();
|
|
56
|
+
const handle = this.handle;
|
|
57
|
+
const maximum = Math.min(65536, this.maxBytes);
|
|
58
|
+
input = { async *[Symbol.asyncIterator]() {
|
|
59
|
+
let position = 0;
|
|
60
|
+
while (true) {
|
|
61
|
+
const bytes = await handle.read(position, maximum, { signal });
|
|
62
|
+
signal.throwIfAborted();
|
|
63
|
+
if (!bytes.length)
|
|
64
|
+
return;
|
|
65
|
+
position += bytes.length;
|
|
66
|
+
yield new Uint8Array(bytes);
|
|
67
|
+
}
|
|
68
|
+
} };
|
|
69
|
+
}
|
|
70
|
+
else if (fs.readStream && capabilities.streamingRead !== false) {
|
|
71
|
+
input = fs.readStream(path, { signal, chunkSize: Math.min(65536, this.maxBytes) });
|
|
72
|
+
}
|
|
73
|
+
else {
|
|
74
|
+
if (capabilities.read === false)
|
|
75
|
+
throw new FsError("ENOTSUP");
|
|
76
|
+
const maxBytes = this.maxBytes;
|
|
77
|
+
input = { async *[Symbol.asyncIterator]() {
|
|
78
|
+
const bytes = await fs.readFile(path, { signal, ...(Number.isFinite(maxBytes) ? { maxBytes } : {}) });
|
|
79
|
+
if (bytes.byteLength > maxBytes)
|
|
80
|
+
throw new Diagnostic("shuf: maxInputBytes limit exceeded\n");
|
|
81
|
+
yield bytes;
|
|
82
|
+
} };
|
|
83
|
+
}
|
|
84
|
+
this.source = ownedBytes(input, signal);
|
|
85
|
+
signal.throwIfAborted();
|
|
86
|
+
});
|
|
87
|
+
await this.opening;
|
|
88
|
+
}
|
|
89
|
+
next() {
|
|
90
|
+
this.signal.throwIfAborted();
|
|
91
|
+
return this.source.next();
|
|
92
|
+
}
|
|
93
|
+
close() {
|
|
94
|
+
this.closed = true;
|
|
95
|
+
this.controller.abort(new Error("shuf input is closed"));
|
|
96
|
+
return this.closing ??= Promise.resolve().then(async () => {
|
|
97
|
+
await this.opening?.catch(() => { });
|
|
98
|
+
try {
|
|
99
|
+
await this.source?.return(undefined);
|
|
100
|
+
}
|
|
101
|
+
finally {
|
|
102
|
+
await this.handle?.close();
|
|
103
|
+
}
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
[Symbol.asyncIterator]() {
|
|
107
|
+
return { next: () => this.next(), return: async () => { await this.close(); return { done: true, value: undefined }; } };
|
|
108
|
+
}
|
|
8
109
|
}
|
|
9
110
|
export function ownedBytes(source, signal) {
|
|
10
111
|
const iterator = source[Symbol.asyncIterator]();
|
|
11
112
|
const controller = new AbortController();
|
|
12
113
|
const readingSignal = AbortSignal.any([signal, controller.signal]);
|
|
13
114
|
let finished = false;
|
|
115
|
+
let failed = false;
|
|
116
|
+
let failure;
|
|
14
117
|
let closing;
|
|
15
118
|
const close = () => {
|
|
16
119
|
controller.abort(new Error("shuf input is closed"));
|
|
@@ -24,9 +127,16 @@ export function ownedBytes(source, signal) {
|
|
|
24
127
|
[Symbol.asyncIterator]() {
|
|
25
128
|
return {
|
|
26
129
|
async next() {
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
130
|
+
try {
|
|
131
|
+
const next = await iterator.next();
|
|
132
|
+
finished = next.done === true;
|
|
133
|
+
return next;
|
|
134
|
+
}
|
|
135
|
+
catch (error) {
|
|
136
|
+
failed = true;
|
|
137
|
+
failure = error;
|
|
138
|
+
throw error;
|
|
139
|
+
}
|
|
30
140
|
},
|
|
31
141
|
return: close,
|
|
32
142
|
};
|
|
@@ -41,8 +151,14 @@ export function ownedBytes(source, signal) {
|
|
|
41
151
|
await yieldTurn(readingSignal);
|
|
42
152
|
}
|
|
43
153
|
}
|
|
154
|
+
catch (error) {
|
|
155
|
+
if (failed)
|
|
156
|
+
throw failure;
|
|
157
|
+
throw error;
|
|
158
|
+
}
|
|
44
159
|
finally {
|
|
45
|
-
await close()
|
|
160
|
+
await close().catch(error => { if (!failed)
|
|
161
|
+
throw error; });
|
|
46
162
|
}
|
|
47
163
|
})();
|
|
48
164
|
return {
|
|
@@ -8,6 +8,7 @@ export declare class RandomIntegers {
|
|
|
8
8
|
private bytes;
|
|
9
9
|
private offset;
|
|
10
10
|
private source;
|
|
11
|
+
private state;
|
|
11
12
|
private closed;
|
|
12
13
|
private closing;
|
|
13
14
|
private opening;
|
|
@@ -16,6 +17,8 @@ export declare class RandomIntegers {
|
|
|
16
17
|
constructor(context: CommandContext, name: string | undefined, maxBytes?: number);
|
|
17
18
|
open(): Promise<void>;
|
|
18
19
|
close(): Promise<void>;
|
|
20
|
+
seed(): void;
|
|
21
|
+
private refill;
|
|
19
22
|
private byte;
|
|
20
23
|
chooseSync(size: bigint): bigint;
|
|
21
24
|
choose(size: bigint): Promise<bigint>;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { yieldTurn } from "@poe-platform/safe-bash/contracts/yield";
|
|
2
|
-
import { Diagnostic, quote, wordMax } from "./args.js";
|
|
3
|
-
import {
|
|
2
|
+
import { Diagnostic, quote, unicodeLocale, wordMax } from "./args.js";
|
|
3
|
+
import { FileInput } from "./input.js";
|
|
4
4
|
export class RandomIntegers {
|
|
5
5
|
context;
|
|
6
6
|
name;
|
|
@@ -10,6 +10,7 @@ export class RandomIntegers {
|
|
|
10
10
|
bytes = new Uint8Array();
|
|
11
11
|
offset = 0;
|
|
12
12
|
source;
|
|
13
|
+
state;
|
|
13
14
|
closed = false;
|
|
14
15
|
closing;
|
|
15
16
|
opening;
|
|
@@ -26,38 +27,45 @@ export class RandomIntegers {
|
|
|
26
27
|
if (this.closed)
|
|
27
28
|
return Promise.reject(this.signal.reason);
|
|
28
29
|
return this.opening ??= Promise.resolve().then(async () => {
|
|
29
|
-
const { fs } = this.context;
|
|
30
30
|
const signal = this.signal;
|
|
31
31
|
signal.throwIfAborted();
|
|
32
32
|
if (this.name === undefined)
|
|
33
33
|
return;
|
|
34
|
-
|
|
35
|
-
await
|
|
36
|
-
signal.throwIfAborted();
|
|
37
|
-
if (fs.readStream)
|
|
38
|
-
this.source = ownedBytes(fs.readStream(path, { signal, chunkSize: Math.min(4096, this.maxBytes) }), signal);
|
|
39
|
-
else {
|
|
40
|
-
const maxBytes = this.maxBytes;
|
|
41
|
-
this.source = ownedBytes((async function* () {
|
|
42
|
-
const bytes = await fs.readFile(path, { signal, ...(Number.isFinite(maxBytes) ? { maxBytes } : {}) });
|
|
43
|
-
if (bytes.byteLength > maxBytes)
|
|
44
|
-
throw new Diagnostic("shuf: maxInputBytes limit exceeded\n");
|
|
45
|
-
yield bytes;
|
|
46
|
-
})(), signal);
|
|
47
|
-
}
|
|
34
|
+
this.source = new FileInput({ ...this.context, signal }, this.maxBytes);
|
|
35
|
+
await this.source.open(this.name);
|
|
48
36
|
signal.throwIfAborted();
|
|
49
37
|
});
|
|
50
38
|
}
|
|
51
|
-
|
|
39
|
+
close() {
|
|
52
40
|
this.closed = true;
|
|
53
41
|
this.controller.abort(new Error("shuf random source is closed"));
|
|
54
42
|
return this.closing ??= (async () => {
|
|
55
|
-
await this.
|
|
56
|
-
await this.source?.return(undefined);
|
|
43
|
+
await this.source?.close();
|
|
57
44
|
this.bytes.fill(0);
|
|
58
45
|
this.value = this.maximum = 0n;
|
|
59
46
|
})();
|
|
60
47
|
}
|
|
48
|
+
seed() {
|
|
49
|
+
if (!this.state) {
|
|
50
|
+
this.state = globalThis.crypto.getRandomValues(new Uint32Array(4));
|
|
51
|
+
if (this.state.every(value => value === 0))
|
|
52
|
+
this.state[0] = 1;
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
refill() {
|
|
56
|
+
this.seed();
|
|
57
|
+
const state = this.state;
|
|
58
|
+
const product = Math.imul(state[1], 5);
|
|
59
|
+
const result = Math.imul((product << 7) | (product >>> 25), 9) >>> 0;
|
|
60
|
+
const shifted = state[1] << 9;
|
|
61
|
+
state[2] = state[2] ^ state[0];
|
|
62
|
+
state[3] = state[3] ^ state[1];
|
|
63
|
+
state[1] = state[1] ^ state[2];
|
|
64
|
+
state[0] = state[0] ^ state[3];
|
|
65
|
+
state[2] = state[2] ^ shifted;
|
|
66
|
+
state[3] = (state[3] << 11) | (state[3] >>> 21);
|
|
67
|
+
this.bytes = Uint8Array.of(result & 255, (result >>> 8) & 255, (result >>> 16) & 255, result >>> 24);
|
|
68
|
+
}
|
|
61
69
|
async byte() {
|
|
62
70
|
this.signal.throwIfAborted();
|
|
63
71
|
if (this.closed)
|
|
@@ -67,13 +75,13 @@ export class RandomIntegers {
|
|
|
67
75
|
const next = await this.source.next();
|
|
68
76
|
this.signal.throwIfAborted();
|
|
69
77
|
if (next.done)
|
|
70
|
-
throw new Diagnostic(`shuf: ${quote(this.name)}: end of file\n`);
|
|
78
|
+
throw new Diagnostic(`shuf: ${quote(this.name, unicodeLocale(this.context))}: end of file\n`);
|
|
71
79
|
if (next.value.byteLength > this.maxBytes)
|
|
72
80
|
throw new Diagnostic("shuf: maxInputBytes limit exceeded\n");
|
|
73
81
|
this.bytes = new Uint8Array(next.value);
|
|
74
82
|
}
|
|
75
83
|
else {
|
|
76
|
-
this.
|
|
84
|
+
this.refill();
|
|
77
85
|
}
|
|
78
86
|
this.offset = 0;
|
|
79
87
|
this.signal.throwIfAborted();
|
|
@@ -90,7 +98,7 @@ export class RandomIntegers {
|
|
|
90
98
|
throw new Error("shuf random source is closed");
|
|
91
99
|
while (this.maximum < target) {
|
|
92
100
|
if (this.offset === this.bytes.length) {
|
|
93
|
-
this.
|
|
101
|
+
this.refill();
|
|
94
102
|
this.offset = 0;
|
|
95
103
|
}
|
|
96
104
|
this.value = ((this.value << 8n) + BigInt(this.bytes[this.offset++])) & wordMax;
|
|
@@ -1,12 +1,14 @@
|
|
|
1
|
-
import { builtInDirectContextExecutors, commandRuntimeIdentity, createBufferedOutput, getCommandArguments, isFsError,
|
|
1
|
+
import { builtInDirectContextExecutors, commandRuntimeIdentity, createBufferedOutput, getCommandArguments, isFsError, writeBytes, } from "@poe-platform/safe-bash/contracts";
|
|
2
2
|
import { openFileOutput } from "./filesystem-output.js";
|
|
3
3
|
import { hasYieldCheckpoint, inheritYieldCheckpoint, monotonicNow, runYieldCheckpoint, yieldTurn } from "@poe-platform/safe-bash/contracts/yield";
|
|
4
|
-
import { countMax, Diagnostic, fileQuote, parse, quote } from "./args.js";
|
|
5
|
-
import { ownedBytes, readAllRecords, records, virtualPath } from "./input.js";
|
|
4
|
+
import { countMax, Diagnostic, fileQuote, parse, quote, unicodeLocale } from "./args.js";
|
|
5
|
+
import { FileInput, ownedBytes, readAllRecords, records, virtualPath } from "./input.js";
|
|
6
6
|
import { settings } from "./options.js";
|
|
7
7
|
import { RandomIntegers } from "./random.js";
|
|
8
8
|
import { shellValueByteLength } from "@poe-platform/safe-bash/contracts/value";
|
|
9
9
|
import { textOutputRequirements } from "@poe-platform/safe-bash/optional-host";
|
|
10
|
+
import { publicDiagnosticMessage } from "@poe-platform/safe-bash/contracts/diagnostics";
|
|
11
|
+
import { writeDiagnostic } from "@poe-platform/safe-bash/contracts/escaping";
|
|
10
12
|
const encoder = new TextEncoder();
|
|
11
13
|
const errors = {
|
|
12
14
|
ENOENT: "No such file or directory", EACCES: "Permission denied", EPERM: "Operation not permitted",
|
|
@@ -29,7 +31,7 @@ export function createShufCommand(options = {}) {
|
|
|
29
31
|
let output;
|
|
30
32
|
let source;
|
|
31
33
|
let inputSource;
|
|
32
|
-
let
|
|
34
|
+
let fileInput;
|
|
33
35
|
const inputController = new AbortController();
|
|
34
36
|
const inputSignal = AbortSignal.any([context.signal, inputController.signal]);
|
|
35
37
|
inheritYieldCheckpoint(context.signal, inputSignal);
|
|
@@ -42,8 +44,7 @@ export function createShufCommand(options = {}) {
|
|
|
42
44
|
closed = true;
|
|
43
45
|
inputController.abort(new Error("shuf input is closed"));
|
|
44
46
|
return cleanup ??= (async () => {
|
|
45
|
-
await
|
|
46
|
-
const results = await Promise.allSettled([source?.return(undefined), inputSource?.return(undefined), random?.close()]);
|
|
47
|
+
const results = await Promise.allSettled([source?.return(undefined), inputSource?.return(undefined), fileInput?.close(), random?.close()]);
|
|
47
48
|
const failed = results.find(result => result.status === "rejected");
|
|
48
49
|
if (failed?.status === "rejected")
|
|
49
50
|
throw failed.reason;
|
|
@@ -52,6 +53,7 @@ export function createShufCommand(options = {}) {
|
|
|
52
53
|
context.registerCleanup?.(close);
|
|
53
54
|
try {
|
|
54
55
|
context.signal.throwIfAborted();
|
|
56
|
+
const unicode = unicodeLocale(context);
|
|
55
57
|
const parsed = parse(context);
|
|
56
58
|
if (parsed.action) {
|
|
57
59
|
const { help, version } = await import("./usage.js");
|
|
@@ -78,38 +80,28 @@ export function createShufCommand(options = {}) {
|
|
|
78
80
|
}
|
|
79
81
|
size = BigInt(lines.length);
|
|
80
82
|
}
|
|
81
|
-
else if (parsed.count !== 0n
|
|
82
|
-
const name = parsed.operands.length ? context.
|
|
83
|
+
else if (!parsed.range && !parsed.echo && (parsed.count !== 0n || parsed.repeat)) {
|
|
84
|
+
const name = parsed.count === 0n || !parsed.operands.length ? "-" : Array.from(getCommandArguments(context).bytes(parsed.operands[0]), byte => String.fromCharCode(byte)).join("");
|
|
83
85
|
let inputSize = Infinity;
|
|
84
|
-
|
|
86
|
+
{
|
|
85
87
|
inputSignal.throwIfAborted();
|
|
86
88
|
let input;
|
|
87
89
|
if (name === "-")
|
|
88
90
|
input = context.stdin;
|
|
89
91
|
else {
|
|
90
|
-
diagnostic = fileQuote(name);
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
92
|
+
diagnostic = fileQuote(name, unicode);
|
|
93
|
+
fileInput = new FileInput({ ...context, signal: inputSignal }, limits.maxInputBytes);
|
|
94
|
+
await fileInput.open(name);
|
|
95
|
+
const stat = fileInput.stat;
|
|
96
|
+
if (stat?.type === "file" && Number.isSafeInteger(stat.size) && stat.size >= 0)
|
|
94
97
|
inputSize = stat.size;
|
|
95
|
-
|
|
96
|
-
await context.fs.access(path, 4, { signal: inputSignal });
|
|
97
|
-
inputSignal.throwIfAborted();
|
|
98
|
-
if (context.fs.readStream)
|
|
99
|
-
input = context.fs.readStream(path, { signal: inputSignal });
|
|
100
|
-
else {
|
|
101
|
-
const bytes = await context.fs.readFile(path, { signal: inputSignal, ...(Number.isFinite(limits.maxInputBytes) ? { maxBytes: limits.maxInputBytes } : {}) });
|
|
102
|
-
if (bytes.byteLength > limits.maxInputBytes)
|
|
103
|
-
throw new Diagnostic("shuf: maxInputBytes limit exceeded\n");
|
|
104
|
-
input = toByteSource(bytes);
|
|
105
|
-
}
|
|
98
|
+
input = fileInput;
|
|
106
99
|
}
|
|
107
100
|
inputSource = ownedBytes(input, inputSignal);
|
|
108
101
|
inputSignal.throwIfAborted();
|
|
109
102
|
if (closed)
|
|
110
103
|
throw new Error("shuf input is closed");
|
|
111
|
-
}
|
|
112
|
-
await openingInput;
|
|
104
|
+
}
|
|
113
105
|
diagnostic = "read error";
|
|
114
106
|
reservoir = !parsed.repeat && parsed.count < countMax && inputSize > 8 * 1024 * 1024;
|
|
115
107
|
if (!reservoir) {
|
|
@@ -122,10 +114,12 @@ export function createShufCommand(options = {}) {
|
|
|
122
114
|
}
|
|
123
115
|
random = new RandomIntegers(context, parsed.random, limits.maxInputBytes);
|
|
124
116
|
const count = parsed.repeat || parsed.count < size ? parsed.count : size;
|
|
125
|
-
diagnostic = fileQuote(parsed.random ?? "getrandom");
|
|
126
|
-
if (reservoir || parsed.repeat || count > 0n)
|
|
117
|
+
diagnostic = fileQuote(parsed.random ?? "getrandom", unicode);
|
|
118
|
+
if (reservoir && parsed.count > 0n || parsed.repeat || count > 0n)
|
|
127
119
|
await random.open();
|
|
128
|
-
|
|
120
|
+
if (parsed.repeat && parsed.random === undefined)
|
|
121
|
+
random.seed();
|
|
122
|
+
diagnostic = parsed.random === undefined ? "getrandom" : `${quote(parsed.random, unicode)}: read error`;
|
|
129
123
|
if (reservoir) {
|
|
130
124
|
let seen = 0n;
|
|
131
125
|
while (seen < parsed.count) {
|
|
@@ -143,7 +137,7 @@ export function createShufCommand(options = {}) {
|
|
|
143
137
|
}
|
|
144
138
|
if (seen === parsed.count) {
|
|
145
139
|
while (true) {
|
|
146
|
-
diagnostic = parsed.random === undefined ? "getrandom" : `${quote(parsed.random)}: read error`;
|
|
140
|
+
diagnostic = parsed.random === undefined ? "getrandom" : `${quote(parsed.random, unicode)}: read error`;
|
|
147
141
|
const chosen = await random.choose(seen + 1n);
|
|
148
142
|
diagnostic = "read error";
|
|
149
143
|
const next = await source.next();
|
|
@@ -181,12 +175,13 @@ export function createShufCommand(options = {}) {
|
|
|
181
175
|
const streamDirect = !parsed.repeat && parsed.output === undefined && parsed.random === undefined;
|
|
182
176
|
const permutation = [];
|
|
183
177
|
const swaps = new Map();
|
|
184
|
-
|
|
178
|
+
const sparse = ahead > 0n && (size > 0xffffffffn || size >= 128n * 1024n && size / ahead >= 32n);
|
|
179
|
+
diagnostic = parsed.random === undefined ? "getrandom" : `${quote(parsed.random, unicode)}: read error`;
|
|
185
180
|
const useSyncRandom = parsed.random === undefined;
|
|
186
181
|
if (!parsed.repeat && !streamDirect) {
|
|
187
182
|
for (let index = 0n; index < ahead; index++) {
|
|
188
183
|
const chosen = index + (useSyncRandom ? random.chooseSync(size - index) : await random.choose(size - index));
|
|
189
|
-
permutation.push(swaps.get(chosen) ?? chosen);
|
|
184
|
+
permutation.push(sparse && chosen === index ? index : swaps.get(chosen) ?? chosen);
|
|
190
185
|
swaps.set(chosen, swaps.get(index) ?? index);
|
|
191
186
|
swaps.delete(index);
|
|
192
187
|
if (index % 1024n === 1023n) {
|
|
@@ -202,7 +197,7 @@ export function createShufCommand(options = {}) {
|
|
|
202
197
|
}
|
|
203
198
|
}
|
|
204
199
|
if (parsed.output !== undefined) {
|
|
205
|
-
diagnostic = fileQuote(parsed.output);
|
|
200
|
+
diagnostic = fileQuote(parsed.output, unicode);
|
|
206
201
|
openingRootOutput = parsed.output.length > 0 && Array.from(parsed.output).every(character => character === "/");
|
|
207
202
|
output = await openFileOutput(context, virtualPath(context.cwd, parsed.output), "w");
|
|
208
203
|
openingRootOutput = false;
|
|
@@ -212,14 +207,14 @@ export function createShufCommand(options = {}) {
|
|
|
212
207
|
const sink = createBufferedOutput(output?.sink ?? context.stdout, context.signal);
|
|
213
208
|
for (let index = 0n; index < ahead; index++) {
|
|
214
209
|
context.signal.throwIfAborted();
|
|
215
|
-
diagnostic = parsed.random === undefined ? "getrandom" : `${quote(parsed.random)}: read error`;
|
|
210
|
+
diagnostic = parsed.random === undefined ? "getrandom" : `${quote(parsed.random, unicode)}: read error`;
|
|
216
211
|
let chosen;
|
|
217
212
|
if (parsed.repeat) {
|
|
218
213
|
chosen = useSyncRandom ? random.chooseSync(size) : await random.choose(size);
|
|
219
214
|
}
|
|
220
215
|
else if (streamDirect) {
|
|
221
216
|
const pick = index + (useSyncRandom ? random.chooseSync(size - index) : await random.choose(size - index));
|
|
222
|
-
chosen = swaps.get(pick) ?? pick;
|
|
217
|
+
chosen = sparse && pick === index ? index : swaps.get(pick) ?? pick;
|
|
223
218
|
swaps.set(pick, swaps.get(index) ?? index);
|
|
224
219
|
swaps.delete(index);
|
|
225
220
|
}
|
|
@@ -229,6 +224,8 @@ export function createShufCommand(options = {}) {
|
|
|
229
224
|
const line = parsed.range ? encoder.encode(`${parsed.range.low + chosen}${parsed.delimiter === 0 ? "\0" : "\n"}`) : lines[Number(chosen)];
|
|
230
225
|
diagnostic = "write error";
|
|
231
226
|
await sink.write(line);
|
|
227
|
+
if (parsed.repeat && parsed.random !== undefined)
|
|
228
|
+
await sink.flush();
|
|
232
229
|
if (index % 256n === 255n) {
|
|
233
230
|
if (++yieldChecks % 16 === 0 || hasYieldCheckpoint(context.signal) || monotonicNow() - lastYield >= 25) {
|
|
234
231
|
await yieldTurn(context.signal);
|
|
@@ -251,10 +248,15 @@ export function createShufCommand(options = {}) {
|
|
|
251
248
|
if (output)
|
|
252
249
|
await output.abort(error);
|
|
253
250
|
context.signal.throwIfAborted();
|
|
251
|
+
inputSignal.throwIfAborted();
|
|
254
252
|
if (isFsError(error) && error.code === "EPIPE" && diagnostic === "write error")
|
|
255
253
|
return { exitCode: 141 };
|
|
256
|
-
if (!(error instanceof Diagnostic) && !isFsError(error))
|
|
257
|
-
|
|
254
|
+
if (!(error instanceof Diagnostic) && !isFsError(error)) {
|
|
255
|
+
if ((diagnostic === "write error" || diagnostic === "getrandom") && error instanceof Error)
|
|
256
|
+
throw error;
|
|
257
|
+
await writeDiagnostic(context.stderr, `${context.command}: ${publicDiagnosticMessage(error, context.onInternalError)}\n`, context.signal);
|
|
258
|
+
return { exitCode: 1 };
|
|
259
|
+
}
|
|
258
260
|
const message = error instanceof Diagnostic ? error.message : `shuf: ${diagnostic}: ${openingRootOutput && error.code === "EISDIR" ? "File exists" : errors[error.code] ?? error.code}\n`;
|
|
259
261
|
await writeBytes(context.stderr, error instanceof Diagnostic && error.bytes ? error.bytes : encoder.encode(message), context.signal);
|
|
260
262
|
return { exitCode: 1 };
|
|
@@ -672,7 +672,7 @@ export declare class Runtime {
|
|
|
672
672
|
private ifsSeparator;
|
|
673
673
|
substring(part: Extract<WordPart, {
|
|
674
674
|
kind: "variable";
|
|
675
|
-
}>, value:
|
|
675
|
+
}>, value: ShellValue | undefined, state: State, io: IO): Promise<ShellValue>;
|
|
676
676
|
private tryFastParameterPatternSync;
|
|
677
677
|
parameterPattern(part: Extract<WordPart, {
|
|
678
678
|
kind: "variable";
|
|
@@ -25377,7 +25377,7 @@ export class Runtime {
|
|
|
25377
25377
|
if (part.length)
|
|
25378
25378
|
return this.valueLength(value ?? "", state, io);
|
|
25379
25379
|
if (part.substring)
|
|
25380
|
-
return this.substring(part, value
|
|
25380
|
+
return this.substring(part, value, state, io);
|
|
25381
25381
|
if (part.transform)
|
|
25382
25382
|
return part.transform === "a" || part.transform === "A" || part.transform === "K" || part.transform === "k" ? this.variableMetaTransform(part, state, io) : value === undefined ? "" : this.transformValue(value, part.transform, state, io);
|
|
25383
25383
|
if (part.operator)
|
|
@@ -25870,7 +25870,7 @@ export class Runtime {
|
|
|
25870
25870
|
}
|
|
25871
25871
|
if (part.substring) {
|
|
25872
25872
|
this.requireParameter(value, part.name, state, io, part.line);
|
|
25873
|
-
return this.substring(part,
|
|
25873
|
+
return this.substring(part, retained, state, io);
|
|
25874
25874
|
}
|
|
25875
25875
|
if (part.operator) {
|
|
25876
25876
|
if (["#", "##", "%", "%%", "^", "^^", ",", ",,"].includes(part.operator) || part.operator.startsWith("/")) {
|
|
@@ -25999,10 +25999,10 @@ export class Runtime {
|
|
|
25999
25999
|
const offsetExpression = await arithmetic(expression.offset);
|
|
26000
26000
|
let bytes;
|
|
26001
26001
|
if (byteLocale(state.variables)) {
|
|
26002
|
-
|
|
26003
|
-
bytes = fastSharedTextEncoder.encode(value);
|
|
26002
|
+
bytes = shellValueBytes(value, scratch);
|
|
26004
26003
|
}
|
|
26005
|
-
const
|
|
26004
|
+
const text = shellValueText(value);
|
|
26005
|
+
const size = BigInt(bytes?.byteLength ?? (await scanString(text, work)).count);
|
|
26006
26006
|
const offset = offsetExpression.value < 0n ? size + offsetExpression.value : offsetExpression.value;
|
|
26007
26007
|
if (offset < 0n || offset > size)
|
|
26008
26008
|
return "";
|
|
@@ -26017,19 +26017,13 @@ export class Runtime {
|
|
|
26017
26017
|
}
|
|
26018
26018
|
this.signal.throwIfAborted();
|
|
26019
26019
|
if (!bytes) {
|
|
26020
|
-
const start = (await scanString(
|
|
26021
|
-
const finish = (await scanString(
|
|
26020
|
+
const start = (await scanString(text, work, 0, text.length, Number(offset))).end;
|
|
26021
|
+
const finish = (await scanString(text, work, start, text.length, Number(end - offset))).end;
|
|
26022
26022
|
if (finish > start)
|
|
26023
26023
|
scratch.reserve((finish - start) * 2, 0);
|
|
26024
|
-
return
|
|
26025
|
-
}
|
|
26026
|
-
scratch.reserve(Number(end - offset) * 2, 0);
|
|
26027
|
-
try {
|
|
26028
|
-
return new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(bytes.subarray(Number(offset), Number(end)));
|
|
26029
|
-
}
|
|
26030
|
-
catch {
|
|
26031
|
-
throw new ExpansionFailure("substring expansion splits a UTF-8 character in a byte locale", line);
|
|
26024
|
+
return text.slice(start, finish);
|
|
26032
26025
|
}
|
|
26026
|
+
return shellValueFromBytes(bytes.subarray(Number(offset), Number(end)), io[valueScope]);
|
|
26033
26027
|
}
|
|
26034
26028
|
finally {
|
|
26035
26029
|
scratch.close();
|
|
@@ -26148,7 +26142,7 @@ export class Runtime {
|
|
|
26148
26142
|
const replacementIO = part.replacement ? this.parameterOperandIO(part.replacement, state, io) : io;
|
|
26149
26143
|
for (const [index, entry] of (part.replacement?.parts ?? []).entries()) {
|
|
26150
26144
|
let value = entry.kind === "text" ? entry.value : await this.part(entry, state, replacementIO, hereString);
|
|
26151
|
-
if (index === 0 && entry.kind === "text" && !
|
|
26145
|
+
if (index === 0 && entry.kind === "text" && !entry.quoted && /^~(?:\/|$)/u.test(value)) {
|
|
26152
26146
|
const home = state.variables.HOME ?? "~";
|
|
26153
26147
|
scratch.reserve((home.length + value.length - 1) * 2, 0);
|
|
26154
26148
|
value = home + value.slice(1);
|
|
@@ -5,19 +5,21 @@ export declare class Diagnostic extends Error {
|
|
|
5
5
|
readonly bytes?: Uint8Array | undefined;
|
|
6
6
|
constructor(message: string, bytes?: Uint8Array | undefined);
|
|
7
7
|
}
|
|
8
|
-
export declare function
|
|
9
|
-
export declare function
|
|
8
|
+
export declare function unicodeLocale(context: CommandContext): boolean;
|
|
9
|
+
export declare function quote(value: string | Uint8Array, unicode?: boolean): string;
|
|
10
|
+
export declare function fileQuote(value: string, unicode?: boolean): string;
|
|
10
11
|
export declare function parse(context: CommandContext): {
|
|
11
12
|
echo: boolean;
|
|
12
13
|
repeat: boolean;
|
|
13
14
|
delimiter: number;
|
|
14
15
|
count: bigint;
|
|
16
|
+
hasCount: boolean;
|
|
17
|
+
operands: number[];
|
|
15
18
|
range: {
|
|
16
19
|
low: bigint;
|
|
17
20
|
size: bigint;
|
|
18
21
|
} | undefined;
|
|
19
22
|
output: string | undefined;
|
|
20
23
|
random: string | undefined;
|
|
21
|
-
action:
|
|
22
|
-
operands: number[];
|
|
24
|
+
action: string | undefined;
|
|
23
25
|
};
|