@poe-platform/safe-bash 0.1.797 → 0.1.798
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/pdf/index.d.ts +1 -1
- package/dist/pdf/model.d.ts +7 -2
- package/dist/safe-bash/chunks/{chunk-TGOF7VO5.js → chunk-AMTCC256.js} +1 -1
- package/dist/safe-bash/chunks/{chunk-TCWZNUVP.js → chunk-CSNXID73.js} +1 -1
- package/dist/safe-bash/chunks/{chunk-C2NYCU7O.js → chunk-JSVX4VIA.js} +5 -1
- package/dist/safe-bash/chunks/{chunk-ZXKVOJNL.js → chunk-V3VEATEA.js} +6 -5
- package/dist/safe-bash/chunks/{chunk-D2XB7XIC.js → chunk-XH26YVGW.js} +6 -0
- package/dist/safe-bash/command-chunks/{chunk-A7RIYQIF.js → chunk-DDLQ5YDB.js} +172 -167
- package/dist/safe-bash/command-chunks/{chunk-C2DUYVYZ.js → chunk-ESMCX3YV.js} +138 -36
- package/dist/safe-bash/command-chunks/chunk-EZ7D6DXC.js +6 -0
- package/dist/safe-bash/command-chunks/{chunk-ECAK3JHR.js → chunk-QRRYN2KO.js} +21 -12
- package/dist/safe-bash/command-chunks/{chunk-UUR2S7AD.js → chunk-Y6TUZWPS.js} +28 -1
- package/dist/safe-bash/command-chunks/{pptx-KY2OJWZW-76LBHCHD.js → pptx-GBPNWNEP-7EQEVXV5.js} +8 -2
- package/dist/safe-bash/commands/llm/index.browser.js +9 -1
- package/dist/safe-bash/commands/llm/providers/index.browser.js +6 -2
- package/dist/safe-bash/core.browser.js +1 -1
- package/dist/safe-bash/core.d.ts +5 -1
- package/dist/safe-bash/core.js +5 -1
- package/dist/safe-bash/full-core.browser.js +134 -5
- package/dist/safe-bash/lazy-optional.d.ts +13 -0
- package/dist/safe-bash/lazy-optional.js +118 -0
- package/dist/safe-bash/plugins/index.browser.js +3 -3
- package/dist/safe-bash/registry-entry.browser.js +2 -2
- package/dist/safe-bash/shell/env-warning.js +3 -2
- package/dist/safe-bash/shell/runtime.js +2 -2
- package/dist/safe-bash/shell-entry.browser.js +2 -2
- package/dist/safe-bash-command-csvkit/index.js +528 -498
- package/dist/safe-bash-command-git/git_rust.wasm +0 -0
- package/dist/safe-bash-command-git/wasm.generated.js +1 -1
- package/dist/safe-bash-command-llm/base64-stream.d.ts +2 -0
- package/dist/safe-bash-command-llm/chat-json.d.ts +10 -0
- package/dist/safe-bash-command-llm/index.d.ts +2 -1
- package/dist/safe-bash-command-llm/index.js +11 -2
- package/dist/safe-bash-command-llm/json-string.d.ts +2 -0
- package/dist/safe-bash-command-llm/json-value.d.ts +2 -0
- package/dist/safe-bash-command-llm/openai.d.ts +1 -0
- package/dist/safe-bash-command-llm/openai.js +4 -2
- package/dist/safe-bash-command-llm/provider-serialization.d.ts +6 -0
- package/dist/safe-bash-command-llm/providers/index.d.ts +2 -1
- package/dist/safe-bash-command-llm/providers/index.js +12 -3
- package/dist/safe-bash-command-llm/providers/openai.d.ts +1 -1
- package/dist/safe-bash-command-llm/providers/openai.js +4 -2
- package/dist/safe-bash-command-pandoc/citeproc-filters.js +1 -1
- package/dist/safe-bash-command-pandoc/command.js +3 -3
- package/dist/safe-bash-command-pandoc/index.js +3 -3
- package/dist/safe-bash-command-ssconvert/index.js +1 -1
- package/package.json +5 -4
|
@@ -615,12 +615,12 @@ function normalizedHeader(value) {
|
|
|
615
615
|
if (word) words.push(word);
|
|
616
616
|
return words.join(" ");
|
|
617
617
|
}
|
|
618
|
-
async function clean(
|
|
619
|
-
const o =
|
|
620
|
-
await
|
|
618
|
+
async function clean(runtime2) {
|
|
619
|
+
const o = runtime2.options;
|
|
620
|
+
await runtime2.prompt();
|
|
621
621
|
if (!o.length_mismatch && !o.empty_columns && !o.enable_all_checks && !o.header_normalize_space && !o.join_short_rows && !o.fill_short_rows)
|
|
622
|
-
|
|
623
|
-
if (o.join_short_rows && o.fill_short_rows)
|
|
622
|
+
runtime2.error("No checks or fixes were enabled. See available options with: csvclean --help");
|
|
623
|
+
if (o.join_short_rows && o.fill_short_rows) runtime2.error("The --join-short-rows and --fill-short-rows options are mutually exclusive.");
|
|
624
624
|
const checkLength = Boolean(o.length_mismatch || o.enable_all_checks);
|
|
625
625
|
const checkEmpty = Boolean(o.empty_columns || o.enable_all_checks);
|
|
626
626
|
const errors = [];
|
|
@@ -628,11 +628,11 @@ async function clean(runtime) {
|
|
|
628
628
|
let headers;
|
|
629
629
|
let empties = [];
|
|
630
630
|
let count = 0;
|
|
631
|
-
for await (const record of
|
|
631
|
+
for await (const record of runtime2.records()) {
|
|
632
632
|
if (headers === void 0) {
|
|
633
633
|
headers = record.cells.map((cell2) => o.header_normalize_space ? normalizedHeader(cell2) : cell2);
|
|
634
634
|
empties = headers.map(() => 0);
|
|
635
|
-
await
|
|
635
|
+
await runtime2.row(headers);
|
|
636
636
|
continue;
|
|
637
637
|
}
|
|
638
638
|
count++;
|
|
@@ -640,20 +640,20 @@ async function clean(runtime) {
|
|
|
640
640
|
const error = { line: record.line - 1, row, message: `Expected ${headers.length} columns, found ${row.length} columns` };
|
|
641
641
|
let retained = false;
|
|
642
642
|
if (o.fill_short_rows) while (row.length < headers.length) {
|
|
643
|
-
|
|
643
|
+
runtime2.step();
|
|
644
644
|
row.push(o.fillvalue);
|
|
645
645
|
}
|
|
646
646
|
else if (o.join_short_rows) {
|
|
647
647
|
if (row.length >= headers.length) joinable = [];
|
|
648
648
|
else {
|
|
649
|
-
|
|
649
|
+
runtime2.retain(64 + row.reduce((size, cell2) => size + (cell2?.length ?? 0) * 2 + 16, 0));
|
|
650
650
|
retained = true;
|
|
651
651
|
joinable.push(error);
|
|
652
652
|
if (joinable.length > 1) {
|
|
653
653
|
while (joinable.length) {
|
|
654
654
|
const merged = [...joinable[0].row];
|
|
655
655
|
for (const next of joinable.slice(1)) {
|
|
656
|
-
|
|
656
|
+
runtime2.step();
|
|
657
657
|
if (merged.length === 0) throw new CsvkitDiagnostic("IndexError: list index out of range");
|
|
658
658
|
merged[merged.length - 1] = String(merged.at(-1) ?? "") + String(o.separator) + (next.row[0] ?? "");
|
|
659
659
|
merged.push(...next.row.slice(1));
|
|
@@ -675,18 +675,18 @@ async function clean(runtime) {
|
|
|
675
675
|
}
|
|
676
676
|
}
|
|
677
677
|
if (checkLength && row.length !== headers.length) {
|
|
678
|
-
if (!retained)
|
|
678
|
+
if (!retained) runtime2.retain(64 + error.row.reduce((size, cell2) => size + (cell2?.length ?? 0) * 2 + 16, 0));
|
|
679
679
|
errors.push(error);
|
|
680
680
|
}
|
|
681
681
|
if (checkEmpty) for (let index = 0; index < headers.length; index++) {
|
|
682
|
-
|
|
682
|
+
runtime2.step();
|
|
683
683
|
if (index >= row.length || row[index] === "") empties[index]++;
|
|
684
684
|
}
|
|
685
|
-
if (!o.omit_error_rows || row.length === headers.length) await
|
|
685
|
+
if (!o.omit_error_rows || row.length === headers.length) await runtime2.row(row);
|
|
686
686
|
}
|
|
687
687
|
if (headers === void 0) {
|
|
688
688
|
headers = [];
|
|
689
|
-
await
|
|
689
|
+
await runtime2.row([]);
|
|
690
690
|
}
|
|
691
691
|
const emptyColumns = count ? empties.flatMap((value, index) => value === count ? [index] : []) : [];
|
|
692
692
|
if (emptyColumns.length) errors.push({ line: 1, row: headers.map(() => ""), message: `Empty columns named ${emptyColumns.map((index) => repr(headers[index])).join(", ")}! Try: csvcut -C ${emptyColumns.map((index) => index + (o.zero_based ? 0 : 1)).join(",")}` });
|
|
@@ -694,10 +694,10 @@ async function clean(runtime) {
|
|
|
694
694
|
const label = o.label === "-" ? !o.input_path || o.input_path === "-" ? "stdin" : String(o.input_path) : String(o.label);
|
|
695
695
|
const errorHeaders = [...o.label ? ["label"] : [], "line_number", "msg", ...headers];
|
|
696
696
|
const numbered = Boolean(o.line_numbers);
|
|
697
|
-
await
|
|
697
|
+
await runtime2.write(writeCsvRow(numbered ? ["line_number", ...errorHeaders] : errorHeaders), "stderr");
|
|
698
698
|
for (const [index, error] of errors.entries()) {
|
|
699
699
|
const cells = [...o.label ? [label] : [], error.line, error.message, ...error.row];
|
|
700
|
-
await
|
|
700
|
+
await runtime2.write(writeCsvRow(numbered ? [index + 1, ...cells] : cells), "stderr");
|
|
701
701
|
}
|
|
702
702
|
return 1;
|
|
703
703
|
}
|
|
@@ -707,14 +707,14 @@ init_buffer();
|
|
|
707
707
|
|
|
708
708
|
// packages/safe-bash-command-csvkit/src/columns.ts
|
|
709
709
|
init_buffer();
|
|
710
|
-
async function printNames(
|
|
711
|
-
if (
|
|
712
|
-
const rows =
|
|
710
|
+
async function printNames(runtime2) {
|
|
711
|
+
if (runtime2.options.no_header_row) throw new CsvkitDiagnostic("RequiredHeaderError: You cannot use --no-header-row with the -n or --names options.");
|
|
712
|
+
const rows = runtime2.records();
|
|
713
713
|
try {
|
|
714
714
|
const first = await rows.next();
|
|
715
715
|
if (first.done) throw new CsvkitDiagnostic("StopIteration: ");
|
|
716
|
-
const offset =
|
|
717
|
-
for (const [index, name] of first.value.cells.entries()) await
|
|
716
|
+
const offset = runtime2.options.zero_based ? 0 : 1;
|
|
717
|
+
for (const [index, name] of first.value.cells.entries()) await runtime2.write(`${String(index + offset).padStart(3)}: ${name}
|
|
718
718
|
`);
|
|
719
719
|
return 0;
|
|
720
720
|
} finally {
|
|
@@ -723,31 +723,31 @@ async function printNames(runtime) {
|
|
|
723
723
|
}
|
|
724
724
|
|
|
725
725
|
// packages/safe-bash-command-csvkit/src/commands/csvcut.ts
|
|
726
|
-
async function cut(
|
|
727
|
-
if (
|
|
728
|
-
await
|
|
726
|
+
async function cut(runtime2) {
|
|
727
|
+
if (runtime2.options.names_only) return printNames(runtime2);
|
|
728
|
+
await runtime2.prompt();
|
|
729
729
|
let headers;
|
|
730
730
|
let columns = [];
|
|
731
|
-
for await (const record of
|
|
731
|
+
for await (const record of runtime2.records(void 0, void 0, void 0, void 0, void 0, true)) {
|
|
732
732
|
if (headers === void 0) {
|
|
733
|
-
const generated = Boolean(
|
|
733
|
+
const generated = Boolean(runtime2.options.no_header_row);
|
|
734
734
|
headers = generated ? defaultHeaders(record.cells.length) : record.cells.map((cell2) => pythonValueText(inputWriteCell(cell2)));
|
|
735
735
|
columns = parseColumnIdentifiers(
|
|
736
|
-
|
|
736
|
+
runtime2.options.columns,
|
|
737
737
|
headers,
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
738
|
+
runtime2.options.zero_based ? 0 : 1,
|
|
739
|
+
runtime2.options.not_columns,
|
|
740
|
+
runtime2.step,
|
|
741
741
|
generated
|
|
742
742
|
);
|
|
743
|
-
await
|
|
743
|
+
await runtime2.row(columns.map((index) => headers[index]));
|
|
744
744
|
if (!generated) continue;
|
|
745
745
|
}
|
|
746
746
|
const selected = columns.map((index) => record.cells[index] ?? null);
|
|
747
|
-
if (!
|
|
748
|
-
await
|
|
747
|
+
if (!runtime2.options.delete_empty || selected.some((value) => typeof value === "number" ? value !== 0 : Boolean(value)))
|
|
748
|
+
await runtime2.row(selected.map(inputWriteCell));
|
|
749
749
|
}
|
|
750
|
-
if (headers === void 0) await
|
|
750
|
+
if (headers === void 0) await runtime2.row([]);
|
|
751
751
|
return 0;
|
|
752
752
|
}
|
|
753
753
|
var csvcut = {
|
|
@@ -1116,9 +1116,9 @@ var csvcut = {
|
|
|
1116
1116
|
|
|
1117
1117
|
// packages/safe-bash-command-csvkit/src/commands/csvformat.ts
|
|
1118
1118
|
init_buffer();
|
|
1119
|
-
async function format(
|
|
1120
|
-
const o =
|
|
1121
|
-
await
|
|
1119
|
+
async function format(runtime2) {
|
|
1120
|
+
const o = runtime2.options;
|
|
1121
|
+
await runtime2.prompt();
|
|
1122
1122
|
const dialect = {
|
|
1123
1123
|
delimiter: o.out_asv ? "" : o.out_tabs ? " " : String(o.out_delimiter || ","),
|
|
1124
1124
|
lineterminator: o.out_asv ? "" : String(o.out_lineterminator || "\n"),
|
|
@@ -1129,19 +1129,19 @@ async function format(runtime) {
|
|
|
1129
1129
|
};
|
|
1130
1130
|
writeCsvRow([], dialect);
|
|
1131
1131
|
if (o.out_quoting === 2) {
|
|
1132
|
-
const table = await readTable(
|
|
1133
|
-
if (!o.skip_header) await
|
|
1134
|
-
for (const row of table.rows) await
|
|
1132
|
+
const table = await readTable(runtime2, void 0, void 0, true, false, true);
|
|
1133
|
+
if (!o.skip_header) await runtime2.row(table.headers, dialect);
|
|
1134
|
+
for (const row of table.rows) await runtime2.row(row, dialect);
|
|
1135
1135
|
return 0;
|
|
1136
1136
|
}
|
|
1137
1137
|
let first = true;
|
|
1138
|
-
for await (const record of
|
|
1138
|
+
for await (const record of runtime2.records(void 0, void 0, void 0, void 0, void 0, true)) {
|
|
1139
1139
|
if (first) {
|
|
1140
1140
|
first = false;
|
|
1141
|
-
if (o.no_header_row && !o.skip_header) await
|
|
1141
|
+
if (o.no_header_row && !o.skip_header) await runtime2.row(defaultHeaders(record.cells.length), dialect);
|
|
1142
1142
|
if (!o.no_header_row && o.skip_header) continue;
|
|
1143
1143
|
}
|
|
1144
|
-
await
|
|
1144
|
+
await runtime2.row(record.cells.map(inputWriteCell), dialect);
|
|
1145
1145
|
}
|
|
1146
1146
|
if (first && (o.no_header_row || o.skip_header)) throw new CsvkitDiagnostic("StopIteration: ");
|
|
1147
1147
|
return 0;
|
|
@@ -1615,13 +1615,13 @@ var csvformat = {
|
|
|
1615
1615
|
|
|
1616
1616
|
// packages/safe-bash-command-csvkit/src/commands/csvgrep.ts
|
|
1617
1617
|
init_buffer();
|
|
1618
|
-
async function grep(
|
|
1619
|
-
const o =
|
|
1620
|
-
if (o.names_only) return printNames(
|
|
1621
|
-
await
|
|
1622
|
-
if (!o.columns)
|
|
1623
|
-
if (o.regex === null && o.pattern === null && o.matchfile === null)
|
|
1624
|
-
const rows =
|
|
1618
|
+
async function grep(runtime2) {
|
|
1619
|
+
const o = runtime2.options;
|
|
1620
|
+
if (o.names_only) return printNames(runtime2);
|
|
1621
|
+
await runtime2.prompt();
|
|
1622
|
+
if (!o.columns) runtime2.error("You must specify at least one column to search using the -c option.");
|
|
1623
|
+
if (o.regex === null && o.pattern === null && o.matchfile === null) runtime2.error("One of -r, -m or -f must be specified, unless using the -n option.");
|
|
1624
|
+
const rows = runtime2.records();
|
|
1625
1625
|
try {
|
|
1626
1626
|
const first = await rows.next();
|
|
1627
1627
|
const numbered = Boolean(o.line_numbers);
|
|
@@ -1638,42 +1638,42 @@ async function grep(runtime) {
|
|
|
1638
1638
|
headers,
|
|
1639
1639
|
(o.zero_based ? 0 : 1) - (numbered ? 1 : 0),
|
|
1640
1640
|
null,
|
|
1641
|
-
|
|
1641
|
+
runtime2.step,
|
|
1642
1642
|
generated
|
|
1643
1643
|
);
|
|
1644
1644
|
let predicate;
|
|
1645
|
-
if (o.regex) predicate = compilePythonSearch(String(o.regex),
|
|
1645
|
+
if (o.regex) predicate = compilePythonSearch(String(o.regex), runtime2.step, runtime2.context.limits.maxRegexWork, runtime2.retain.bind(runtime2));
|
|
1646
1646
|
else if (o.matchfile) {
|
|
1647
1647
|
const file = o.matchfile;
|
|
1648
1648
|
const lines = /* @__PURE__ */ new Set();
|
|
1649
1649
|
for await (const line of file.lines()) {
|
|
1650
|
-
|
|
1650
|
+
runtime2.step();
|
|
1651
1651
|
let end = line.length;
|
|
1652
1652
|
while (end > 0 && pythonWhitespace(line[end - 1])) {
|
|
1653
|
-
|
|
1653
|
+
runtime2.step();
|
|
1654
1654
|
end--;
|
|
1655
1655
|
}
|
|
1656
1656
|
const value = line.slice(0, end);
|
|
1657
|
-
|
|
1657
|
+
runtime2.retain(64 + value.length * 2);
|
|
1658
1658
|
lines.add(value);
|
|
1659
1659
|
}
|
|
1660
|
-
await
|
|
1660
|
+
await runtime2.matchFiles.close(file);
|
|
1661
1661
|
predicate = (value) => lines.has(value);
|
|
1662
1662
|
} else if (o.pattern) predicate = (value) => value.includes(String(o.pattern));
|
|
1663
1663
|
const selected = predicate ? [...new Set(columns)] : [];
|
|
1664
1664
|
const writeMatch = async (row) => {
|
|
1665
1665
|
let matches = !o.any_match;
|
|
1666
1666
|
for (const column of selected) {
|
|
1667
|
-
|
|
1667
|
+
runtime2.step();
|
|
1668
1668
|
const result = predicate(row[column] ?? "");
|
|
1669
1669
|
if (o.any_match ? result : !result) {
|
|
1670
1670
|
matches = result;
|
|
1671
1671
|
break;
|
|
1672
1672
|
}
|
|
1673
1673
|
}
|
|
1674
|
-
if (matches !== Boolean(o.inverse)) await
|
|
1674
|
+
if (matches !== Boolean(o.inverse)) await runtime2.row(row, {}, false);
|
|
1675
1675
|
};
|
|
1676
|
-
await
|
|
1676
|
+
await runtime2.row(headers, {}, false);
|
|
1677
1677
|
if (!first.done && generated) await writeMatch(firstRow);
|
|
1678
1678
|
for await (const record of rows) await writeMatch(numberedRow(record));
|
|
1679
1679
|
return 0;
|
|
@@ -2095,8 +2095,8 @@ var csvgrep = {
|
|
|
2095
2095
|
|
|
2096
2096
|
// packages/safe-bash-command-csvkit/src/commands/csvjoin.ts
|
|
2097
2097
|
init_buffer();
|
|
2098
|
-
function keyValue(value,
|
|
2099
|
-
|
|
2098
|
+
function keyValue(value, runtime2) {
|
|
2099
|
+
runtime2.step();
|
|
2100
2100
|
if (value === null) return "null";
|
|
2101
2101
|
if (typeof value === "string") return "text:" + value;
|
|
2102
2102
|
if (typeof value === "boolean") return value ? "number:1:0" : "number:0";
|
|
@@ -2108,7 +2108,7 @@ function keyValue(value, runtime) {
|
|
|
2108
2108
|
const digits2 = number2.coefficient.toString();
|
|
2109
2109
|
let end = digits2.length;
|
|
2110
2110
|
while (digits2[end - 1] === "0") {
|
|
2111
|
-
|
|
2111
|
+
runtime2.step();
|
|
2112
2112
|
end--;
|
|
2113
2113
|
}
|
|
2114
2114
|
return "number:" + (number2.negative ? "-" : "") + digits2.slice(0, end) + ":" + (number2.exponent + digits2.length - end);
|
|
@@ -2118,14 +2118,14 @@ function keyValue(value, runtime) {
|
|
|
2118
2118
|
const aware = value.value.slice(10).includes("+") || value.value.slice(10).includes("-");
|
|
2119
2119
|
return "datetime:" + aware + ":" + temporalOrder(value.value);
|
|
2120
2120
|
}
|
|
2121
|
-
async function combine(
|
|
2121
|
+
async function combine(runtime2, left, right, leftKey, rightKey, full, inner) {
|
|
2122
2122
|
let omittedKey = rightKey;
|
|
2123
2123
|
if (!full && rightKey !== void 0) {
|
|
2124
2124
|
for (let index = 0; index < rightKey; index++) {
|
|
2125
|
-
|
|
2125
|
+
runtime2.step();
|
|
2126
2126
|
if (right.rows.every((row) => {
|
|
2127
|
-
|
|
2128
|
-
return row[index] === row[rightKey] || keyValue(row[index],
|
|
2127
|
+
runtime2.step();
|
|
2128
|
+
return row[index] === row[rightKey] || keyValue(row[index], runtime2) === keyValue(row[rightKey], runtime2);
|
|
2129
2129
|
})) {
|
|
2130
2130
|
omittedKey = index;
|
|
2131
2131
|
break;
|
|
@@ -2133,15 +2133,15 @@ async function combine(runtime, left, right, leftKey, rightKey, full, inner) {
|
|
|
2133
2133
|
}
|
|
2134
2134
|
}
|
|
2135
2135
|
const included = right.headers.map((_, index) => index).filter((index) => full || index !== omittedKey);
|
|
2136
|
-
if (left.headers.length + included.length >
|
|
2137
|
-
const headers = await normalizeHeaders([...left.headers, ...included.map((index) => left.headers.includes(right.headers[index]) ? right.headers[index] + "2" : right.headers[index])],
|
|
2136
|
+
if (left.headers.length + included.length > runtime2.context.limits.maxColumns) throw new CsvkitBlocked("column budget exceeded");
|
|
2137
|
+
const headers = await normalizeHeaders([...left.headers, ...included.map((index) => left.headers.includes(right.headers[index]) ? right.headers[index] + "2" : right.headers[index])], runtime2);
|
|
2138
2138
|
const hash = /* @__PURE__ */ new Map();
|
|
2139
2139
|
const rightKeys = [];
|
|
2140
2140
|
for (const [index, row] of right.rows.entries()) {
|
|
2141
|
-
|
|
2142
|
-
|
|
2143
|
-
const key = rightKey === void 0 ? "row:" + index : keyValue(row[rightKey],
|
|
2144
|
-
|
|
2141
|
+
runtime2.step();
|
|
2142
|
+
runtime2.retain(96);
|
|
2143
|
+
const key = rightKey === void 0 ? "row:" + index : keyValue(row[rightKey], runtime2);
|
|
2144
|
+
runtime2.retain(typeof key === "string" ? key.length * 2 : 16);
|
|
2145
2145
|
rightKeys.push(key);
|
|
2146
2146
|
const matches = hash.get(key);
|
|
2147
2147
|
if (matches) matches.push(row);
|
|
@@ -2150,43 +2150,43 @@ async function combine(runtime, left, right, leftKey, rightKey, full, inner) {
|
|
|
2150
2150
|
const rows = [];
|
|
2151
2151
|
const seen = /* @__PURE__ */ new Set();
|
|
2152
2152
|
const append = (a, b) => {
|
|
2153
|
-
|
|
2154
|
-
if (rows.length >=
|
|
2155
|
-
|
|
2153
|
+
runtime2.step();
|
|
2154
|
+
if (rows.length >= runtime2.context.limits.maxRows) throw new CsvkitBlocked("join result row budget exceeded");
|
|
2155
|
+
runtime2.retain(64 + headers.length * 16);
|
|
2156
2156
|
rows.push([...a, ...included.map((index) => {
|
|
2157
|
-
|
|
2157
|
+
runtime2.step();
|
|
2158
2158
|
return b?.[index] ?? null;
|
|
2159
2159
|
})]);
|
|
2160
2160
|
};
|
|
2161
2161
|
for (const [index, row] of left.rows.entries()) {
|
|
2162
|
-
|
|
2163
|
-
|
|
2164
|
-
const key = leftKey === void 0 ? "row:" + index : keyValue(row[leftKey],
|
|
2165
|
-
|
|
2162
|
+
runtime2.step();
|
|
2163
|
+
runtime2.retain(32);
|
|
2164
|
+
const key = leftKey === void 0 ? "row:" + index : keyValue(row[leftKey], runtime2);
|
|
2165
|
+
runtime2.retain(typeof key === "string" ? key.length * 2 : 16);
|
|
2166
2166
|
seen.add(key);
|
|
2167
2167
|
const matches = hash.get(key);
|
|
2168
2168
|
if (matches) for (const match2 of matches) append(row, match2);
|
|
2169
2169
|
else if (!inner) append(row);
|
|
2170
2170
|
}
|
|
2171
2171
|
if (full) for (const [index, row] of right.rows.entries()) {
|
|
2172
|
-
|
|
2172
|
+
runtime2.step();
|
|
2173
2173
|
if (!seen.has(rightKeys[index])) append(left.headers.map(() => null), row);
|
|
2174
2174
|
}
|
|
2175
2175
|
return { headers, rows };
|
|
2176
2176
|
}
|
|
2177
|
-
async function join(
|
|
2178
|
-
const o =
|
|
2177
|
+
async function join(runtime2) {
|
|
2178
|
+
const o = runtime2.options;
|
|
2179
2179
|
const paths = o.input_paths;
|
|
2180
|
-
if (
|
|
2180
|
+
if (runtime2.context.terminal.stdinIsTTY && paths.length === 1 && paths[0] === "-") runtime2.error("You must provide an input file or piped data.");
|
|
2181
2181
|
let names = o.columns ? String(o.columns).split(",").map(stripWhitespace) : [];
|
|
2182
2182
|
if (names.length === 1) names = paths.map(() => names[0]);
|
|
2183
|
-
if (names.length && names.length !== paths.length)
|
|
2184
|
-
if ((o.left_join || o.right_join || o.outer_join) && !o.columns)
|
|
2185
|
-
if (o.left_join && o.right_join)
|
|
2183
|
+
if (names.length && names.length !== paths.length) runtime2.error("The number of join column names must match the number of files, or be a single column name that exists in all files.");
|
|
2184
|
+
if ((o.left_join || o.right_join || o.outer_join) && !o.columns) runtime2.error("You must provide join column names when performing an outer join.");
|
|
2185
|
+
if (o.left_join && o.right_join) runtime2.error("It is not valid to specify both a left and a right join.");
|
|
2186
2186
|
const tables = [];
|
|
2187
2187
|
for (const path of paths) {
|
|
2188
|
-
tables.push(await readTable(
|
|
2189
|
-
if (path === "-") await
|
|
2188
|
+
tables.push(await readTable(runtime2, path, void 0, true));
|
|
2189
|
+
if (path === "-") await runtime2.input(path, true).close();
|
|
2190
2190
|
}
|
|
2191
2191
|
if (!tables.length) throw new CsvkitDiagnostic("IndexError: list index out of range");
|
|
2192
2192
|
const keys = names.map((name, index) => match(tables[index].headers, name, 1, true));
|
|
@@ -2195,10 +2195,10 @@ async function join(runtime) {
|
|
|
2195
2195
|
keys.reverse();
|
|
2196
2196
|
}
|
|
2197
2197
|
let table = tables[0];
|
|
2198
|
-
for (let index = 1; index < tables.length; index++) table = await combine(
|
|
2199
|
-
await
|
|
2200
|
-
for (const row of table.rows) await
|
|
2201
|
-
|
|
2198
|
+
for (let index = 1; index < tables.length; index++) table = await combine(runtime2, table, tables[index], keys[0], keys[index], !names.length || Boolean(o.outer_join && !o.left_join && !o.right_join), Boolean(names.length && !o.outer_join && !o.left_join && !o.right_join));
|
|
2199
|
+
await runtime2.row(table.headers);
|
|
2200
|
+
for (const row of table.rows) await runtime2.row(row.map((value) => {
|
|
2201
|
+
runtime2.step();
|
|
2202
2202
|
return typeof value === "object" && value?.kind === "datetime" ? { ...value, value: value.value.replace(" ", "T") } : value;
|
|
2203
2203
|
}));
|
|
2204
2204
|
return 0;
|
|
@@ -2792,7 +2792,7 @@ init_buffer();
|
|
|
2792
2792
|
|
|
2793
2793
|
// packages/safe-bash-command-csvkit/src/json-input.ts
|
|
2794
2794
|
init_buffer();
|
|
2795
|
-
function decodeJson(text2,
|
|
2795
|
+
function decodeJson(text2, runtime2) {
|
|
2796
2796
|
if (text2.startsWith("\uFEFF")) throw new CsvkitDiagnostic("JSONDecodeError: Unexpected UTF-8 BOM (decode using utf-8-sig): line 1 column 1 (char 0)");
|
|
2797
2797
|
let position = 0;
|
|
2798
2798
|
const error = (message, at = position) => {
|
|
@@ -2805,7 +2805,7 @@ function decodeJson(text2, runtime) {
|
|
|
2805
2805
|
while (position < text2.length) {
|
|
2806
2806
|
const c = text2.charCodeAt(position);
|
|
2807
2807
|
if (c !== 32 && c !== 9 && c !== 13 && c !== 10) break;
|
|
2808
|
-
|
|
2808
|
+
runtime2.step();
|
|
2809
2809
|
position++;
|
|
2810
2810
|
}
|
|
2811
2811
|
};
|
|
@@ -2817,18 +2817,18 @@ function decodeJson(text2, runtime) {
|
|
|
2817
2817
|
let hasEscape = false;
|
|
2818
2818
|
let hasSurrogate = false;
|
|
2819
2819
|
while (position < text2.length) {
|
|
2820
|
-
|
|
2820
|
+
runtime2.step();
|
|
2821
2821
|
const char = text2[position++];
|
|
2822
2822
|
if (char === '"') {
|
|
2823
|
-
|
|
2823
|
+
runtime2.retain(32 + (position - start) * 4);
|
|
2824
2824
|
if (!hasEscape && !hasSurrogate) {
|
|
2825
2825
|
const result3 = text2.slice(start + 1, position - 1);
|
|
2826
|
-
|
|
2826
|
+
runtime2.step();
|
|
2827
2827
|
return result3;
|
|
2828
2828
|
}
|
|
2829
2829
|
const result2 = JSON.parse(text2.slice(start, position));
|
|
2830
2830
|
for (const decoded of result2) {
|
|
2831
|
-
|
|
2831
|
+
runtime2.step();
|
|
2832
2832
|
const point = decoded.codePointAt(0);
|
|
2833
2833
|
if (point >= 55296 && point <= 57343) throw new CsvkitBlocked("JSON unpaired surrogate output encoding profile");
|
|
2834
2834
|
}
|
|
@@ -2856,22 +2856,22 @@ function decodeJson(text2, runtime) {
|
|
|
2856
2856
|
if (!(precedingHighSurrogate && unit >= 56320 && unit <= 57343)) characters++;
|
|
2857
2857
|
if (unit >= 55296) hasSurrogate = true;
|
|
2858
2858
|
precedingHighSurrogate = unit >= 55296 && unit <= 56319;
|
|
2859
|
-
if (characters >
|
|
2859
|
+
if (characters > runtime2.context.limits.maxFieldCharacters) throw new CsvkitBlocked("JSON string field budget exceeded");
|
|
2860
2860
|
}
|
|
2861
2861
|
return error("Unterminated string starting at", start);
|
|
2862
2862
|
};
|
|
2863
2863
|
const value = (depth) => {
|
|
2864
|
-
|
|
2864
|
+
runtime2.step();
|
|
2865
2865
|
white();
|
|
2866
2866
|
const char = text2[position];
|
|
2867
2867
|
if (char === '"') return string();
|
|
2868
2868
|
if (char === "[" || char === "{") {
|
|
2869
|
-
if (depth >
|
|
2869
|
+
if (depth > runtime2.context.limits.maxNestingDepth) throw new CsvkitBlocked("JSON nesting budget exceeded");
|
|
2870
2870
|
const object = char === "{";
|
|
2871
2871
|
const close = object ? "}" : "]";
|
|
2872
2872
|
const map = /* @__PURE__ */ new Map();
|
|
2873
2873
|
const array = [];
|
|
2874
|
-
|
|
2874
|
+
runtime2.retain(64);
|
|
2875
2875
|
position++;
|
|
2876
2876
|
white();
|
|
2877
2877
|
if (text2[position] === close) {
|
|
@@ -2879,7 +2879,7 @@ function decodeJson(text2, runtime) {
|
|
|
2879
2879
|
return object ? map : array;
|
|
2880
2880
|
}
|
|
2881
2881
|
while (true) {
|
|
2882
|
-
|
|
2882
|
+
runtime2.step();
|
|
2883
2883
|
white();
|
|
2884
2884
|
let key;
|
|
2885
2885
|
if (object) {
|
|
@@ -2889,7 +2889,7 @@ function decodeJson(text2, runtime) {
|
|
|
2889
2889
|
if (text2[position++] !== ":") error("Expecting ':' delimiter", position - 1);
|
|
2890
2890
|
}
|
|
2891
2891
|
const child2 = value(depth + 1);
|
|
2892
|
-
|
|
2892
|
+
runtime2.retain(48);
|
|
2893
2893
|
if (object) map.set(key, child2);
|
|
2894
2894
|
else array.push(child2);
|
|
2895
2895
|
white();
|
|
@@ -2912,7 +2912,7 @@ function decodeJson(text2, runtime) {
|
|
|
2912
2912
|
for (const word of ["NaN", "Infinity", "-Infinity"]) {
|
|
2913
2913
|
if (text2.startsWith(word, position)) {
|
|
2914
2914
|
position += word.length;
|
|
2915
|
-
|
|
2915
|
+
runtime2.retain(32 + word.length * 2);
|
|
2916
2916
|
return { token: word };
|
|
2917
2917
|
}
|
|
2918
2918
|
}
|
|
@@ -2921,13 +2921,13 @@ function decodeJson(text2, runtime) {
|
|
|
2921
2921
|
if (!digit()) return error("Expecting value", start);
|
|
2922
2922
|
if (text2[position] === "0") position++;
|
|
2923
2923
|
else while (digit()) {
|
|
2924
|
-
|
|
2924
|
+
runtime2.step();
|
|
2925
2925
|
position++;
|
|
2926
2926
|
}
|
|
2927
2927
|
if (text2[position] === "." && text2[position + 1] !== void 0 && text2[position + 1] >= "0" && text2[position + 1] <= "9") {
|
|
2928
2928
|
position++;
|
|
2929
2929
|
while (digit()) {
|
|
2930
|
-
|
|
2930
|
+
runtime2.step();
|
|
2931
2931
|
position++;
|
|
2932
2932
|
}
|
|
2933
2933
|
}
|
|
@@ -2936,17 +2936,17 @@ function decodeJson(text2, runtime) {
|
|
|
2936
2936
|
if (text2[position] === "+" || text2[position] === "-") position++;
|
|
2937
2937
|
if (!digit()) position = exponentStart;
|
|
2938
2938
|
else while (digit()) {
|
|
2939
|
-
|
|
2939
|
+
runtime2.step();
|
|
2940
2940
|
position++;
|
|
2941
2941
|
}
|
|
2942
2942
|
}
|
|
2943
2943
|
const token = text2.slice(start, position);
|
|
2944
|
-
if (token.length >
|
|
2944
|
+
if (token.length > runtime2.context.limits.maxDecimalDigits) throw new CsvkitBlocked("JSON number digit budget exceeded");
|
|
2945
2945
|
if (!token.includes(".") && !token.toLowerCase().includes("e")) {
|
|
2946
2946
|
const digits2 = token.length - (token.startsWith("-") ? 1 : 0);
|
|
2947
2947
|
if (digits2 > 4300) throw new CsvkitDiagnostic(`ValueError: Exceeds the limit (4300 digits) for integer string conversion: value has ${digits2} digits; use sys.set_int_max_str_digits() to increase the limit`);
|
|
2948
2948
|
}
|
|
2949
|
-
|
|
2949
|
+
runtime2.retain(32 + token.length * 2);
|
|
2950
2950
|
return { token };
|
|
2951
2951
|
};
|
|
2952
2952
|
const result = value(0);
|
|
@@ -2954,7 +2954,7 @@ function decodeJson(text2, runtime) {
|
|
|
2954
2954
|
if (position !== text2.length) error("Extra data");
|
|
2955
2955
|
return result;
|
|
2956
2956
|
}
|
|
2957
|
-
function jsonNumberText(number2,
|
|
2957
|
+
function jsonNumberText(number2, runtime2) {
|
|
2958
2958
|
const token = number2.token;
|
|
2959
2959
|
if (token === "NaN") return "nan";
|
|
2960
2960
|
if (token === "Infinity") return "inf";
|
|
@@ -2964,11 +2964,11 @@ function jsonNumberText(number2, runtime) {
|
|
|
2964
2964
|
const [mantissa, exponent = "0"] = token.slice(negative ? 1 : 0).toLowerCase().split("e");
|
|
2965
2965
|
const [integer3, fraction = ""] = mantissa.split(".");
|
|
2966
2966
|
const explicit = BigInt(exponent);
|
|
2967
|
-
if (Number.isFinite(
|
|
2967
|
+
if (Number.isFinite(runtime2.context.limits.maxDecimalExponent) && (explicit > BigInt(runtime2.context.limits.maxDecimalExponent) || explicit < -BigInt(runtime2.context.limits.maxDecimalExponent))) throw new CsvkitBlocked("JSON decimal exponent budget exceeded");
|
|
2968
2968
|
let digits2 = integer3 + fraction;
|
|
2969
2969
|
let first = 0;
|
|
2970
2970
|
while (first < digits2.length - 1 && digits2[first] === "0") {
|
|
2971
|
-
|
|
2971
|
+
runtime2.step();
|
|
2972
2972
|
first++;
|
|
2973
2973
|
}
|
|
2974
2974
|
digits2 = digits2.slice(first);
|
|
@@ -2979,16 +2979,16 @@ function jsonNumberText(number2, runtime) {
|
|
|
2979
2979
|
if (scale === 0) return sign + digits2;
|
|
2980
2980
|
const point = digits2.length + scale;
|
|
2981
2981
|
if (point > 0) return sign + digits2.slice(0, point) + "." + digits2.slice(point);
|
|
2982
|
-
|
|
2982
|
+
runtime2.retain((-point + 2) * 2);
|
|
2983
2983
|
return sign + "0." + "0".repeat(-point) + digits2;
|
|
2984
2984
|
}
|
|
2985
2985
|
|
|
2986
2986
|
// packages/safe-bash-command-csvkit/src/geojson/index.ts
|
|
2987
2987
|
var GeoJsonGenerator = class {
|
|
2988
|
-
constructor(
|
|
2989
|
-
this.runtime =
|
|
2988
|
+
constructor(runtime2, headers, tuple) {
|
|
2989
|
+
this.runtime = runtime2;
|
|
2990
2990
|
this.headers = headers;
|
|
2991
|
-
const o =
|
|
2991
|
+
const o = runtime2.options;
|
|
2992
2992
|
const column = (name) => match(headers, String(o[name]), Number(Boolean(o.zero_based)), tuple);
|
|
2993
2993
|
this.lat = column("lat");
|
|
2994
2994
|
this.lon = column("lon");
|
|
@@ -3147,53 +3147,53 @@ var GeoJsonGenerator = class {
|
|
|
3147
3147
|
};
|
|
3148
3148
|
|
|
3149
3149
|
// packages/safe-bash-command-csvkit/src/commands/csvjson.ts
|
|
3150
|
-
async function json(
|
|
3151
|
-
const o =
|
|
3152
|
-
if (o.lat && !o.lon)
|
|
3153
|
-
if (o.lon && !o.lat)
|
|
3154
|
-
for (const option of ["crs", "type", "geometry"]) if (o[option] && !o.lat)
|
|
3150
|
+
async function json(runtime2) {
|
|
3151
|
+
const o = runtime2.options;
|
|
3152
|
+
if (o.lat && !o.lon) runtime2.error("--lon is required whenever --lat is specified.");
|
|
3153
|
+
if (o.lon && !o.lat) runtime2.error("--lat is required whenever --lon is specified.");
|
|
3154
|
+
for (const option of ["crs", "type", "geometry"]) if (o[option] && !o.lat) runtime2.error(`--${option} is only allowed when --lat and --lon are also specified.`);
|
|
3155
3155
|
const geo = Boolean(o.lat && o.lon);
|
|
3156
|
-
if (o.key && o.streamOutput && !geo)
|
|
3156
|
+
if (o.key && o.streamOutput && !geo) runtime2.error("--key is only allowed with --stream when --lat and --lon are also specified.");
|
|
3157
3157
|
const raw = Boolean(o.streamOutput && o.no_inference && o.sniff_limit === 0 && !o.skip_lines);
|
|
3158
|
-
if (raw && !
|
|
3159
|
-
if (
|
|
3160
|
-
if (raw) await
|
|
3158
|
+
if (raw && !runtime2.input().codec.decodeStream) throw new CsvkitBlocked("incremental JSON input requires a streaming codec");
|
|
3159
|
+
if (runtime2.context.terminal.stdinIsTTY && !o.input_path && !raw) runtime2.error("You must provide an input file or piped data.");
|
|
3160
|
+
if (raw) await runtime2.prompt();
|
|
3161
3161
|
const indent = o.indent === null ? null : Math.max(0, Number(o.indent));
|
|
3162
|
-
if (indent !== null && (!Number.isSafeInteger(indent) || indent >
|
|
3163
|
-
if (geo && indent !== null)
|
|
3162
|
+
if (indent !== null && (!Number.isSafeInteger(indent) || indent > runtime2.context.limits.maxOutputBytes)) throw new CsvkitBlocked("JSON indentation budget exceeded");
|
|
3163
|
+
if (geo && indent !== null) runtime2.retain(indent * 12);
|
|
3164
3164
|
if (!raw) {
|
|
3165
|
-
if (!geo) return jsonTable(
|
|
3166
|
-
const table = await readTable(
|
|
3167
|
-
const generator2 = new GeoJsonGenerator(
|
|
3165
|
+
if (!geo) return jsonTable(runtime2, indent);
|
|
3166
|
+
const table = await readTable(runtime2, void 0, void 0, true);
|
|
3167
|
+
const generator2 = new GeoJsonGenerator(runtime2, table.headers, true);
|
|
3168
3168
|
if (o.streamOutput) {
|
|
3169
3169
|
for (const row of table.rows) {
|
|
3170
|
-
await emit(generator2.feature(row),
|
|
3171
|
-
await
|
|
3170
|
+
await emit(generator2.feature(row), runtime2, indent);
|
|
3171
|
+
await runtime2.write("\n");
|
|
3172
3172
|
}
|
|
3173
|
-
} else await emit(generator2.collection(table.rows),
|
|
3173
|
+
} else await emit(generator2.collection(table.rows), runtime2, indent);
|
|
3174
3174
|
return 0;
|
|
3175
3175
|
}
|
|
3176
3176
|
let headers;
|
|
3177
3177
|
let generator;
|
|
3178
|
-
for await (const record of
|
|
3178
|
+
for await (const record of runtime2.records()) {
|
|
3179
3179
|
if (headers === void 0) {
|
|
3180
3180
|
headers = record.cells;
|
|
3181
|
-
if (geo) generator = new GeoJsonGenerator(
|
|
3181
|
+
if (geo) generator = new GeoJsonGenerator(runtime2, headers, false);
|
|
3182
3182
|
continue;
|
|
3183
3183
|
}
|
|
3184
3184
|
let value;
|
|
3185
3185
|
if (generator) value = generator.feature(record.cells);
|
|
3186
3186
|
else {
|
|
3187
3187
|
const fields = /* @__PURE__ */ new Map();
|
|
3188
|
-
|
|
3188
|
+
runtime2.retain(64 + headers.length * 64 + (indent ?? 0) * headers.length * 2);
|
|
3189
3189
|
for (const [index, name] of headers.entries()) {
|
|
3190
|
-
|
|
3190
|
+
runtime2.step();
|
|
3191
3191
|
fields.set(name, record.cells[index] ?? null);
|
|
3192
3192
|
}
|
|
3193
3193
|
value = fields;
|
|
3194
3194
|
}
|
|
3195
|
-
await emit(value,
|
|
3196
|
-
await
|
|
3195
|
+
await emit(value, runtime2, indent);
|
|
3196
|
+
await runtime2.write("\n");
|
|
3197
3197
|
}
|
|
3198
3198
|
if (headers === void 0) throw new CsvkitDiagnostic("StopIteration: ");
|
|
3199
3199
|
return 0;
|
|
@@ -3807,22 +3807,22 @@ function numberText(value, places, ellipsis) {
|
|
|
3807
3807
|
}
|
|
3808
3808
|
return (d.negative ? "-" : "") + grouped + (places ? "." + digits2.slice(-places) : "") + ellipsis;
|
|
3809
3809
|
}
|
|
3810
|
-
async function printTable(
|
|
3811
|
-
const o =
|
|
3810
|
+
async function printTable(runtime2, table) {
|
|
3811
|
+
const o = runtime2.options;
|
|
3812
3812
|
const maxColumns = o.max_columns === null ? table.headers.length : Number(o.max_columns);
|
|
3813
3813
|
const width = o.max_column_width === null ? void 0 : Number(o.max_column_width);
|
|
3814
3814
|
const maxPrecision = o.max_precision === null ? 3 : Number(o.max_precision);
|
|
3815
3815
|
const length = (value) => {
|
|
3816
3816
|
let size = 0;
|
|
3817
3817
|
for (const ignoredChar of value) {
|
|
3818
|
-
|
|
3818
|
+
runtime2.step();
|
|
3819
3819
|
size++;
|
|
3820
3820
|
}
|
|
3821
3821
|
return size;
|
|
3822
3822
|
};
|
|
3823
3823
|
const truncate = (value) => {
|
|
3824
|
-
|
|
3825
|
-
|
|
3824
|
+
runtime2.step();
|
|
3825
|
+
runtime2.retain(32 + value.length * 16);
|
|
3826
3826
|
const chars = Array.from(value);
|
|
3827
3827
|
return width !== void 0 && chars.length > width ? chars.slice(0, width - 3).join("") + "..." : value;
|
|
3828
3828
|
};
|
|
@@ -3830,16 +3830,16 @@ async function printTable(runtime, table) {
|
|
|
3830
3830
|
if (maxColumns < table.headers.length) names.push("...");
|
|
3831
3831
|
const widths = names.map(length);
|
|
3832
3832
|
const formats2 = table.columns.map((column, index) => {
|
|
3833
|
-
|
|
3833
|
+
runtime2.step();
|
|
3834
3834
|
if (index >= maxColumns || column.type !== "Number") return void 0;
|
|
3835
3835
|
const places = precision(table.rows.map((row) => row[index]));
|
|
3836
3836
|
return { places: Math.min(places, maxPrecision), ellipsis: places > maxPrecision && !o.no_number_ellipsis ? "\u2026" : "" };
|
|
3837
3837
|
});
|
|
3838
3838
|
const rows = table.rows.map((row) => {
|
|
3839
|
-
|
|
3839
|
+
runtime2.retain(32 + names.length * 16);
|
|
3840
3840
|
const cells = [];
|
|
3841
3841
|
for (const [j, value] of row.entries()) {
|
|
3842
|
-
|
|
3842
|
+
runtime2.step();
|
|
3843
3843
|
const format3 = formats2[j];
|
|
3844
3844
|
let text2 = j >= maxColumns ? "..." : value === null ? "" : format3 && typeof value === "object" && value.kind === "decimal" ? numberText(value.value, format3.places, format3.ellipsis) : pythonValueText(value).replaceAll("\n", "\u21B5");
|
|
3845
3845
|
text2 = truncate(text2);
|
|
@@ -3851,28 +3851,28 @@ async function printTable(runtime, table) {
|
|
|
3851
3851
|
});
|
|
3852
3852
|
const writeRow = async (cells) => {
|
|
3853
3853
|
const fields = cells.map((cell2, j) => {
|
|
3854
|
-
|
|
3854
|
+
runtime2.step();
|
|
3855
3855
|
if (!table.columns[j]) throw new CsvkitDiagnostic("IndexError: tuple index out of range");
|
|
3856
3856
|
const spaces = widths[j] - length(cell2);
|
|
3857
|
-
|
|
3857
|
+
runtime2.retain(16 + Math.max(0, spaces) * 2);
|
|
3858
3858
|
const padding = " ".repeat(Math.max(0, spaces));
|
|
3859
3859
|
return table.columns[j].type === "Text" ? " " + cell2 + padding + " " : " " + padding + cell2 + " ";
|
|
3860
3860
|
});
|
|
3861
|
-
await
|
|
3861
|
+
await runtime2.write("|" + fields.join("|") + "|\n");
|
|
3862
3862
|
};
|
|
3863
3863
|
await writeRow(names);
|
|
3864
|
-
await
|
|
3865
|
-
|
|
3866
|
-
|
|
3864
|
+
await runtime2.write("| " + widths.map((size) => {
|
|
3865
|
+
runtime2.step();
|
|
3866
|
+
runtime2.retain(size * 2);
|
|
3867
3867
|
return "-".repeat(size);
|
|
3868
3868
|
}).join(" | ") + " |\n");
|
|
3869
3869
|
for (const row of rows) await writeRow(row);
|
|
3870
3870
|
}
|
|
3871
3871
|
|
|
3872
3872
|
// packages/safe-bash-command-csvkit/src/commands/csvlook.ts
|
|
3873
|
-
async function look(
|
|
3874
|
-
const o =
|
|
3875
|
-
if (
|
|
3873
|
+
async function look(runtime2) {
|
|
3874
|
+
const o = runtime2.options;
|
|
3875
|
+
if (runtime2.context.terminal.stdinIsTTY && !o.input_path) runtime2.error("You must provide an input file or piped data.");
|
|
3876
3876
|
const requestedRows = o.max_rows === null ? void 0 : BigInt(o.max_rows);
|
|
3877
3877
|
if (requestedRows !== void 0 && (requestedRows < 0n || requestedRows > 9223372036854775807n)) throw new CsvkitDiagnostic("ValueError: Stop argument for islice() must be None or an integer: 0 <= x <= sys.maxsize.");
|
|
3878
3878
|
const rowLimit = requestedRows === void 0 ? void 0 : Number(requestedRows);
|
|
@@ -3880,17 +3880,17 @@ async function look(runtime) {
|
|
|
3880
3880
|
if (o[key] !== null && !Number.isSafeInteger(Number(o[key]))) throw new CsvkitBlocked("Agate display integer budget");
|
|
3881
3881
|
}
|
|
3882
3882
|
if (rowLimit === 0 && o.no_header_row && o.input_path && o.input_path !== "-" && Number(o.sniff_limit) === 0 && Number(o.skip_lines) <= 0) {
|
|
3883
|
-
if (!
|
|
3883
|
+
if (!runtime2.context.probeInputOpen) throw new CsvkitBlocked("zero-row named input open capability");
|
|
3884
3884
|
try {
|
|
3885
|
-
await
|
|
3886
|
-
|
|
3885
|
+
await runtime2.context.probeInputOpen(String(o.input_path), { cwd: runtime2.context.cwd, signal: runtime2.context.signal });
|
|
3886
|
+
runtime2.step();
|
|
3887
3887
|
} catch (failure) {
|
|
3888
|
-
|
|
3888
|
+
runtime2.context.signal.throwIfAborted();
|
|
3889
3889
|
throw fileException(failure, String(o.input_path));
|
|
3890
3890
|
}
|
|
3891
3891
|
}
|
|
3892
|
-
const table = await readTable(
|
|
3893
|
-
await printTable(
|
|
3892
|
+
const table = await readTable(runtime2, void 0, rowLimit, true, Boolean(o.line_numbers));
|
|
3893
|
+
await printTable(runtime2, table);
|
|
3894
3894
|
return 0;
|
|
3895
3895
|
}
|
|
3896
3896
|
var csvlook = {
|
|
@@ -4863,26 +4863,26 @@ var csvpy = {
|
|
|
4863
4863
|
}
|
|
4864
4864
|
]
|
|
4865
4865
|
};
|
|
4866
|
-
async function executeCsvpy(
|
|
4867
|
-
const o =
|
|
4868
|
-
if (!o.input_path || o.input_path === "-")
|
|
4869
|
-
if (
|
|
4866
|
+
async function executeCsvpy(runtime2) {
|
|
4867
|
+
const o = runtime2.options;
|
|
4868
|
+
if (!o.input_path || o.input_path === "-") runtime2.error("csvpy cannot accept input as piped data via STDIN.");
|
|
4869
|
+
if (runtime2.context.probeInputOpen) {
|
|
4870
4870
|
try {
|
|
4871
|
-
await
|
|
4871
|
+
await runtime2.context.probeInputOpen(String(o.input_path), { cwd: runtime2.context.cwd, signal: runtime2.context.signal });
|
|
4872
4872
|
} catch (failure2) {
|
|
4873
|
-
|
|
4873
|
+
runtime2.context.signal.throwIfAborted();
|
|
4874
4874
|
throw fileException(failure2, String(o.input_path));
|
|
4875
4875
|
}
|
|
4876
4876
|
}
|
|
4877
4877
|
const mode = o.as_dict ? "dict" : o.as_agate ? "agate" : "reader";
|
|
4878
|
-
const provider =
|
|
4878
|
+
const provider = runtime2.context.interpreter;
|
|
4879
4879
|
if (!provider || !provider.modes.includes(mode)) throw new CsvkitBlocked(`interpreter capability ${mode}`);
|
|
4880
|
-
const scope = new ResourceScope(
|
|
4880
|
+
const scope = new ResourceScope(runtime2.context.signal, runtime2.context.registerCleanup);
|
|
4881
4881
|
let consumed = 0;
|
|
4882
|
-
const work = Object.freeze({ limit:
|
|
4883
|
-
|
|
4882
|
+
const work = Object.freeze({ limit: runtime2.context.limits.maxInterpreterWork, consume: (units = 1) => {
|
|
4883
|
+
runtime2.step();
|
|
4884
4884
|
if (!Number.isSafeInteger(units) || units < 0) throw new RangeError("invalid interpreter work admission");
|
|
4885
|
-
if (units >
|
|
4885
|
+
if (units > runtime2.context.limits.maxInterpreterWork - consumed) throw new CsvkitBlocked("interpreter work budget exceeded");
|
|
4886
4886
|
consumed += units;
|
|
4887
4887
|
} });
|
|
4888
4888
|
let completed = false;
|
|
@@ -4892,10 +4892,10 @@ async function executeCsvpy(runtime) {
|
|
|
4892
4892
|
work.consume();
|
|
4893
4893
|
const session = await scope.acquire(() => provider.loadConverted ? provider.loadConverted({
|
|
4894
4894
|
mode,
|
|
4895
|
-
retainOutput: (bytes) =>
|
|
4896
|
-
write: (text2, channel) =>
|
|
4895
|
+
retainOutput: (bytes) => runtime2.retain(bytes),
|
|
4896
|
+
write: (text2, channel) => runtime2.write(text2, channel),
|
|
4897
4897
|
reader: async () => {
|
|
4898
|
-
const text2 = await
|
|
4898
|
+
const text2 = await runtime2.text(String(o.input_path));
|
|
4899
4899
|
const dialect = {
|
|
4900
4900
|
delimiter: o.tabs ? " " : String(o.delimiter ?? ","),
|
|
4901
4901
|
quotechar: String(o.quotechar ?? '"'),
|
|
@@ -4903,25 +4903,25 @@ async function executeCsvpy(runtime) {
|
|
|
4903
4903
|
doublequote: o.doublequote !== false,
|
|
4904
4904
|
skipinitialspace: Boolean(o.skipinitialspace),
|
|
4905
4905
|
fieldLimit: Number(o.field_size_limit ?? Infinity),
|
|
4906
|
-
fieldBudget:
|
|
4906
|
+
fieldBudget: runtime2.context.limits.maxFieldCharacters,
|
|
4907
4907
|
...o.escapechar === null || o.escapechar === void 0 ? {} : { escapechar: String(o.escapechar) }
|
|
4908
4908
|
};
|
|
4909
|
-
const reader = readCsvRecoverable(text2, dialect,
|
|
4909
|
+
const reader = readCsvRecoverable(text2, dialect, runtime2.step);
|
|
4910
4910
|
return {
|
|
4911
4911
|
next() {
|
|
4912
4912
|
const next = reader.next();
|
|
4913
|
-
if (!next.done)
|
|
4913
|
+
if (!next.done) runtime2.admitRecord(next.value);
|
|
4914
4914
|
return next;
|
|
4915
4915
|
}
|
|
4916
4916
|
};
|
|
4917
4917
|
},
|
|
4918
|
-
table: () => readTable(
|
|
4918
|
+
table: () => readTable(runtime2, String(o.input_path), void 0, true),
|
|
4919
4919
|
settings: o
|
|
4920
|
-
},
|
|
4920
|
+
}, runtime2.context.signal, work) : provider.load(mode, runtime2.bytes(String(o.input_path), true), o, runtime2.context.signal, work), (session2) => session2.close());
|
|
4921
4921
|
const className = mode === "dict" ? "agate.csv.DictReader" : mode === "agate" ? "agate.Table" : "agate.csv.reader";
|
|
4922
4922
|
const name = mode === "agate" ? "table" : "reader";
|
|
4923
|
-
status = await session.interact(`Welcome! "${o.input_path}" has been loaded in an ${className} object named "${name}".`,
|
|
4924
|
-
|
|
4923
|
+
status = await session.interact(`Welcome! "${o.input_path}" has been loaded in an ${className} object named "${name}".`, runtime2.context.signal) ?? 0;
|
|
4924
|
+
runtime2.context.signal.throwIfAborted();
|
|
4925
4925
|
completed = true;
|
|
4926
4926
|
} catch (caught) {
|
|
4927
4927
|
failure = caught;
|
|
@@ -4931,7 +4931,7 @@ async function executeCsvpy(runtime) {
|
|
|
4931
4931
|
} catch (cleanupFailure) {
|
|
4932
4932
|
if (completed) throw cleanupFailure;
|
|
4933
4933
|
}
|
|
4934
|
-
|
|
4934
|
+
runtime2.context.signal.throwIfAborted();
|
|
4935
4935
|
if (!completed) throw failure;
|
|
4936
4936
|
return status;
|
|
4937
4937
|
}
|
|
@@ -4982,11 +4982,11 @@ function compareDecimals(left, right, step) {
|
|
|
4982
4982
|
}
|
|
4983
4983
|
|
|
4984
4984
|
// packages/safe-bash-command-csvkit/src/commands/csvsort.ts
|
|
4985
|
-
function compareText(a, b,
|
|
4985
|
+
function compareText(a, b, runtime2) {
|
|
4986
4986
|
const left = a[Symbol.iterator]();
|
|
4987
4987
|
const right = b[Symbol.iterator]();
|
|
4988
4988
|
while (true) {
|
|
4989
|
-
|
|
4989
|
+
runtime2.step();
|
|
4990
4990
|
const x = left.next();
|
|
4991
4991
|
const y = right.next();
|
|
4992
4992
|
if (x.done || y.done) return x.done ? y.done ? 0 : -1 : 1;
|
|
@@ -4994,37 +4994,37 @@ function compareText(a, b, runtime) {
|
|
|
4994
4994
|
if (difference) return difference;
|
|
4995
4995
|
}
|
|
4996
4996
|
}
|
|
4997
|
-
async function sort(
|
|
4998
|
-
const o =
|
|
4999
|
-
if (o.names_only) return printNames(
|
|
5000
|
-
if (
|
|
5001
|
-
const table = await readTable(
|
|
4997
|
+
async function sort(runtime2) {
|
|
4998
|
+
const o = runtime2.options;
|
|
4999
|
+
if (o.names_only) return printNames(runtime2);
|
|
5000
|
+
if (runtime2.context.terminal.stdinIsTTY && !o.input_path) runtime2.error("You must provide an input file or piped data.");
|
|
5001
|
+
const table = await readTable(runtime2, void 0, void 0, true);
|
|
5002
5002
|
const headers = table.headers;
|
|
5003
|
-
|
|
5003
|
+
runtime2.retain(32 + table.rows.length * 8);
|
|
5004
5004
|
const rows = [...table.rows];
|
|
5005
|
-
const columns = parseColumnIdentifiers(o.columns, headers, o.zero_based ? 0 : 1, null,
|
|
5006
|
-
|
|
5005
|
+
const columns = parseColumnIdentifiers(o.columns, headers, o.zero_based ? 0 : 1, null, runtime2.step, true);
|
|
5006
|
+
runtime2.retain(64 + rows.length * (32 + columns.length * 8));
|
|
5007
5007
|
const keyed = rows.map((row) => columns.map((index) => {
|
|
5008
|
-
|
|
5008
|
+
runtime2.step();
|
|
5009
5009
|
const value = row[index];
|
|
5010
|
-
const key = o.ignore_case && typeof value === "string" ? upperText(value,
|
|
5011
|
-
if (key !== value && typeof key === "string")
|
|
5010
|
+
const key = o.ignore_case && typeof value === "string" ? upperText(value, runtime2.step) : value;
|
|
5011
|
+
if (key !== value && typeof key === "string") runtime2.retain(32 + key.length * 2);
|
|
5012
5012
|
return key;
|
|
5013
5013
|
}));
|
|
5014
|
-
|
|
5014
|
+
runtime2.retain(32 + rows.length * 8);
|
|
5015
5015
|
const indices = rows.map((_, index) => index);
|
|
5016
5016
|
indices.sort((a, b) => {
|
|
5017
|
-
|
|
5017
|
+
runtime2.step();
|
|
5018
5018
|
for (let index = 0; index < columns.length; index++) {
|
|
5019
5019
|
const x = keyed[a][index];
|
|
5020
5020
|
const y = keyed[b][index];
|
|
5021
5021
|
let order;
|
|
5022
5022
|
if (x === null || y === null) order = x === null ? y === null ? 0 : 1 : -1;
|
|
5023
|
-
else if (typeof x === "string" && typeof y === "string") order = compareText(x, y,
|
|
5023
|
+
else if (typeof x === "string" && typeof y === "string") order = compareText(x, y, runtime2);
|
|
5024
5024
|
else if (typeof x === "boolean" && typeof y === "boolean") order = Number(x) - Number(y);
|
|
5025
5025
|
else if (typeof x === "object" && typeof y === "object" && x.kind === "decimal" && y.kind === "decimal") {
|
|
5026
5026
|
if (x.value.includes("NaN") || y.value.includes("NaN")) throw new CsvkitDiagnostic("InvalidOperation: [<class 'decimal.InvalidOperation'>]");
|
|
5027
|
-
order = compareDecimals(x.value, y.value,
|
|
5027
|
+
order = compareDecimals(x.value, y.value, runtime2.step);
|
|
5028
5028
|
} else if (typeof x === "object" && typeof y === "object" && x.kind === "timedelta" && y.kind === "timedelta") order = x.microseconds < y.microseconds ? -1 : x.microseconds > y.microseconds ? 1 : 0;
|
|
5029
5029
|
else if (typeof x === "object" && typeof y === "object" && (x.kind === "date" || x.kind === "datetime") && x.kind === y.kind) {
|
|
5030
5030
|
if (x.kind === "datetime" && /[+-]\d{2}:\d{2}(?::\d{2}(?:\.\d{6})?)?$/.test(x.value) !== /[+-]\d{2}:\d{2}(?::\d{2}(?:\.\d{6})?)?$/.test(y.value)) throw new CsvkitDiagnostic("TypeError: can't compare offset-naive and offset-aware datetimes");
|
|
@@ -5036,9 +5036,9 @@ async function sort(runtime) {
|
|
|
5036
5036
|
}
|
|
5037
5037
|
return 0;
|
|
5038
5038
|
});
|
|
5039
|
-
await
|
|
5040
|
-
for (const index of indices) await
|
|
5041
|
-
|
|
5039
|
+
await runtime2.row(headers);
|
|
5040
|
+
for (const index of indices) await runtime2.row(rows[index].map((value) => {
|
|
5041
|
+
runtime2.step();
|
|
5042
5042
|
return typeof value === "object" && value?.kind === "datetime" ? { ...value, value: value.value.replace(" ", "T") } : value;
|
|
5043
5043
|
}));
|
|
5044
5044
|
return 0;
|
|
@@ -6606,24 +6606,24 @@ var databases = Object.freeze([generic, mssql, mysql, oracle, postgresql, sqlite
|
|
|
6606
6606
|
function typeName(column, dialect) {
|
|
6607
6607
|
return (column === "Text" ? dialect.textType : void 0) ?? dialect.typedTypes?.[column] ?? databases.find((binding) => binding.default)?.typedTypes?.[column];
|
|
6608
6608
|
}
|
|
6609
|
-
function identifier(name, dialect,
|
|
6610
|
-
|
|
6609
|
+
function identifier(name, dialect, runtime2) {
|
|
6610
|
+
runtime2.step();
|
|
6611
6611
|
if (!name) throw new CsvkitDiagnostic("IndexError: string index out of range");
|
|
6612
6612
|
const legal = "abcdefghijklmnopqrstuvwxyz0123456789_$\u0131\u017F";
|
|
6613
6613
|
const candidate = name.endsWith("\n") ? name.slice(0, -1) : name;
|
|
6614
6614
|
const quote = !candidate || dialect.illegalInitial.includes(name[0]) || dialect.reserved.includes(name.toLowerCase()) || name.toLowerCase() !== name || [...candidate].some((char) => !legal.includes(char));
|
|
6615
6615
|
if (!quote) return name;
|
|
6616
|
-
|
|
6616
|
+
runtime2.retain(name.length * 4 + 16);
|
|
6617
6617
|
const escaped = name.split(dialect.quoteEnd).join(dialect.quoteEnd + dialect.quoteEnd);
|
|
6618
6618
|
return dialect.quoteStart + (dialect.doublePercent ? escaped.split("%").join("%%") : escaped) + dialect.quoteEnd;
|
|
6619
6619
|
}
|
|
6620
|
-
function schemaIdentifier(name, dialect,
|
|
6621
|
-
if (!dialect.multipartSchema || name.startsWith("__[SCHEMA_")) return identifier(name, dialect,
|
|
6622
|
-
|
|
6620
|
+
function schemaIdentifier(name, dialect, runtime2) {
|
|
6621
|
+
if (!dialect.multipartSchema || name.startsWith("__[SCHEMA_")) return identifier(name, dialect, runtime2);
|
|
6622
|
+
runtime2.retain(name.length * 8 + 64);
|
|
6623
6623
|
const parts = [];
|
|
6624
6624
|
let symbol = "", bracket = false, hasBrackets = false;
|
|
6625
6625
|
for (const char of name) {
|
|
6626
|
-
|
|
6626
|
+
runtime2.step();
|
|
6627
6627
|
if (char === "[") {
|
|
6628
6628
|
bracket = true;
|
|
6629
6629
|
hasBrackets = true;
|
|
@@ -6644,24 +6644,24 @@ function schemaIdentifier(name, dialect, runtime) {
|
|
|
6644
6644
|
while (database.startsWith("[")) database = database.slice(1);
|
|
6645
6645
|
while (database.endsWith("]")) database = database.slice(0, -1);
|
|
6646
6646
|
}
|
|
6647
|
-
if (database) return (raw ? database : identifier(database, dialect,
|
|
6648
|
-
return owner ? identifier(owner, dialect,
|
|
6647
|
+
if (database) return (raw ? database : identifier(database, dialect, runtime2)) + "." + identifier(owner, dialect, runtime2);
|
|
6648
|
+
return owner ? identifier(owner, dialect, runtime2) : "";
|
|
6649
6649
|
}
|
|
6650
|
-
function deriveSchema(
|
|
6651
|
-
const o =
|
|
6650
|
+
function deriveSchema(runtime2, table, name, dialect, connected) {
|
|
6651
|
+
const o = runtime2.options;
|
|
6652
6652
|
const unique = o.unique_constraint ? [...new Set(String(o.unique_constraint).split(","))] : [];
|
|
6653
6653
|
const missing = unique.find((column) => !table.headers.includes(column));
|
|
6654
6654
|
if (missing !== void 0) throw new CsvkitDiagnostic(`ConstraintColumnNotFoundError: Can't create UniqueConstraint on table '${name}': no column named '${missing}' is present.`);
|
|
6655
6655
|
const columns = [];
|
|
6656
6656
|
for (const [column, header] of table.headers.entries()) {
|
|
6657
|
-
|
|
6657
|
+
runtime2.step();
|
|
6658
6658
|
const columnType = "columns" in table ? table.columns[column].type : "Text";
|
|
6659
6659
|
let type = typeName(columnType, dialect);
|
|
6660
6660
|
if (!type) throw new CsvkitBlocked(`typed SQL ${columnType} dialect profile ${dialect.name}`);
|
|
6661
6661
|
if (dialect.textLength && columnType === "Text" && !o.no_constraints) {
|
|
6662
6662
|
let length = 0;
|
|
6663
6663
|
for (const row of table.rows) {
|
|
6664
|
-
|
|
6664
|
+
runtime2.step();
|
|
6665
6665
|
length = Math.max(length, Array.from(pythonValueText(row[column] ?? null)).length);
|
|
6666
6666
|
}
|
|
6667
6667
|
const product = Decimal.parse(String(length)).multiply(Decimal.parse(connected ? String(o.col_len_multiplier) : "1"));
|
|
@@ -6672,7 +6672,7 @@ function deriveSchema(runtime, table, name, dialect, connected) {
|
|
|
6672
6672
|
if (columnType === "Number" && dialect.numericPrecision && !o.no_constraints) {
|
|
6673
6673
|
let whole = 1, fractional = 0;
|
|
6674
6674
|
for (const row of table.rows) {
|
|
6675
|
-
|
|
6675
|
+
runtime2.step();
|
|
6676
6676
|
const value = row[column];
|
|
6677
6677
|
if (!value || typeof value !== "object" || value.kind !== "decimal") continue;
|
|
6678
6678
|
if (!Number.isFinite(Number(value.value))) continue;
|
|
@@ -6684,26 +6684,26 @@ function deriveSchema(runtime, table, name, dialect, connected) {
|
|
|
6684
6684
|
type += `(${dialect.numericPrecision}, ${Math.min(fractional, 28 - whole)})`;
|
|
6685
6685
|
}
|
|
6686
6686
|
const required = columnType !== "DateTime" && !o.no_constraints && table.rows.every((row) => {
|
|
6687
|
-
|
|
6687
|
+
runtime2.step();
|
|
6688
6688
|
return row[column] !== null;
|
|
6689
6689
|
});
|
|
6690
6690
|
columns.push(Object.freeze({ name: header, type, required, nullableSuffix: (columnType === "DateTime" ? dialect.timestampNullable : void 0) ?? dialect.nullable ?? "" }));
|
|
6691
|
-
|
|
6691
|
+
runtime2.retain(64 + (header.length + type.length) * 4);
|
|
6692
6692
|
}
|
|
6693
6693
|
return Object.freeze({ name, schema: o.db_schema ? String(o.db_schema) : null, columns: Object.freeze(columns), unique: Object.freeze(unique) });
|
|
6694
6694
|
}
|
|
6695
|
-
function compileCreateTable(
|
|
6696
|
-
const qualified = (schema.schema ? schemaIdentifier(schema.schema, dialect,
|
|
6695
|
+
function compileCreateTable(runtime2, schema, dialect) {
|
|
6696
|
+
const qualified = (schema.schema ? schemaIdentifier(schema.schema, dialect, runtime2) + "." : "") + identifier(schema.name, dialect, runtime2);
|
|
6697
6697
|
const fields = schema.columns.map((column) => {
|
|
6698
|
-
|
|
6698
|
+
runtime2.step();
|
|
6699
6699
|
if (dialect.textLength && column.type === typeName("Text", dialect)) {
|
|
6700
6700
|
throw new CsvkitDiagnostic(`CompileError: (in table '${schema.name}', column '${column.name}'): ${column.type} requires a length on dialect ${dialect.name}`);
|
|
6701
6701
|
}
|
|
6702
|
-
return identifier(column.name, dialect,
|
|
6702
|
+
return identifier(column.name, dialect, runtime2) + " " + column.type + (column.required ? " NOT NULL" : column.nullableSuffix);
|
|
6703
6703
|
});
|
|
6704
|
-
if (schema.unique.length) fields.push("UNIQUE (" + schema.unique.map((column) => identifier(column, dialect,
|
|
6704
|
+
if (schema.unique.length) fields.push("UNIQUE (" + schema.unique.map((column) => identifier(column, dialect, runtime2)).join(", ") + ")");
|
|
6705
6705
|
const statement = "\nCREATE TABLE " + qualified + " (\n" + (fields.length ? " " + fields.join(", \n ") + "\n" : "") + ")\n\n";
|
|
6706
|
-
|
|
6706
|
+
runtime2.retain(statement.length * 4 + 64);
|
|
6707
6707
|
return { qualified, statement };
|
|
6708
6708
|
}
|
|
6709
6709
|
|
|
@@ -8466,23 +8466,23 @@ var csvsql = {
|
|
|
8466
8466
|
usage: reference.usage.replaceAll("{mssql,mysql,oracle,postgresql,sqlite}", choices),
|
|
8467
8467
|
help: reference.help.replaceAll("{mssql,mysql,oracle,postgresql,sqlite}", choices)
|
|
8468
8468
|
};
|
|
8469
|
-
async function executeCsvsql(
|
|
8470
|
-
const o =
|
|
8469
|
+
async function executeCsvsql(runtime2) {
|
|
8470
|
+
const o = runtime2.options, context = runtime2.context;
|
|
8471
8471
|
const paths = o.input_paths;
|
|
8472
|
-
if (context.terminal.stdinIsTTY && paths.length === 1 && paths[0] === "-")
|
|
8472
|
+
if (context.terminal.stdinIsTTY && paths.length === 1 && paths[0] === "-") runtime2.error("You must provide an input file or piped data.");
|
|
8473
8473
|
const querying = Array.isArray(o.queries) && o.queries.length > 0;
|
|
8474
8474
|
const connection = o.connection_string || (querying ? "sqlite:///:memory:" : null);
|
|
8475
8475
|
const insert = o.insert || Boolean(querying && !o.connection_string);
|
|
8476
|
-
if (o.dialect && connection)
|
|
8477
|
-
if (insert && !connection)
|
|
8476
|
+
if (o.dialect && connection) runtime2.error("The --dialect option is only valid when neither --db nor --query are specified.");
|
|
8477
|
+
if (insert && !connection) runtime2.error("The --insert option is only valid when either --db or --query is specified.");
|
|
8478
8478
|
for (const [option, label] of [["no_create", "no-create"], ["create_if_not_exists", "create-if-not-exists"], ["overwrite", "overwrite"]]) {
|
|
8479
|
-
if (o[option] && !insert)
|
|
8479
|
+
if (o[option] && !insert) runtime2.error(`The --${label} option is only valid if --insert is also specified.`);
|
|
8480
8480
|
}
|
|
8481
|
-
if (o.overwrite && o.no_create)
|
|
8481
|
+
if (o.overwrite && o.no_create) runtime2.error("The --overwrite option is only valid if --no-create is not specified.");
|
|
8482
8482
|
for (const [option, label] of [["before_insert", "before-insert"], ["after_insert", "after-insert"], ["chunk_size", "chunk-size"]]) {
|
|
8483
|
-
if (o[option] && !insert)
|
|
8483
|
+
if (o[option] && !insert) runtime2.error(`The --${label} option is only valid if --insert is also specified.`);
|
|
8484
8484
|
}
|
|
8485
|
-
if (o.no_create && o.create_if_not_exists)
|
|
8485
|
+
if (o.no_create && o.create_if_not_exists) runtime2.error("The --no-create and --create-if-not-exists options are mutually exclusive.");
|
|
8486
8486
|
const names = o.table_names ? String(o.table_names).split(",") : [];
|
|
8487
8487
|
const resources = new ResourceScope(context.signal, (cleanup) => context.registerCleanup(cleanup, "invocation"));
|
|
8488
8488
|
let committed = false, succeeded = false, failure;
|
|
@@ -8490,9 +8490,9 @@ async function executeCsvsql(runtime) {
|
|
|
8490
8490
|
let session;
|
|
8491
8491
|
let dialect = databases.find((binding) => binding.name === o.dialect) ?? databases.find((binding) => binding.default);
|
|
8492
8492
|
if (connection) {
|
|
8493
|
-
|
|
8493
|
+
runtime2.sideEffects = true;
|
|
8494
8494
|
const url = String(connection);
|
|
8495
|
-
const engineOptions = sqlOptions(o.engine_option,
|
|
8495
|
+
const engineOptions = sqlOptions(o.engine_option, runtime2.step.bind(runtime2));
|
|
8496
8496
|
const { provider, url: parsedUrl } = resolveDatabaseProvider(url, context.databases);
|
|
8497
8497
|
session = await resources.acquire(() => provider.connect(url, engineOptions, context.signal, { cwd: context.cwd }), async (session2) => {
|
|
8498
8498
|
const errors = [];
|
|
@@ -8519,30 +8519,30 @@ async function executeCsvsql(runtime) {
|
|
|
8519
8519
|
const delimiter = String(o.sql_delimiter);
|
|
8520
8520
|
const split = (sql) => {
|
|
8521
8521
|
if (!delimiter) throw new CsvkitDiagnostic("ValueError: empty separator");
|
|
8522
|
-
|
|
8522
|
+
runtime2.retain(sql.length * 4 + 64);
|
|
8523
8523
|
return sql.split(delimiter);
|
|
8524
8524
|
};
|
|
8525
8525
|
const execute2 = async (sql, values = []) => {
|
|
8526
|
-
|
|
8527
|
-
|
|
8526
|
+
runtime2.step();
|
|
8527
|
+
runtime2.retain(256 + values.length * 16);
|
|
8528
8528
|
let closing;
|
|
8529
8529
|
const result = await resources.acquire(() => session.query(sql, values, {}, context.signal), (result2) => closing ??= result2.close());
|
|
8530
8530
|
return { columns: result.columns, rows: result.rows, close: () => closing ??= result.close() };
|
|
8531
8531
|
};
|
|
8532
8532
|
for (const [index, path] of paths.entries()) {
|
|
8533
|
-
|
|
8533
|
+
runtime2.step();
|
|
8534
8534
|
const basename = path.slice(path.lastIndexOf("/") + 1), period = basename.lastIndexOf(".");
|
|
8535
8535
|
const fileName = period > 0 && [...basename.slice(0, period)].some((char) => char !== ".") ? basename.slice(0, period) : basename;
|
|
8536
8536
|
const name = names[index] ?? (!path || path === "-" ? "stdin" : fileName);
|
|
8537
|
-
const table = await readTable(
|
|
8537
|
+
const table = await readTable(runtime2, path, void 0, true);
|
|
8538
8538
|
if (!table.rows.length) continue;
|
|
8539
8539
|
if (!dialect) throw new CsvkitBlocked("database DDL dialect metadata");
|
|
8540
8540
|
if (!session) {
|
|
8541
|
-
await
|
|
8541
|
+
await runtime2.write(compileCreateTable(runtime2, deriveSchema(runtime2, table, name, dialect, false), dialect).statement.trim() + ";\n");
|
|
8542
8542
|
continue;
|
|
8543
8543
|
}
|
|
8544
8544
|
if (o.before_insert) for (const sql of split(String(o.before_insert))) await (await execute2(sql)).close();
|
|
8545
|
-
const qualified = () => (o.db_schema ? schemaIdentifier(String(o.db_schema), dialect,
|
|
8545
|
+
const qualified = () => (o.db_schema ? schemaIdentifier(String(o.db_schema), dialect, runtime2) + "." : "") + identifier(name, dialect, runtime2);
|
|
8546
8546
|
if (o.unique_constraint) {
|
|
8547
8547
|
const missing = String(o.unique_constraint).split(",").find((column) => !table.headers.includes(column));
|
|
8548
8548
|
if (missing !== void 0) throw new CsvkitDiagnostic(`ConstraintColumnNotFoundError: Can't create UniqueConstraint on table '${name}': no column named '${missing}' is present.`);
|
|
@@ -8552,7 +8552,7 @@ async function executeCsvsql(runtime) {
|
|
|
8552
8552
|
if (!session.hasTable) throw new CsvkitBlocked("database table-existence capability");
|
|
8553
8553
|
const exists = await resources.acquire(() => session.hasTable(name, o.db_schema ? String(o.db_schema) : null, context.signal), async () => {
|
|
8554
8554
|
});
|
|
8555
|
-
|
|
8555
|
+
runtime2.step();
|
|
8556
8556
|
if (exists) await (await execute2("DROP TABLE " + qualified())).close();
|
|
8557
8557
|
}
|
|
8558
8558
|
let create = true;
|
|
@@ -8560,9 +8560,9 @@ async function executeCsvsql(runtime) {
|
|
|
8560
8560
|
if (!session.hasTable) throw new CsvkitBlocked("database table-existence capability");
|
|
8561
8561
|
create = !await resources.acquire(() => session.hasTable(name, o.db_schema ? String(o.db_schema) : null, context.signal), async () => {
|
|
8562
8562
|
});
|
|
8563
|
-
|
|
8563
|
+
runtime2.step();
|
|
8564
8564
|
}
|
|
8565
|
-
if (create) await (await execute2(compileCreateTable(
|
|
8565
|
+
if (create) await (await execute2(compileCreateTable(runtime2, deriveSchema(runtime2, table, name, dialect, true), dialect).statement)).close();
|
|
8566
8566
|
}
|
|
8567
8567
|
if (insert) {
|
|
8568
8568
|
if (o.chunk_size === 0 || o.chunk_size === 0n) throw new CsvkitDiagnostic("ZeroDivisionError: division by zero");
|
|
@@ -8572,7 +8572,7 @@ async function executeCsvsql(runtime) {
|
|
|
8572
8572
|
const batchCount = numerator / chunkSize - (chunkSize < 0n && numerator % chunkSize !== 0n ? 1n : 0n) + 1n;
|
|
8573
8573
|
const prefixes = o.prefix.join(" ");
|
|
8574
8574
|
for (let batch = 0n; batch < batchCount; batch++) {
|
|
8575
|
-
|
|
8575
|
+
runtime2.step();
|
|
8576
8576
|
const bind = dialect.bindValue;
|
|
8577
8577
|
if (!bind || !(dialect.placeholder || dialect.parameter)) throw new CsvkitBlocked(`database insert driver profile ${dialect.name}`);
|
|
8578
8578
|
const start = batch * chunkSize;
|
|
@@ -8583,8 +8583,8 @@ async function executeCsvsql(runtime) {
|
|
|
8583
8583
|
await (await execute2("INSERT " + (prefixes ? prefixes + " " : "") + "INTO " + qualified() + " DEFAULT VALUES")).close();
|
|
8584
8584
|
continue;
|
|
8585
8585
|
}
|
|
8586
|
-
const sql = "INSERT " + (prefixes ? prefixes + " " : "") + "INTO " + qualified() + " (" + table.headers.map((header) => identifier(header, dialect,
|
|
8587
|
-
|
|
8586
|
+
const sql = "INSERT " + (prefixes ? prefixes + " " : "") + "INTO " + qualified() + " (" + table.headers.map((header) => identifier(header, dialect, runtime2)).join(", ") + ") VALUES (" + table.headers.map((_, index2) => dialect.parameter?.(index2) ?? dialect.placeholder).join(", ") + ")";
|
|
8587
|
+
runtime2.retain(rows.length * (32 + table.headers.length * 16));
|
|
8588
8588
|
if (session.executeMany) {
|
|
8589
8589
|
let closing;
|
|
8590
8590
|
const result = await resources.acquire(() => session.executeMany(sql, rows, context.signal), (result2) => closing ??= result2.close());
|
|
@@ -8598,15 +8598,15 @@ async function executeCsvsql(runtime) {
|
|
|
8598
8598
|
if (querying) {
|
|
8599
8599
|
const queries = [];
|
|
8600
8600
|
for (const query2 of o.queries) {
|
|
8601
|
-
|
|
8601
|
+
runtime2.step();
|
|
8602
8602
|
let text2 = query2;
|
|
8603
8603
|
const exists = context.fs.exists ? await context.fs.exists(virtualPath(context.cwd, query2), { signal: context.signal }) : void 0;
|
|
8604
|
-
|
|
8604
|
+
runtime2.step();
|
|
8605
8605
|
if (exists !== false) try {
|
|
8606
|
-
const bytes = await
|
|
8607
|
-
|
|
8606
|
+
const bytes = await runtime2.readFileBytes(query2);
|
|
8607
|
+
runtime2.step();
|
|
8608
8608
|
if (bytes.byteLength > context.limits.maxInputBytes) throw new CsvkitBlocked("query file input byte budget exceeded");
|
|
8609
|
-
|
|
8609
|
+
runtime2.retain(bytes.byteLength * 4 + 64);
|
|
8610
8610
|
const codec = context.codecs.find((binding) => binding.names.includes("utf-8"));
|
|
8611
8611
|
if (!codec) throw new CsvkitBlocked("query file UTF-8 codec");
|
|
8612
8612
|
text2 = (await codec.decode(bytes, "utf-8", context.signal)).replaceAll("\r\n", "\n").replaceAll("\r", "\n");
|
|
@@ -8623,7 +8623,7 @@ async function executeCsvsql(runtime) {
|
|
|
8623
8623
|
}
|
|
8624
8624
|
if (!last) throw new CsvkitDiagnostic("AttributeError: 'NoneType' object has no attribute 'returns_rows'");
|
|
8625
8625
|
if (last.columns !== null) {
|
|
8626
|
-
await
|
|
8626
|
+
await runtime2.row(last.columns);
|
|
8627
8627
|
let reading;
|
|
8628
8628
|
let returning;
|
|
8629
8629
|
const releaseIterator = (iterator2) => returning ??= (async () => {
|
|
@@ -8634,24 +8634,24 @@ async function executeCsvsql(runtime) {
|
|
|
8634
8634
|
const iterator = await resources.acquire(async () => last.rows[Symbol.asyncIterator](), releaseIterator);
|
|
8635
8635
|
let count = 0;
|
|
8636
8636
|
while (true) {
|
|
8637
|
-
|
|
8637
|
+
runtime2.step();
|
|
8638
8638
|
reading = Promise.resolve().then(() => iterator.next());
|
|
8639
8639
|
const next = await reading;
|
|
8640
8640
|
reading = void 0;
|
|
8641
|
-
|
|
8641
|
+
runtime2.step();
|
|
8642
8642
|
if (next.done) break;
|
|
8643
8643
|
if (++count > context.limits.maxDatabaseResultRows) throw new CsvkitBlocked("database result row budget exceeded");
|
|
8644
8644
|
const row = next.value.map((value) => {
|
|
8645
8645
|
if (value instanceof Uint8Array) return bytesRepr(value);
|
|
8646
8646
|
return value;
|
|
8647
8647
|
});
|
|
8648
|
-
await
|
|
8648
|
+
await runtime2.row(row);
|
|
8649
8649
|
}
|
|
8650
8650
|
await releaseIterator(iterator);
|
|
8651
8651
|
}
|
|
8652
8652
|
await last.close();
|
|
8653
8653
|
}
|
|
8654
|
-
|
|
8654
|
+
runtime2.step();
|
|
8655
8655
|
await resources.acquire(async () => {
|
|
8656
8656
|
await session.commit(context.signal);
|
|
8657
8657
|
committed = true;
|
|
@@ -8674,14 +8674,14 @@ async function executeCsvsql(runtime) {
|
|
|
8674
8674
|
|
|
8675
8675
|
// packages/safe-bash-command-csvkit/src/commands/csvstack.ts
|
|
8676
8676
|
init_buffer();
|
|
8677
|
-
async function stack(
|
|
8678
|
-
const o =
|
|
8677
|
+
async function stack(runtime2) {
|
|
8678
|
+
const o = runtime2.options;
|
|
8679
8679
|
const paths = o.input_paths;
|
|
8680
|
-
if (
|
|
8681
|
-
await
|
|
8680
|
+
if (runtime2.context.terminal.stdinIsTTY && paths.length === 1 && paths[0] === "-")
|
|
8681
|
+
await runtime2.write("No input file or piped data provided. Waiting for standard input:\n", "stderr");
|
|
8682
8682
|
const groups = o.groups === null || o.group_by_filenames ? void 0 : String(o.groups).split(",");
|
|
8683
8683
|
if (groups && groups.length !== paths.length)
|
|
8684
|
-
|
|
8684
|
+
runtime2.error("The number of grouping values must be equal to the number of CSV files being stacked.");
|
|
8685
8685
|
const grouped = o.groups !== null || Boolean(o.group_by_filenames);
|
|
8686
8686
|
const groupName = String(o.group_name || "group");
|
|
8687
8687
|
const useFieldnames = !o.no_header_row;
|
|
@@ -8690,7 +8690,7 @@ async function stack(runtime) {
|
|
|
8690
8690
|
let stdinHeaders = [];
|
|
8691
8691
|
let stdinFirst = [];
|
|
8692
8692
|
for (const path of paths) {
|
|
8693
|
-
const reader =
|
|
8693
|
+
const reader = runtime2.records(path);
|
|
8694
8694
|
let first;
|
|
8695
8695
|
try {
|
|
8696
8696
|
first = await reader.next();
|
|
@@ -8700,9 +8700,9 @@ async function stack(runtime) {
|
|
|
8700
8700
|
const row = first.done ? [] : first.value.cells;
|
|
8701
8701
|
const names = useFieldnames ? row : defaultHeaders(row.length);
|
|
8702
8702
|
for (const name of names) {
|
|
8703
|
-
|
|
8703
|
+
runtime2.step();
|
|
8704
8704
|
if (!seen.has(name)) {
|
|
8705
|
-
|
|
8705
|
+
runtime2.retain(64 + name.length * 2);
|
|
8706
8706
|
seen.add(name);
|
|
8707
8707
|
headers.push(name);
|
|
8708
8708
|
}
|
|
@@ -8714,22 +8714,22 @@ async function stack(runtime) {
|
|
|
8714
8714
|
if (!useFieldnames) break;
|
|
8715
8715
|
}
|
|
8716
8716
|
if (grouped) headers.unshift(groupName);
|
|
8717
|
-
if (headers.length + (o.line_numbers ? 1 : 0) >
|
|
8717
|
+
if (headers.length + (o.line_numbers ? 1 : 0) > runtime2.context.limits.maxColumns)
|
|
8718
8718
|
throw new CsvkitBlocked("output column budget exceeded");
|
|
8719
8719
|
const output = useFieldnames ? new DictionaryWriter(headers, { lineNumbers: Boolean(o.line_numbers) }) : void 0;
|
|
8720
|
-
if (output) await
|
|
8721
|
-
else await
|
|
8720
|
+
if (output) await runtime2.write(output.writeheader());
|
|
8721
|
+
else await runtime2.row(headers);
|
|
8722
8722
|
for (const [index, path] of paths.entries()) {
|
|
8723
|
-
const file =
|
|
8723
|
+
const file = runtime2.input(path, true);
|
|
8724
8724
|
const group = groups?.[index] ?? file.name.split("/").at(-1);
|
|
8725
8725
|
let names = path === "-" ? stdinHeaders : void 0;
|
|
8726
8726
|
try {
|
|
8727
|
-
if (path === "-" && stdinFirst.length) await
|
|
8728
|
-
for await (const record of
|
|
8727
|
+
if (path === "-" && stdinFirst.length) await runtime2.row(stdinFirst);
|
|
8728
|
+
for await (const record of runtime2.records(path, file, path === "-" ? 0 : Number(o.skip_lines ?? 0))) {
|
|
8729
8729
|
const cells = record.cells;
|
|
8730
|
-
|
|
8730
|
+
runtime2.step();
|
|
8731
8731
|
if (!useFieldnames) {
|
|
8732
|
-
await
|
|
8732
|
+
await runtime2.row(grouped ? [group, ...cells] : cells);
|
|
8733
8733
|
continue;
|
|
8734
8734
|
}
|
|
8735
8735
|
if (names === void 0) {
|
|
@@ -8741,11 +8741,11 @@ async function stack(runtime) {
|
|
|
8741
8741
|
throw new CsvkitDiagnostic("ValueError: dict contains fields not in fieldnames: None");
|
|
8742
8742
|
const row = /* @__PURE__ */ Object.create(null);
|
|
8743
8743
|
for (const [position, name] of names.entries()) {
|
|
8744
|
-
|
|
8744
|
+
runtime2.step();
|
|
8745
8745
|
row[name] = cells[position] ?? null;
|
|
8746
8746
|
}
|
|
8747
8747
|
if (grouped) row[groupName] = group;
|
|
8748
|
-
await
|
|
8748
|
+
await runtime2.write(output.writerow(row));
|
|
8749
8749
|
}
|
|
8750
8750
|
} finally {
|
|
8751
8751
|
await file.close();
|
|
@@ -9164,7 +9164,7 @@ var CalculationError = class extends Error {
|
|
|
9164
9164
|
function text(value) {
|
|
9165
9165
|
return value === null ? "None" : pythonValueText(value);
|
|
9166
9166
|
}
|
|
9167
|
-
function sqrt(value,
|
|
9167
|
+
function sqrt(value, runtime2) {
|
|
9168
9168
|
if (value.special === "NaN" || value.special === "Infinity" && !value.negative) return value;
|
|
9169
9169
|
if (value.negative && value.coefficient !== 0n || value.special) throw new DecimalTrap("InvalidOperation");
|
|
9170
9170
|
if (value.coefficient === 0n) return Decimal.parse((value.negative ? "-" : "") + "0e" + Math.floor(value.exponent / 2));
|
|
@@ -9172,7 +9172,7 @@ function sqrt(value, runtime) {
|
|
|
9172
9172
|
const radicand = value.coefficient * 10n ** BigInt(value.exponent - 2 * exponent);
|
|
9173
9173
|
let root = 10n ** BigInt(Math.ceil(radicand.toString().length / 2));
|
|
9174
9174
|
while (true) {
|
|
9175
|
-
|
|
9175
|
+
runtime2.step();
|
|
9176
9176
|
const next = (root + radicand / root) / 2n;
|
|
9177
9177
|
if (next >= root) break;
|
|
9178
9178
|
root = next;
|
|
@@ -9181,14 +9181,14 @@ function sqrt(value, runtime) {
|
|
|
9181
9181
|
if (4n * radicand > midpoint || 4n * radicand === midpoint && root % 2n !== 0n) root++;
|
|
9182
9182
|
let scale = exponent;
|
|
9183
9183
|
if (root * root === radicand) while (root % 10n === 0n && scale < Math.floor(value.exponent / 2)) {
|
|
9184
|
-
|
|
9184
|
+
runtime2.step();
|
|
9185
9185
|
root /= 10n;
|
|
9186
9186
|
scale++;
|
|
9187
9187
|
}
|
|
9188
9188
|
return Decimal.parse(root + "e" + scale).multiply(Decimal.parse("1"));
|
|
9189
9189
|
}
|
|
9190
|
-
function compare(a, b,
|
|
9191
|
-
|
|
9190
|
+
function compare(a, b, runtime2) {
|
|
9191
|
+
runtime2.step();
|
|
9192
9192
|
if (typeof a !== "object" || !a || typeof b !== "object" || !b) throw new CalculationError("inapplicable ordering");
|
|
9193
9193
|
if (a.kind === "decimal" && b.kind === "decimal") return Decimal.parse(a.value).compare(Decimal.parse(b.value));
|
|
9194
9194
|
if (a.kind === "timedelta" && b.kind === "timedelta") return a.microseconds < b.microseconds ? -1 : a.microseconds > b.microseconds ? 1 : 0;
|
|
@@ -9201,23 +9201,23 @@ function compare(a, b, runtime) {
|
|
|
9201
9201
|
}
|
|
9202
9202
|
throw new CalculationError("inapplicable ordering");
|
|
9203
9203
|
}
|
|
9204
|
-
function format2(value,
|
|
9204
|
+
function format2(value, runtime2) {
|
|
9205
9205
|
if (typeof value !== "object" || !value || value.kind !== "decimal" || Decimal.parse(value.value).special) return value;
|
|
9206
|
-
let result =
|
|
9206
|
+
let result = runtime2.context.locale.formatNumber(value.value, runtime2.context.locale.profile, String(runtime2.options.decimal_format), !runtime2.options.no_grouping_separator);
|
|
9207
9207
|
while (result.endsWith("0")) {
|
|
9208
|
-
|
|
9208
|
+
runtime2.step();
|
|
9209
9209
|
result = result.slice(0, -1);
|
|
9210
9210
|
}
|
|
9211
9211
|
while (result.endsWith(".")) {
|
|
9212
|
-
|
|
9212
|
+
runtime2.step();
|
|
9213
9213
|
result = result.slice(0, -1);
|
|
9214
9214
|
}
|
|
9215
9215
|
return result;
|
|
9216
9216
|
}
|
|
9217
|
-
function frequencies(values,
|
|
9217
|
+
function frequencies(values, runtime2) {
|
|
9218
9218
|
const counts = /* @__PURE__ */ new Map();
|
|
9219
9219
|
for (const [index, value] of values.entries()) {
|
|
9220
|
-
|
|
9220
|
+
runtime2.step();
|
|
9221
9221
|
let key;
|
|
9222
9222
|
if (typeof value === "object" && value) {
|
|
9223
9223
|
if (value.kind === "decimal") {
|
|
@@ -9232,16 +9232,16 @@ function frequencies(values, runtime) {
|
|
|
9232
9232
|
const existing = counts.get(key);
|
|
9233
9233
|
if (existing) existing.count++;
|
|
9234
9234
|
else {
|
|
9235
|
-
|
|
9235
|
+
runtime2.retain(96 + key.length * 2);
|
|
9236
9236
|
counts.set(key, { value, count: 1 });
|
|
9237
9237
|
}
|
|
9238
9238
|
}
|
|
9239
9239
|
return [...counts.values()];
|
|
9240
9240
|
}
|
|
9241
|
-
function calculate(op, type, values, distinct,
|
|
9242
|
-
|
|
9241
|
+
function calculate(op, type, values, distinct, runtime2) {
|
|
9242
|
+
runtime2.retain(64 + values.length * 16);
|
|
9243
9243
|
const data = values.filter((value) => {
|
|
9244
|
-
|
|
9244
|
+
runtime2.step();
|
|
9245
9245
|
return value !== null;
|
|
9246
9246
|
});
|
|
9247
9247
|
if (op === "type") return type;
|
|
@@ -9249,9 +9249,9 @@ function calculate(op, type, values, distinct, runtime) {
|
|
|
9249
9249
|
if (op === "nonnulls") return data.length;
|
|
9250
9250
|
if (op === "unique") return distinct.length;
|
|
9251
9251
|
if (op === "freq") {
|
|
9252
|
-
const count = Number(
|
|
9252
|
+
const count = Number(runtime2.options.freq_count || 5);
|
|
9253
9253
|
return [...distinct].sort((a, b) => {
|
|
9254
|
-
|
|
9254
|
+
runtime2.step();
|
|
9255
9255
|
return b.count - a.count;
|
|
9256
9256
|
}).slice(0, Math.max(0, count));
|
|
9257
9257
|
}
|
|
@@ -9262,7 +9262,7 @@ function calculate(op, type, values, distinct, runtime) {
|
|
|
9262
9262
|
for (const value of data) {
|
|
9263
9263
|
let size = 0;
|
|
9264
9264
|
for (const ignoredChar of String(value)) {
|
|
9265
|
-
|
|
9265
|
+
runtime2.step();
|
|
9266
9266
|
size++;
|
|
9267
9267
|
}
|
|
9268
9268
|
length = Math.max(length, size);
|
|
@@ -9272,7 +9272,7 @@ function calculate(op, type, values, distinct, runtime) {
|
|
|
9272
9272
|
let whole = 1;
|
|
9273
9273
|
let places = 0;
|
|
9274
9274
|
for (const value of data) {
|
|
9275
|
-
|
|
9275
|
+
runtime2.step();
|
|
9276
9276
|
const spelling = value.value;
|
|
9277
9277
|
if (!Number.isFinite(Number(spelling))) continue;
|
|
9278
9278
|
const decimal = Decimal.parse(spelling).normalized();
|
|
@@ -9282,11 +9282,11 @@ function calculate(op, type, values, distinct, runtime) {
|
|
|
9282
9282
|
}
|
|
9283
9283
|
return Math.min(places, 28 - whole);
|
|
9284
9284
|
} else if ((op === "min" || op === "max") && ["Number", "Date", "DateTime", "TimeDelta"].includes(type)) {
|
|
9285
|
-
for (const value of data) if (result === null || (op === "min" ? compare(value, result,
|
|
9285
|
+
for (const value of data) if (result === null || (op === "min" ? compare(value, result, runtime2) < 0 : compare(value, result, runtime2) > 0)) result = value;
|
|
9286
9286
|
} else if ((op === "sum" || op === "mean") && type === "TimeDelta") {
|
|
9287
9287
|
let total = 0n;
|
|
9288
9288
|
for (const value of data) {
|
|
9289
|
-
|
|
9289
|
+
runtime2.step();
|
|
9290
9290
|
total += value.microseconds;
|
|
9291
9291
|
pythonValueText({ kind: "timedelta", microseconds: total });
|
|
9292
9292
|
}
|
|
@@ -9303,23 +9303,23 @@ function calculate(op, type, values, distinct, runtime) {
|
|
|
9303
9303
|
result = { kind: "timedelta", microseconds: total };
|
|
9304
9304
|
} else if (type === "Number" && ["sum", "mean", "median", "stdev"].includes(op)) {
|
|
9305
9305
|
const numbers = data.map((value) => {
|
|
9306
|
-
|
|
9306
|
+
runtime2.step();
|
|
9307
9307
|
return Decimal.parse(value.value);
|
|
9308
9308
|
});
|
|
9309
9309
|
let decimal;
|
|
9310
9310
|
if (op === "median") {
|
|
9311
|
-
|
|
9312
|
-
const median = decimalPercentiles(numbers,
|
|
9311
|
+
runtime2.retain(4096 + numbers.length * 16);
|
|
9312
|
+
const median = decimalPercentiles(numbers, runtime2.step)[50];
|
|
9313
9313
|
if (!median) return null;
|
|
9314
9314
|
decimal = median;
|
|
9315
9315
|
} else if (op === "stdev") {
|
|
9316
|
-
const variance = sampleVariance(numbers,
|
|
9316
|
+
const variance = sampleVariance(numbers, runtime2.step);
|
|
9317
9317
|
if (!variance) return null;
|
|
9318
|
-
decimal = sqrt(variance,
|
|
9318
|
+
decimal = sqrt(variance, runtime2);
|
|
9319
9319
|
} else {
|
|
9320
9320
|
decimal = Decimal.parse("0");
|
|
9321
9321
|
for (const number2 of numbers) {
|
|
9322
|
-
|
|
9322
|
+
runtime2.step();
|
|
9323
9323
|
decimal = decimal.add(number2);
|
|
9324
9324
|
}
|
|
9325
9325
|
if (op !== "sum") {
|
|
@@ -9329,57 +9329,57 @@ function calculate(op, type, values, distinct, runtime) {
|
|
|
9329
9329
|
}
|
|
9330
9330
|
result = { kind: "decimal", value: decimal.toString() };
|
|
9331
9331
|
}
|
|
9332
|
-
return
|
|
9332
|
+
return runtime2.options.json_output ? result : format2(result, runtime2);
|
|
9333
9333
|
} catch (failure) {
|
|
9334
|
-
|
|
9334
|
+
runtime2.step();
|
|
9335
9335
|
if (failure instanceof CsvkitBlocked) throw failure;
|
|
9336
9336
|
if (failure instanceof DecimalTrap || failure instanceof CsvkitDiagnostic || failure instanceof CalculationError) return null;
|
|
9337
9337
|
throw failure;
|
|
9338
9338
|
}
|
|
9339
9339
|
}
|
|
9340
|
-
async function statistics(
|
|
9341
|
-
const o =
|
|
9342
|
-
if (o.names_only) return printNames(
|
|
9343
|
-
if (
|
|
9340
|
+
async function statistics(runtime2) {
|
|
9341
|
+
const o = runtime2.options;
|
|
9342
|
+
if (o.names_only) return printNames(runtime2);
|
|
9343
|
+
if (runtime2.context.terminal.stdinIsTTY && !o.input_path) runtime2.error("You must provide an input file or piped data.");
|
|
9344
9344
|
const requested = operations.filter(([op]) => o[op + "_only"]);
|
|
9345
|
-
if (requested.length > 1)
|
|
9345
|
+
if (requested.length > 1) runtime2.error("Only one operation argument may be specified (--mean, --median, etc).");
|
|
9346
9346
|
for (const [option, label] of [["csv_output", "csv"], ["json_output", "json"], ["count_only", "count"]]) {
|
|
9347
|
-
if (requested.length && o[option])
|
|
9347
|
+
if (requested.length && o[option]) runtime2.error(`You may not specify --${label} and an operation (--mean, --median, etc) at the same time.`);
|
|
9348
9348
|
}
|
|
9349
9349
|
if (o.count_only) {
|
|
9350
9350
|
let count = o.no_header_row ? 0 : -1;
|
|
9351
|
-
for await (const ignoredRecord of
|
|
9352
|
-
await
|
|
9351
|
+
for await (const ignoredRecord of runtime2.records()) count++;
|
|
9352
|
+
await runtime2.write(`${count}
|
|
9353
9353
|
`);
|
|
9354
9354
|
return 0;
|
|
9355
9355
|
}
|
|
9356
|
-
const table = await readTable(
|
|
9357
|
-
const ids = parseColumnIdentifiers(o.columns, table.headers, o.zero_based ? 0 : 1, void 0,
|
|
9356
|
+
const table = await readTable(runtime2, void 0, void 0, true);
|
|
9357
|
+
const ids = parseColumnIdentifiers(o.columns, table.headers, o.zero_based ? 0 : 1, void 0, runtime2.step, true);
|
|
9358
9358
|
const stats = /* @__PURE__ */ new Map();
|
|
9359
9359
|
for (const id of ids) {
|
|
9360
|
-
|
|
9360
|
+
runtime2.retain(128 + table.rows.length * 32);
|
|
9361
9361
|
const values = table.rows.map((row) => {
|
|
9362
|
-
|
|
9362
|
+
runtime2.step();
|
|
9363
9363
|
return row[id];
|
|
9364
9364
|
});
|
|
9365
|
-
const distinct = frequencies(values,
|
|
9365
|
+
const distinct = frequencies(values, runtime2);
|
|
9366
9366
|
const column = /* @__PURE__ */ new Map();
|
|
9367
9367
|
for (const [op] of requested.length ? requested : operations) {
|
|
9368
|
-
|
|
9369
|
-
column.set(op, calculate(op, table.columns[id].type, values, distinct,
|
|
9368
|
+
runtime2.step();
|
|
9369
|
+
column.set(op, calculate(op, table.columns[id].type, values, distinct, runtime2));
|
|
9370
9370
|
}
|
|
9371
9371
|
stats.set(id, column);
|
|
9372
9372
|
if (requested.length) {
|
|
9373
9373
|
const op = requested[0][0];
|
|
9374
9374
|
const stat = column.get(op);
|
|
9375
9375
|
const output = op === "freq" ? "{ " + stat.map((row) => `"${text(row.value)}": ${row.count}`).join(", ") + " }" : text(stat);
|
|
9376
|
-
await
|
|
9376
|
+
await runtime2.write((ids.length === 1 ? "" : `${String(id + 1).padStart(3)}. ${table.headers[id]}: `) + output + "\n");
|
|
9377
9377
|
}
|
|
9378
9378
|
}
|
|
9379
9379
|
if (requested.length) return 0;
|
|
9380
9380
|
if (o.csv_output) {
|
|
9381
|
-
await
|
|
9382
|
-
for (const id of ids) await
|
|
9381
|
+
await runtime2.row(["column_id", "column_name", ...operations.map(([op]) => op)], {}, false);
|
|
9382
|
+
for (const id of ids) await runtime2.row([id + 1, table.headers[id], ...operations.map(([op]) => {
|
|
9383
9383
|
const value = stats.get(id).get(op);
|
|
9384
9384
|
return op === "freq" ? value.map((row) => text(row.value)).join(", ") : value;
|
|
9385
9385
|
})], {}, false);
|
|
@@ -9392,11 +9392,11 @@ async function statistics(runtime) {
|
|
|
9392
9392
|
return value.kind === "datetime" ? value.value.replace(" ", "T") : value.value;
|
|
9393
9393
|
};
|
|
9394
9394
|
const rows = ids.map((id) => {
|
|
9395
|
-
|
|
9396
|
-
|
|
9395
|
+
runtime2.step();
|
|
9396
|
+
runtime2.retain(1024);
|
|
9397
9397
|
const row = /* @__PURE__ */ new Map([["column_id", { token: String(id + 1) }], ["column_name", table.headers[id]]]);
|
|
9398
9398
|
for (const [op, stat] of stats.get(id)) {
|
|
9399
|
-
|
|
9399
|
+
runtime2.step();
|
|
9400
9400
|
if (stat === null) continue;
|
|
9401
9401
|
row.set(op, op === "freq" ? stat.map((item) => /* @__PURE__ */ new Map([["value", jsonValue(item.value)], ["count", { token: String(item.count) }]])) : jsonValue(stat, true));
|
|
9402
9402
|
}
|
|
@@ -9405,13 +9405,13 @@ async function statistics(runtime) {
|
|
|
9405
9405
|
const indent = o.indent === null ? null : Math.max(0, Number(o.indent));
|
|
9406
9406
|
if (indent !== null) {
|
|
9407
9407
|
if (!Number.isSafeInteger(indent)) throw new CsvkitBlocked("JSON indentation budget");
|
|
9408
|
-
|
|
9408
|
+
runtime2.retain(indent * 12);
|
|
9409
9409
|
}
|
|
9410
|
-
await emit(rows,
|
|
9410
|
+
await emit(rows, runtime2, indent);
|
|
9411
9411
|
} else {
|
|
9412
9412
|
const width = Math.max(...operations.map(([, label]) => label.length));
|
|
9413
9413
|
for (const id of ids) {
|
|
9414
|
-
await
|
|
9414
|
+
await runtime2.write(`${String(id + 1).padStart(3)}. "${table.headers[id]}"
|
|
9415
9415
|
|
|
9416
9416
|
`);
|
|
9417
9417
|
for (const [op, label] of operations) {
|
|
@@ -9420,21 +9420,21 @@ async function statistics(runtime) {
|
|
|
9420
9420
|
const padded = label.padEnd(width);
|
|
9421
9421
|
if (op === "freq") {
|
|
9422
9422
|
for (const [index, row] of stat.entries()) {
|
|
9423
|
-
const value = table.columns[id].type === "Number" ? format2(row.value,
|
|
9424
|
-
await
|
|
9423
|
+
const value = table.columns[id].type === "Number" ? format2(row.value, runtime2) : row.value;
|
|
9424
|
+
await runtime2.write(` ${index ? " ".repeat(width) : padded} ${text(value)} (${row.count}x)
|
|
9425
9425
|
`);
|
|
9426
9426
|
}
|
|
9427
9427
|
} else {
|
|
9428
9428
|
let value = text(stat);
|
|
9429
9429
|
if (op === "nulls" && stat) value += " (excluded from calculations)";
|
|
9430
9430
|
if (op === "len") value += " characters";
|
|
9431
|
-
await
|
|
9431
|
+
await runtime2.write(` ${padded} ${value}
|
|
9432
9432
|
`);
|
|
9433
9433
|
}
|
|
9434
9434
|
}
|
|
9435
|
-
await
|
|
9435
|
+
await runtime2.write("\n");
|
|
9436
9436
|
}
|
|
9437
|
-
await
|
|
9437
|
+
await runtime2.write(`Row count: ${table.rows.length}
|
|
9438
9438
|
`);
|
|
9439
9439
|
}
|
|
9440
9440
|
return 0;
|
|
@@ -10213,33 +10213,33 @@ init_buffer();
|
|
|
10213
10213
|
function csvifiedRow(row) {
|
|
10214
10214
|
return row.map((value) => typeof value === "object" && value?.kind === "datetime" ? { ...value, value: value.value.replace(" ", "T") } : value);
|
|
10215
10215
|
}
|
|
10216
|
-
function inputTable(
|
|
10217
|
-
const o =
|
|
10218
|
-
|
|
10216
|
+
function inputTable(runtime2, headers, rows) {
|
|
10217
|
+
const o = runtime2.options;
|
|
10218
|
+
runtime2.retain(64 + rows.length * (64 + headers.length * 64));
|
|
10219
10219
|
return inferTable(headers, rows, {
|
|
10220
10220
|
noInference: Boolean(o.no_inference),
|
|
10221
10221
|
blanks: Boolean(o.blanks),
|
|
10222
10222
|
nullValues: o.null_values ?? [],
|
|
10223
10223
|
noLeadingZeroes: Boolean(o.no_leading_zeroes),
|
|
10224
|
-
now:
|
|
10225
|
-
timezone:
|
|
10226
|
-
maxDecimalDigits:
|
|
10227
|
-
maxDecimalExponent:
|
|
10224
|
+
now: runtime2.context.clock.now(),
|
|
10225
|
+
timezone: runtime2.context.locale.timezone,
|
|
10226
|
+
maxDecimalDigits: runtime2.context.limits.maxDecimalDigits,
|
|
10227
|
+
maxDecimalExponent: runtime2.context.limits.maxDecimalExponent,
|
|
10228
10228
|
...o.locale ? { locale: String(o.locale) } : {},
|
|
10229
10229
|
...o.date_format ? { dateFormat: String(o.date_format) } : {},
|
|
10230
10230
|
...o.datetime_format ? { datetimeFormat: String(o.datetime_format) } : {}
|
|
10231
|
-
},
|
|
10231
|
+
}, runtime2.step);
|
|
10232
10232
|
}
|
|
10233
10233
|
|
|
10234
10234
|
// packages/safe-bash-command-csvkit/src/operations/csv-input.ts
|
|
10235
|
-
async function rawCsvConversion(
|
|
10236
|
-
const o =
|
|
10235
|
+
async function rawCsvConversion(runtime2) {
|
|
10236
|
+
const o = runtime2.options;
|
|
10237
10237
|
if (o.no_inference && !o.no_header_row && !o.skip_lines && o.sniff_limit === 0) {
|
|
10238
|
-
for await (const record of
|
|
10238
|
+
for await (const record of runtime2.records(void 0, runtime2.input(), 0)) await runtime2.row(record.cells);
|
|
10239
10239
|
} else {
|
|
10240
|
-
const table = await readTable(
|
|
10241
|
-
await
|
|
10242
|
-
for (const row of table.rows) await
|
|
10240
|
+
const table = await readTable(runtime2, void 0, void 0, true);
|
|
10241
|
+
await runtime2.row(table.headers);
|
|
10242
|
+
for (const row of table.rows) await runtime2.row(csvifiedRow(row));
|
|
10243
10243
|
}
|
|
10244
10244
|
return 0;
|
|
10245
10245
|
}
|
|
@@ -10709,25 +10709,25 @@ var Runtime = class {
|
|
|
10709
10709
|
};
|
|
10710
10710
|
|
|
10711
10711
|
// packages/safe-bash-command-csvkit/src/operations/dbf-files.ts
|
|
10712
|
-
async function dbfFile(
|
|
10712
|
+
async function dbfFile(runtime2, path, exact = false) {
|
|
10713
10713
|
if ([...path].some((char) => "*?[]".includes(char))) throw new CsvkitBlocked("DBF filename glob patterns");
|
|
10714
|
-
const fs =
|
|
10714
|
+
const fs = runtime2.context.fs;
|
|
10715
10715
|
if (fs.listDirectory) {
|
|
10716
10716
|
const slash = path.lastIndexOf("/");
|
|
10717
10717
|
const directory = slash < 0 ? "." : path.slice(0, slash) || "/";
|
|
10718
10718
|
const name = path.slice(slash + 1);
|
|
10719
10719
|
let names;
|
|
10720
10720
|
try {
|
|
10721
|
-
names = await fs.listDirectory(virtualPath(
|
|
10721
|
+
names = await fs.listDirectory(virtualPath(runtime2.context.cwd, directory), { signal: runtime2.context.signal });
|
|
10722
10722
|
} catch (failure) {
|
|
10723
|
-
|
|
10723
|
+
runtime2.context.signal.throwIfAborted();
|
|
10724
10724
|
if (failure && typeof failure === "object" && "code" in failure && ["ENOENT", "ENOTDIR"].includes(String(failure.code))) return void 0;
|
|
10725
10725
|
throw failure;
|
|
10726
10726
|
}
|
|
10727
|
-
|
|
10728
|
-
|
|
10727
|
+
runtime2.step();
|
|
10728
|
+
runtime2.retain(names.reduce((size2, entry) => size2 + 64 + entry.length * 2, 0));
|
|
10729
10729
|
const found = names.find((entry) => {
|
|
10730
|
-
|
|
10730
|
+
runtime2.step();
|
|
10731
10731
|
return exact ? entry === name : entry.toLowerCase() === name.toLowerCase();
|
|
10732
10732
|
});
|
|
10733
10733
|
if (found === void 0) return void 0;
|
|
@@ -10736,7 +10736,7 @@ async function dbfFile(runtime, path, exact = false) {
|
|
|
10736
10736
|
const chunks = [];
|
|
10737
10737
|
let size = 0;
|
|
10738
10738
|
try {
|
|
10739
|
-
for await (const chunk of
|
|
10739
|
+
for await (const chunk of runtime2.bytes(path)) {
|
|
10740
10740
|
chunks.push(chunk);
|
|
10741
10741
|
size += chunk.length;
|
|
10742
10742
|
}
|
|
@@ -10744,11 +10744,11 @@ async function dbfFile(runtime, path, exact = false) {
|
|
|
10744
10744
|
if (failure instanceof PythonException && failure.exceptionClass === "FileNotFoundError") return void 0;
|
|
10745
10745
|
throw failure;
|
|
10746
10746
|
}
|
|
10747
|
-
|
|
10747
|
+
runtime2.retain(size);
|
|
10748
10748
|
const bytes = new Uint8Array(size);
|
|
10749
10749
|
let offset = 0;
|
|
10750
10750
|
for (const chunk of chunks) {
|
|
10751
|
-
|
|
10751
|
+
runtime2.step();
|
|
10752
10752
|
bytes.set(chunk, offset);
|
|
10753
10753
|
offset += chunk.length;
|
|
10754
10754
|
}
|
|
@@ -10770,24 +10770,24 @@ function bytesRepr2(bytes) {
|
|
|
10770
10770
|
|
|
10771
10771
|
// packages/safe-bash-command-csvkit/src/operations/dbf-memo.ts
|
|
10772
10772
|
init_buffer();
|
|
10773
|
-
async function openDbfMemo(
|
|
10773
|
+
async function openDbfMemo(runtime2, path, version) {
|
|
10774
10774
|
const dot = path.lastIndexOf(".");
|
|
10775
10775
|
const stem = dot > path.lastIndexOf("/") ? path.slice(0, dot) : path;
|
|
10776
|
-
const file = await dbfFile(
|
|
10776
|
+
const file = await dbfFile(runtime2, stem + ".fpt") ?? await dbfFile(runtime2, stem + ".dbt");
|
|
10777
10777
|
if (!file) throw new CsvkitDiagnostic(`MissingMemoFile: missing memo file for ${path}`);
|
|
10778
10778
|
const fpt = file.path.toLowerCase().endsWith(".fpt");
|
|
10779
10779
|
const bytes = file.bytes;
|
|
10780
10780
|
if (fpt && bytes.length < 512) throw new CsvkitDiagnostic("error: unpack requires a buffer of 512 bytes");
|
|
10781
10781
|
const blockSize = fpt ? new DataView(bytes.buffer).getUint16(6, false) : 512;
|
|
10782
10782
|
return (index) => {
|
|
10783
|
-
|
|
10783
|
+
runtime2.step();
|
|
10784
10784
|
if (index <= 0) return null;
|
|
10785
10785
|
const begin = index * blockSize;
|
|
10786
10786
|
if (!Number.isSafeInteger(index) || !Number.isSafeInteger(begin)) throw new CsvkitBlocked("DBF memo offset outside qualified integer range");
|
|
10787
10787
|
if (!fpt && version === 131) {
|
|
10788
10788
|
let end = begin;
|
|
10789
10789
|
while (end < bytes.length && bytes[end] !== 26) {
|
|
10790
|
-
|
|
10790
|
+
runtime2.step();
|
|
10791
10791
|
end++;
|
|
10792
10792
|
}
|
|
10793
10793
|
return { bytes: bytes.subarray(begin, end), binary: false };
|
|
@@ -10809,13 +10809,13 @@ async function openDbfMemo(runtime, path, version) {
|
|
|
10809
10809
|
init_buffer();
|
|
10810
10810
|
var ascii = (bytes) => Array.from(bytes, (byte) => String.fromCharCode(byte)).join("");
|
|
10811
10811
|
var whitespace = (byte) => byte === 32 || byte >= 9 && byte <= 13;
|
|
10812
|
-
async function parseDbfField(field, data, version, decode, memo,
|
|
10813
|
-
|
|
10812
|
+
async function parseDbfField(field, data, version, decode, memo, runtime2) {
|
|
10813
|
+
runtime2.step();
|
|
10814
10814
|
const type = field.type;
|
|
10815
10815
|
if (type === "C" || type === "V") {
|
|
10816
10816
|
let end = data.length;
|
|
10817
10817
|
while (end && (data[end - 1] === 0 || data[end - 1] === 32)) {
|
|
10818
|
-
|
|
10818
|
+
runtime2.step();
|
|
10819
10819
|
end--;
|
|
10820
10820
|
}
|
|
10821
10821
|
return decode(data.subarray(0, end));
|
|
@@ -10829,7 +10829,7 @@ async function parseDbfField(field, data, version, decode, memo, runtime) {
|
|
|
10829
10829
|
while (end > begin && data[end - 1] === 42) end--;
|
|
10830
10830
|
data = data.subarray(begin, end);
|
|
10831
10831
|
const text2 = ascii(data);
|
|
10832
|
-
const int = type === "N" && [...data].every((byte) => byte < 128) ? integer(text2,
|
|
10832
|
+
const int = type === "N" && [...data].every((byte) => byte < 128) ? integer(text2, runtime2.context.limits.maxDecimalDigits) : void 0;
|
|
10833
10833
|
if (int !== void 0) return { kind: "integer", value: String(int) };
|
|
10834
10834
|
if (type === "N" ? [...data].every(whitespace) : !data.length) return null;
|
|
10835
10835
|
try {
|
|
@@ -10901,19 +10901,19 @@ async function parseDbfField(field, data, version, decode, memo, runtime) {
|
|
|
10901
10901
|
|
|
10902
10902
|
// packages/safe-bash-command-csvkit/src/operations/dbf-table.ts
|
|
10903
10903
|
init_buffer();
|
|
10904
|
-
function dbfTable(
|
|
10904
|
+
function dbfTable(runtime2, headers, rows) {
|
|
10905
10905
|
const options = {
|
|
10906
|
-
now:
|
|
10907
|
-
timezone:
|
|
10908
|
-
maxDecimalDigits:
|
|
10909
|
-
maxDecimalExponent:
|
|
10906
|
+
now: runtime2.context.clock.now(),
|
|
10907
|
+
timezone: runtime2.context.locale.timezone,
|
|
10908
|
+
maxDecimalDigits: runtime2.context.limits.maxDecimalDigits,
|
|
10909
|
+
maxDecimalExponent: runtime2.context.limits.maxDecimalExponent
|
|
10910
10910
|
};
|
|
10911
10911
|
const cast = (type, value) => {
|
|
10912
|
-
|
|
10912
|
+
runtime2.step();
|
|
10913
10913
|
if (value === null) return null;
|
|
10914
10914
|
if (typeof value === "string") {
|
|
10915
10915
|
try {
|
|
10916
|
-
return castValue(type, value, options,
|
|
10916
|
+
return castValue(type, value, options, runtime2.step);
|
|
10917
10917
|
} catch (failure) {
|
|
10918
10918
|
if (!(failure instanceof CastError)) throw failure;
|
|
10919
10919
|
return void 0;
|
|
@@ -11105,20 +11105,20 @@ init_buffer();
|
|
|
11105
11105
|
|
|
11106
11106
|
// packages/safe-bash-command-csvkit/src/operations/fixed-input.ts
|
|
11107
11107
|
init_buffer();
|
|
11108
|
-
async function fixedConversion(
|
|
11109
|
-
if (!
|
|
11110
|
-
const input =
|
|
11111
|
-
for (let skipped = 0; skipped < Number(
|
|
11108
|
+
async function fixedConversion(runtime2) {
|
|
11109
|
+
if (!runtime2.options.schema) throw new CsvkitDiagnostic('ValueError: schema must not be null when format is "fixed"');
|
|
11110
|
+
const input = runtime2.input();
|
|
11111
|
+
for (let skipped = 0; skipped < Number(runtime2.options.skip_lines ?? 0); skipped++) {
|
|
11112
11112
|
if (await input.nextLine(false) === null) break;
|
|
11113
11113
|
}
|
|
11114
|
-
const schema =
|
|
11114
|
+
const schema = runtime2.input(String(runtime2.options.schema));
|
|
11115
11115
|
const records = readCsvStream(schema.lines(), {
|
|
11116
|
-
fieldLimit: Number(
|
|
11117
|
-
fieldBudget:
|
|
11118
|
-
},
|
|
11116
|
+
fieldLimit: Number(runtime2.options.field_size_limit ?? Infinity),
|
|
11117
|
+
fieldBudget: runtime2.context.limits.maxFieldCharacters
|
|
11118
|
+
}, runtime2.step);
|
|
11119
11119
|
const first = await records.next();
|
|
11120
11120
|
if (first.done) throw new CsvkitDiagnostic("StopIteration: ");
|
|
11121
|
-
|
|
11121
|
+
runtime2.admitRecord(first.value);
|
|
11122
11122
|
const indexes = ["column", "start", "length"].map((name) => {
|
|
11123
11123
|
const index = first.value.cells.indexOf(name);
|
|
11124
11124
|
if (index < 0) throw new CsvkitDiagnostic(`ValueError: A column named "${name}" must exist in the schema file.`);
|
|
@@ -11128,7 +11128,7 @@ async function fixedConversion(runtime) {
|
|
|
11128
11128
|
let oneBased;
|
|
11129
11129
|
let schemaLine = 1;
|
|
11130
11130
|
for await (const record of records) {
|
|
11131
|
-
|
|
11131
|
+
runtime2.admitRecord(record);
|
|
11132
11132
|
schemaLine++;
|
|
11133
11133
|
const valueAt = (index) => {
|
|
11134
11134
|
const value = record.cells[indexes[index]];
|
|
@@ -11147,20 +11147,20 @@ async function fixedConversion(runtime) {
|
|
|
11147
11147
|
oneBased ??= start === 1n;
|
|
11148
11148
|
const name = valueAt(0);
|
|
11149
11149
|
const length = numberAt(2);
|
|
11150
|
-
if (fields.length >=
|
|
11151
|
-
|
|
11150
|
+
if (fields.length >= runtime2.context.limits.maxColumns) throw new CsvkitBlocked("column budget exceeded");
|
|
11151
|
+
runtime2.retain(64 + name.length * 2);
|
|
11152
11152
|
fields.push({ name, start: start - (oneBased ? 1n : 0n), length });
|
|
11153
11153
|
}
|
|
11154
|
-
await
|
|
11154
|
+
await runtime2.row(fields.map((field) => field.name), {}, false);
|
|
11155
11155
|
let lineNumber = 0;
|
|
11156
11156
|
for await (const line of input.lines()) {
|
|
11157
11157
|
const chars = Array.from(line);
|
|
11158
11158
|
const row = fields.map((field) => {
|
|
11159
|
-
|
|
11159
|
+
runtime2.step();
|
|
11160
11160
|
return stripWhitespace(chars.slice(Number(field.start), Number(field.start + field.length)).join(""));
|
|
11161
11161
|
});
|
|
11162
|
-
|
|
11163
|
-
await
|
|
11162
|
+
runtime2.admitRecord({ cells: row, line: ++lineNumber });
|
|
11163
|
+
await runtime2.row(row, {}, false);
|
|
11164
11164
|
}
|
|
11165
11165
|
return 0;
|
|
11166
11166
|
}
|
|
@@ -11173,15 +11173,15 @@ init_buffer();
|
|
|
11173
11173
|
|
|
11174
11174
|
// packages/safe-bash-command-csvkit/src/operations/geojson-input.ts
|
|
11175
11175
|
init_buffer();
|
|
11176
|
-
function json2(value,
|
|
11177
|
-
|
|
11178
|
-
if (value instanceof Map) return "{" + [...value].map(([key, child2]) => json2(key,
|
|
11179
|
-
if (Array.isArray(value)) return "[" + value.map((child2) => json2(child2,
|
|
11176
|
+
function json2(value, runtime2) {
|
|
11177
|
+
runtime2.step();
|
|
11178
|
+
if (value instanceof Map) return "{" + [...value].map(([key, child2]) => json2(key, runtime2) + ": " + json2(child2, runtime2)).join(", ") + "}";
|
|
11179
|
+
if (Array.isArray(value)) return "[" + value.map((child2) => json2(child2, runtime2)).join(", ") + "]";
|
|
11180
11180
|
if (value !== null && typeof value === "object") return number(value.token, true);
|
|
11181
11181
|
const encoded = JSON.stringify(value);
|
|
11182
11182
|
let ascii2 = "";
|
|
11183
11183
|
for (let index = 0; index < encoded.length; index++) {
|
|
11184
|
-
|
|
11184
|
+
runtime2.step();
|
|
11185
11185
|
const code = encoded.charCodeAt(index);
|
|
11186
11186
|
ascii2 += code >= 127 ? "\\u" + code.toString(16).padStart(4, "0") : encoded[index];
|
|
11187
11187
|
}
|
|
@@ -11193,21 +11193,21 @@ function number(token, dump = false) {
|
|
|
11193
11193
|
if (!dump && !Number.isFinite(value)) return Number.isNaN(value) ? "nan" : value < 0 ? "-inf" : "inf";
|
|
11194
11194
|
return floatText(value);
|
|
11195
11195
|
}
|
|
11196
|
-
function cell(value,
|
|
11196
|
+
function cell(value, runtime2, property = false) {
|
|
11197
11197
|
if (value === void 0 || value === null) return null;
|
|
11198
|
-
if (value instanceof Map) return property ? json2(value,
|
|
11199
|
-
if (Array.isArray(value)) return pythonRepr(value,
|
|
11198
|
+
if (value instanceof Map) return property ? json2(value, runtime2) : pythonRepr(value, runtime2);
|
|
11199
|
+
if (Array.isArray(value)) return pythonRepr(value, runtime2);
|
|
11200
11200
|
if (typeof value === "object") return number(value.token);
|
|
11201
11201
|
if (typeof value === "boolean") return value ? "True" : "False";
|
|
11202
11202
|
return value;
|
|
11203
11203
|
}
|
|
11204
|
-
function pythonRepr(value,
|
|
11205
|
-
|
|
11204
|
+
function pythonRepr(value, runtime2) {
|
|
11205
|
+
runtime2.step();
|
|
11206
11206
|
if (value === null) return "None";
|
|
11207
11207
|
if (typeof value === "string") return repr(value);
|
|
11208
11208
|
if (typeof value === "boolean") return value ? "True" : "False";
|
|
11209
|
-
if (Array.isArray(value)) return "[" + value.map((child2) => pythonRepr(child2,
|
|
11210
|
-
if (value instanceof Map) return value.size ? "OrderedDict({" + [...value].map(([key, child2]) => repr(key) + ": " + pythonRepr(child2,
|
|
11209
|
+
if (Array.isArray(value)) return "[" + value.map((child2) => pythonRepr(child2, runtime2)).join(", ") + "]";
|
|
11210
|
+
if (value instanceof Map) return value.size ? "OrderedDict({" + [...value].map(([key, child2]) => repr(key) + ": " + pythonRepr(child2, runtime2)).join(", ") + "})" : "OrderedDict()";
|
|
11211
11211
|
return number(value.token);
|
|
11212
11212
|
}
|
|
11213
11213
|
function typeName2(value) {
|
|
@@ -11224,28 +11224,28 @@ function truthy(value) {
|
|
|
11224
11224
|
if (Array.isArray(value)) return value.length !== 0;
|
|
11225
11225
|
return typeof value !== "object" || Number(value.token) !== 0;
|
|
11226
11226
|
}
|
|
11227
|
-
async function geojsonConversion(
|
|
11228
|
-
const input = decodeJson(await
|
|
11227
|
+
async function geojsonConversion(runtime2) {
|
|
11228
|
+
const input = decodeJson(await runtime2.text(void 0, 0), runtime2);
|
|
11229
11229
|
if (!(input instanceof Map)) throw new CsvkitDiagnostic("TypeError: JSON document is not valid GeoJSON: Root element is not an object.");
|
|
11230
11230
|
if (!input.has("type")) throw new CsvkitDiagnostic('TypeError: JSON document is not valid GeoJSON: No top-level "type" key.');
|
|
11231
11231
|
const rootType = input.get("type");
|
|
11232
|
-
if (rootType !== "FeatureCollection") throw new CsvkitDiagnostic(`TypeError: Only GeoJSON with root FeatureCollection type is supported. Not ${typeof rootType === "string" ? rootType : pythonRepr(rootType,
|
|
11232
|
+
if (rootType !== "FeatureCollection") throw new CsvkitDiagnostic(`TypeError: Only GeoJSON with root FeatureCollection type is supported. Not ${typeof rootType === "string" ? rootType : pythonRepr(rootType, runtime2)}`);
|
|
11233
11233
|
if (!input.has("features")) throw new CsvkitDiagnostic('TypeError: JSON document is not a valid FeatureCollection: No top-level "features" key.');
|
|
11234
11234
|
const features = input.get("features");
|
|
11235
11235
|
const iterable = Array.isArray(features) ? features : features instanceof Map ? [...features.keys()] : typeof features === "string" ? Array.from(features) : void 0;
|
|
11236
11236
|
if (!iterable) throw new CsvkitDiagnostic(`TypeError: '${typeName2(features)}' object is not iterable`);
|
|
11237
|
-
if (
|
|
11237
|
+
if (runtime2.context.limits.maxColumns < 5) throw new CsvkitBlocked("column budget exceeded");
|
|
11238
11238
|
const properties = [];
|
|
11239
11239
|
const known = /* @__PURE__ */ new Set();
|
|
11240
11240
|
const parsed = [];
|
|
11241
11241
|
for (const feature of iterable) {
|
|
11242
|
-
|
|
11242
|
+
runtime2.step();
|
|
11243
11243
|
if (!(feature instanceof Map)) throw new CsvkitDiagnostic(`AttributeError: '${typeName2(feature)}' object has no attribute 'get'`);
|
|
11244
11244
|
const props = feature.has("properties") ? feature.get("properties") : /* @__PURE__ */ new Map();
|
|
11245
11245
|
if (!(props instanceof Map)) throw new CsvkitDiagnostic(`AttributeError: '${typeName2(props)}' object has no attribute 'keys'`);
|
|
11246
11246
|
for (const key of props.keys()) if (!known.has(key)) {
|
|
11247
|
-
if (properties.length + 5 >=
|
|
11248
|
-
|
|
11247
|
+
if (properties.length + 5 >= runtime2.context.limits.maxColumns) throw new CsvkitBlocked("column budget exceeded");
|
|
11248
|
+
runtime2.retain(64 + key.length * 2);
|
|
11249
11249
|
known.add(key);
|
|
11250
11250
|
properties.push(key);
|
|
11251
11251
|
}
|
|
@@ -11262,19 +11262,19 @@ async function geojsonConversion(runtime) {
|
|
|
11262
11262
|
if (values.length < 2) throw new CsvkitDiagnostic(`ValueError: not enough values to unpack (expected 2, got ${values.length})`);
|
|
11263
11263
|
[lon, lat] = values;
|
|
11264
11264
|
}
|
|
11265
|
-
|
|
11265
|
+
runtime2.retain(128);
|
|
11266
11266
|
parsed.push({ id: feature.get("id"), props, geometry, type, lon, lat });
|
|
11267
11267
|
}
|
|
11268
11268
|
const headers = ["id", ...properties, "geojson", "type", "longitude", "latitude"];
|
|
11269
11269
|
const output = [writeCsvRow(headers)];
|
|
11270
11270
|
for (const feature of parsed) {
|
|
11271
|
-
const cells = [cell(feature.id,
|
|
11272
|
-
|
|
11271
|
+
const cells = [cell(feature.id, runtime2), ...properties.map((key) => cell(feature.props.get(key), runtime2, true)), json2(feature.geometry, runtime2), cell(feature.type, runtime2), cell(feature.lon, runtime2), cell(feature.lat, runtime2)];
|
|
11272
|
+
runtime2.admitRecord({ cells: cells.map((value) => value ?? ""), line: output.length });
|
|
11273
11273
|
const row = writeCsvRow(cells);
|
|
11274
|
-
|
|
11274
|
+
runtime2.retain(row.length * 2);
|
|
11275
11275
|
output.push(row);
|
|
11276
11276
|
}
|
|
11277
|
-
await
|
|
11277
|
+
await runtime2.write(output.join(""));
|
|
11278
11278
|
return 0;
|
|
11279
11279
|
}
|
|
11280
11280
|
|
|
@@ -11289,17 +11289,17 @@ init_buffer();
|
|
|
11289
11289
|
|
|
11290
11290
|
// packages/safe-bash-command-csvkit/src/operations/json-input-table.ts
|
|
11291
11291
|
init_buffer();
|
|
11292
|
-
function jsonInputTable(
|
|
11293
|
-
const o =
|
|
11292
|
+
function jsonInputTable(runtime2, headers, rows) {
|
|
11293
|
+
const o = runtime2.options;
|
|
11294
11294
|
const options = {
|
|
11295
11295
|
noInference: Boolean(o.no_inference),
|
|
11296
11296
|
blanks: Boolean(o.blanks),
|
|
11297
11297
|
nullValues: o.null_values ?? [],
|
|
11298
11298
|
noLeadingZeroes: Boolean(o.no_leading_zeroes),
|
|
11299
|
-
now:
|
|
11300
|
-
timezone:
|
|
11301
|
-
maxDecimalDigits:
|
|
11302
|
-
maxDecimalExponent:
|
|
11299
|
+
now: runtime2.context.clock.now(),
|
|
11300
|
+
timezone: runtime2.context.locale.timezone,
|
|
11301
|
+
maxDecimalDigits: runtime2.context.limits.maxDecimalDigits,
|
|
11302
|
+
maxDecimalExponent: runtime2.context.limits.maxDecimalExponent,
|
|
11303
11303
|
...o.locale ? { locale: String(o.locale) } : {},
|
|
11304
11304
|
...o.date_format ? { dateFormat: String(o.date_format) } : {},
|
|
11305
11305
|
...o.datetime_format ? { datetimeFormat: String(o.datetime_format) } : {}
|
|
@@ -11307,27 +11307,27 @@ function jsonInputTable(runtime, headers, rows) {
|
|
|
11307
11307
|
if (!options.noInference && options.locale && !["en_US", "de_DE"].includes(options.locale))
|
|
11308
11308
|
throw new CsvkitBlocked(`Agate Number locale ${options.locale}`);
|
|
11309
11309
|
const cast = (type, value) => {
|
|
11310
|
-
|
|
11310
|
+
runtime2.step();
|
|
11311
11311
|
if (value === null) return null;
|
|
11312
11312
|
if (typeof value === "string") {
|
|
11313
11313
|
try {
|
|
11314
|
-
return castValue(type, value, options,
|
|
11314
|
+
return castValue(type, value, options, runtime2.step);
|
|
11315
11315
|
} catch (error) {
|
|
11316
11316
|
if (!(error instanceof CastError)) throw error;
|
|
11317
11317
|
return void 0;
|
|
11318
11318
|
}
|
|
11319
11319
|
}
|
|
11320
|
-
if (type === "Text") return typeof value === "boolean" ? value ? "True" : "False" : jsonNumberText(value,
|
|
11321
|
-
if (type === "Number") return { kind: "decimal", value: typeof value === "boolean" ? value ? "1" : "0" : value.token === "NaN" ? "NaN" : value.token === "Infinity" ? "Infinity" : value.token === "-Infinity" ? "-Infinity" : jsonNumberText(value,
|
|
11320
|
+
if (type === "Text") return typeof value === "boolean" ? value ? "True" : "False" : jsonNumberText(value, runtime2);
|
|
11321
|
+
if (type === "Number") return { kind: "decimal", value: typeof value === "boolean" ? value ? "1" : "0" : value.token === "NaN" ? "NaN" : value.token === "Infinity" ? "Infinity" : value.token === "-Infinity" ? "-Infinity" : jsonNumberText(value, runtime2) };
|
|
11322
11322
|
if (type !== "Boolean") return void 0;
|
|
11323
11323
|
if (typeof value === "boolean") return value;
|
|
11324
11324
|
if (["NaN", "Infinity", "-Infinity"].includes(value.token)) return void 0;
|
|
11325
|
-
const decimal = Decimal.parse(jsonNumberText(value,
|
|
11325
|
+
const decimal = Decimal.parse(jsonNumberText(value, runtime2));
|
|
11326
11326
|
if (decimal.coefficient === 0n) return false;
|
|
11327
11327
|
if (!decimal.negative && decimal.exponent <= 0 && decimal.coefficient === 10n ** BigInt(-decimal.exponent)) return true;
|
|
11328
11328
|
return void 0;
|
|
11329
11329
|
};
|
|
11330
|
-
|
|
11330
|
+
runtime2.retain(64 + rows.length * (64 + headers.length * 64));
|
|
11331
11331
|
const order = columnTypeOrder(options);
|
|
11332
11332
|
const castColumns = [];
|
|
11333
11333
|
const columns = headers.map((name, index) => {
|
|
@@ -11353,24 +11353,24 @@ function jsonInputTable(runtime, headers, rows) {
|
|
|
11353
11353
|
}
|
|
11354
11354
|
|
|
11355
11355
|
// packages/safe-bash-command-csvkit/src/operations/json-input.ts
|
|
11356
|
-
async function jsonConversion(
|
|
11357
|
-
const o =
|
|
11356
|
+
async function jsonConversion(runtime2, newline) {
|
|
11357
|
+
const o = runtime2.options;
|
|
11358
11358
|
if (newline && o.key !== null) throw new CsvkitDiagnostic("ValueError: key and newline may not be specified together.");
|
|
11359
|
-
const text2 = await
|
|
11359
|
+
const text2 = await runtime2.text(void 0, 0);
|
|
11360
11360
|
let input;
|
|
11361
11361
|
if (newline) {
|
|
11362
11362
|
const rows2 = [];
|
|
11363
11363
|
let start = 0;
|
|
11364
11364
|
for (let position = 0; position < text2.length; position++) {
|
|
11365
|
-
|
|
11365
|
+
runtime2.step();
|
|
11366
11366
|
if (text2[position] !== "\n") continue;
|
|
11367
|
-
rows2.push(decodeJson(text2.slice(start, position + 1),
|
|
11368
|
-
|
|
11367
|
+
rows2.push(decodeJson(text2.slice(start, position + 1), runtime2));
|
|
11368
|
+
runtime2.retain(16);
|
|
11369
11369
|
start = position + 1;
|
|
11370
11370
|
}
|
|
11371
|
-
if (start < text2.length) rows2.push(decodeJson(text2.slice(start),
|
|
11371
|
+
if (start < text2.length) rows2.push(decodeJson(text2.slice(start), runtime2));
|
|
11372
11372
|
input = rows2;
|
|
11373
|
-
} else input = decodeJson(text2,
|
|
11373
|
+
} else input = decodeJson(text2, runtime2);
|
|
11374
11374
|
if (input instanceof Map) {
|
|
11375
11375
|
if (!o.key) throw new CsvkitDiagnostic("TypeError: When converting a JSON document with a top-level dictionary element, a key must be specified.");
|
|
11376
11376
|
const key = String(o.key);
|
|
@@ -11388,7 +11388,7 @@ async function jsonConversion(runtime, newline) {
|
|
|
11388
11388
|
const known = /* @__PURE__ */ new Set();
|
|
11389
11389
|
const rows = [];
|
|
11390
11390
|
const flatten = (value, path, row) => {
|
|
11391
|
-
|
|
11391
|
+
runtime2.step();
|
|
11392
11392
|
if (value instanceof Map || Array.isArray(value)) {
|
|
11393
11393
|
const entries = value instanceof Map ? value.entries() : value.entries();
|
|
11394
11394
|
for (const [key, child2] of entries) flatten(child2, path + String(key) + "/", row);
|
|
@@ -11399,30 +11399,30 @@ async function jsonConversion(runtime, newline) {
|
|
|
11399
11399
|
while (path[start] === "/") start++;
|
|
11400
11400
|
while (end > start && path[end - 1] === "/") end--;
|
|
11401
11401
|
const name = path.slice(start, end);
|
|
11402
|
-
const cell2 = value === null ? null : typeof value === "string" ? value : typeof value === "boolean" ? value ? "True" : "False" : jsonNumberText(value,
|
|
11403
|
-
if (cell2 !== null && Array.from(cell2).length >
|
|
11404
|
-
|
|
11402
|
+
const cell2 = value === null ? null : typeof value === "string" ? value : typeof value === "boolean" ? value ? "True" : "False" : jsonNumberText(value, runtime2);
|
|
11403
|
+
if (cell2 !== null && Array.from(cell2).length > runtime2.context.limits.maxFieldCharacters) throw new CsvkitBlocked("JSON output field budget exceeded");
|
|
11404
|
+
runtime2.retain(64 + name.length * 2 + (cell2?.length ?? 0) * 2);
|
|
11405
11405
|
row.set(name, value);
|
|
11406
11406
|
};
|
|
11407
11407
|
for (const item of input) {
|
|
11408
|
-
|
|
11409
|
-
|
|
11408
|
+
runtime2.step();
|
|
11409
|
+
runtime2.retain(64);
|
|
11410
11410
|
const row = /* @__PURE__ */ new Map();
|
|
11411
11411
|
flatten(item, "", row);
|
|
11412
11412
|
for (const name of row.keys()) if (!known.has(name)) {
|
|
11413
|
-
if (headers.length >=
|
|
11413
|
+
if (headers.length >= runtime2.context.limits.maxColumns) throw new CsvkitBlocked("column budget exceeded");
|
|
11414
11414
|
known.add(name);
|
|
11415
11415
|
headers.push(name);
|
|
11416
|
-
|
|
11416
|
+
runtime2.retain(32);
|
|
11417
11417
|
}
|
|
11418
|
-
|
|
11418
|
+
runtime2.admitRecord({ cells: [...row.values()].map((cell2) => cell2 === null ? "" : typeof cell2 === "string" ? cell2 : typeof cell2 === "boolean" ? cell2 ? "True" : "False" : jsonNumberText(cell2, runtime2)), line: rows.length + 1 });
|
|
11419
11419
|
rows.push(row);
|
|
11420
11420
|
}
|
|
11421
|
-
if (!headers.length && rows.length && !
|
|
11422
|
-
const utilsPath =
|
|
11421
|
+
if (!headers.length && rows.length && !runtime2.context.columnWarnings?.suppressWarnings) {
|
|
11422
|
+
const utilsPath = runtime2.context.columnWarnings?.utilsPath;
|
|
11423
11423
|
if (!utilsPath) throw new CsvkitBlocked("JSON empty-column warning provenance");
|
|
11424
11424
|
const path = utilsPath.slice(0, utilsPath.lastIndexOf("/") + 1) + "table/from_object.py";
|
|
11425
|
-
await
|
|
11425
|
+
await runtime2.write(warningText({
|
|
11426
11426
|
path,
|
|
11427
11427
|
line: 93,
|
|
11428
11428
|
category: "RuntimeWarning",
|
|
@@ -11430,20 +11430,20 @@ async function jsonConversion(runtime, newline) {
|
|
|
11430
11430
|
source: "return Table(rows, column_names, row_names=row_names, column_types=column_types)"
|
|
11431
11431
|
}), "stderr");
|
|
11432
11432
|
}
|
|
11433
|
-
const names = await normalizeHeaders(headers,
|
|
11434
|
-
const table = jsonInputTable(
|
|
11435
|
-
await
|
|
11436
|
-
if (table.rows.length > 0) await
|
|
11437
|
-
if (table.rows.length > 1) await
|
|
11433
|
+
const names = await normalizeHeaders(headers, runtime2);
|
|
11434
|
+
const table = jsonInputTable(runtime2, names, rows.map((row) => headers.map((name) => row.get(name) ?? null)));
|
|
11435
|
+
await runtime2.row(table.headers);
|
|
11436
|
+
if (table.rows.length > 0) await runtime2.row(csvifiedRow(table.rows[0]));
|
|
11437
|
+
if (table.rows.length > 1) await runtime2.rows(table.rows.slice(1).map(csvifiedRow));
|
|
11438
11438
|
return 0;
|
|
11439
11439
|
}
|
|
11440
11440
|
|
|
11441
11441
|
// packages/safe-bash-command-csvkit/src/formats/json.ts
|
|
11442
|
-
var json3 = { name: "json", extensions: ["json", "js"], inferredBy: "key", convert: (
|
|
11442
|
+
var json3 = { name: "json", extensions: ["json", "js"], inferredBy: "key", convert: (runtime2) => jsonConversion(runtime2, false) };
|
|
11443
11443
|
|
|
11444
11444
|
// packages/safe-bash-command-csvkit/src/formats/ndjson.ts
|
|
11445
11445
|
init_buffer();
|
|
11446
|
-
var ndjson = { name: "ndjson", convert: (
|
|
11446
|
+
var ndjson = { name: "ndjson", convert: (runtime2) => jsonConversion(runtime2, true) };
|
|
11447
11447
|
|
|
11448
11448
|
// packages/safe-bash-command-csvkit/src/formats/xls.ts
|
|
11449
11449
|
init_buffer();
|
|
@@ -11993,8 +11993,8 @@ function legacyWorkbookDateFormat(format3, step) {
|
|
|
11993
11993
|
// packages/safe-bash-command-csvkit/src/operations/workbook-input.ts
|
|
11994
11994
|
var workbookAddressBounds = { rows: Number.MAX_SAFE_INTEGER, columns: Number.MAX_SAFE_INTEGER };
|
|
11995
11995
|
var WorkbookInput = class {
|
|
11996
|
-
constructor(
|
|
11997
|
-
this.runtime =
|
|
11996
|
+
constructor(runtime2, format3) {
|
|
11997
|
+
this.runtime = runtime2;
|
|
11998
11998
|
this.format = format3;
|
|
11999
11999
|
}
|
|
12000
12000
|
runtime;
|
|
@@ -12308,18 +12308,18 @@ function xlsxElapsedFormat(format3) {
|
|
|
12308
12308
|
const lower = format3.split(";", 1)[0].toLowerCase();
|
|
12309
12309
|
return ["[h]", "[hh]", "[m]", "[mm]", "[s]", "[ss]"].some((token) => lower.includes(token));
|
|
12310
12310
|
}
|
|
12311
|
-
function xlsxDateFormat(format3,
|
|
12311
|
+
function xlsxDateFormat(format3, runtime2) {
|
|
12312
12312
|
const section = format3.indexOf(";");
|
|
12313
12313
|
const end = section < 0 ? format3.length : section;
|
|
12314
12314
|
let previous = "";
|
|
12315
12315
|
for (let index = 0; index < end; index++) {
|
|
12316
|
-
|
|
12316
|
+
runtime2.step();
|
|
12317
12317
|
const char = format3[index];
|
|
12318
12318
|
if (char === '"' || char === "[") {
|
|
12319
12319
|
const close = char === '"' ? '"' : "]";
|
|
12320
12320
|
let last = index + 1;
|
|
12321
12321
|
while (last < end && format3[last] !== close && !(char === '"' && format3[last] === "\n")) {
|
|
12322
|
-
|
|
12322
|
+
runtime2.step();
|
|
12323
12323
|
last++;
|
|
12324
12324
|
}
|
|
12325
12325
|
if (last < end && format3[last] === close) {
|
|
@@ -12343,42 +12343,42 @@ function xlsCodepage(encoding) {
|
|
|
12343
12343
|
}
|
|
12344
12344
|
|
|
12345
12345
|
// packages/safe-bash-command-csvkit/src/commands/in2csv.ts
|
|
12346
|
-
async function convert(
|
|
12347
|
-
const o =
|
|
12346
|
+
async function convert(runtime2) {
|
|
12347
|
+
const o = runtime2.options;
|
|
12348
12348
|
const path = o.input_path;
|
|
12349
12349
|
let provider = o.filetype ? formats.find((format3) => format3.name === o.filetype) : formats.find((format3) => format3.inferredBy && o[format3.inferredBy]);
|
|
12350
12350
|
if (!provider) {
|
|
12351
|
-
if (!path || path === "-")
|
|
12351
|
+
if (!path || path === "-") runtime2.error("You must specify a format when providing input as piped data via STDIN.");
|
|
12352
12352
|
const period = path.lastIndexOf(".");
|
|
12353
12353
|
const extension = period < 0 ? void 0 : path.slice(period + 1).toLowerCase();
|
|
12354
12354
|
provider = formats.find((format3) => extension === void 0 ? format3.extensionless : format3.extensions?.includes(extension));
|
|
12355
|
-
if (!provider)
|
|
12355
|
+
if (!provider) runtime2.error("Unable to automatically determine the format of the input file. Try specifying a format with --format.");
|
|
12356
12356
|
}
|
|
12357
|
-
const workbook = provider.sheetNames ? new WorkbookInput(
|
|
12357
|
+
const workbook = provider.sheetNames ? new WorkbookInput(runtime2, provider.name) : void 0;
|
|
12358
12358
|
if (o.names_only) {
|
|
12359
|
-
if (!workbook)
|
|
12360
|
-
for (const name of (await workbook.open(true)).SheetNames) await
|
|
12359
|
+
if (!workbook) runtime2.error("You cannot use the -n or --names options with non-Excel files.");
|
|
12360
|
+
for (const name of (await workbook.open(true)).SheetNames) await runtime2.write(name + "\n");
|
|
12361
12361
|
return 0;
|
|
12362
12362
|
}
|
|
12363
12363
|
if (o.schema && provider.name !== "fixed") {
|
|
12364
|
-
|
|
12364
|
+
runtime2.input(String(o.schema));
|
|
12365
12365
|
}
|
|
12366
12366
|
if (workbook) {
|
|
12367
12367
|
const [, table] = await workbook.table(await workbook.open(), o.sheet);
|
|
12368
|
-
await
|
|
12369
|
-
if (table.rows.length > 0) await
|
|
12370
|
-
if (table.rows.length > 1) await
|
|
12368
|
+
await runtime2.row(table.headers);
|
|
12369
|
+
if (table.rows.length > 0) await runtime2.row(csvifiedRow(table.rows[0]));
|
|
12370
|
+
if (table.rows.length > 1) await runtime2.rows(table.rows.slice(1).map(csvifiedRow));
|
|
12371
12371
|
} else {
|
|
12372
12372
|
if (!provider.convert) throw new CsvkitBlocked(`in2csv ${provider.name} format`);
|
|
12373
|
-
await provider.convert(
|
|
12373
|
+
await provider.convert(runtime2);
|
|
12374
12374
|
}
|
|
12375
12375
|
if (o.write_sheets) {
|
|
12376
|
-
|
|
12376
|
+
runtime2.sideEffects = true;
|
|
12377
12377
|
if (workbook) await workbook.writeSheets();
|
|
12378
12378
|
else {
|
|
12379
12379
|
if (!path || path === "-") throw new CsvkitDiagnostic("ValueError: read of closed file");
|
|
12380
|
-
for await (const ignored of
|
|
12381
|
-
|
|
12380
|
+
for await (const ignored of runtime2.bytes()) {
|
|
12381
|
+
runtime2.step();
|
|
12382
12382
|
}
|
|
12383
12383
|
if (o.write_sheets === "-") throw new CsvkitBlocked("non-Excel --write-sheets - workbook decoding");
|
|
12384
12384
|
throw new CsvkitDiagnostic("UnboundLocalError: cannot access local variable 'tables' where it is not associated with a value");
|
|
@@ -12950,12 +12950,12 @@ var in2csv = {
|
|
|
12950
12950
|
|
|
12951
12951
|
// packages/safe-bash-command-csvkit/src/commands/sql2csv.ts
|
|
12952
12952
|
init_buffer();
|
|
12953
|
-
async function query(
|
|
12954
|
-
const o =
|
|
12955
|
-
const context =
|
|
12956
|
-
if (context.terminal.stdinIsTTY && !o.input_path && !o.query)
|
|
12953
|
+
async function query(runtime2) {
|
|
12954
|
+
const o = runtime2.options;
|
|
12955
|
+
const context = runtime2.context;
|
|
12956
|
+
if (context.terminal.stdinIsTTY && !o.input_path && !o.query) runtime2.error("You must provide an input file or piped data.");
|
|
12957
12957
|
const url = String(o.connection_string);
|
|
12958
|
-
const engineOptions = sqlOptions(o.engine_option,
|
|
12958
|
+
const engineOptions = sqlOptions(o.engine_option, runtime2.step.bind(runtime2));
|
|
12959
12959
|
const { provider } = resolveDatabaseProvider(url, context.databases);
|
|
12960
12960
|
const resources = new ResourceScope(context.signal, context.registerCleanup);
|
|
12961
12961
|
let completed = false;
|
|
@@ -12975,17 +12975,17 @@ async function query(runtime) {
|
|
|
12975
12975
|
}
|
|
12976
12976
|
if (errors.length) throw new AggregateError(errors, "database session cleanup failed");
|
|
12977
12977
|
});
|
|
12978
|
-
if (o.query)
|
|
12979
|
-
const sql = o.query ? stripWhitespace(String(o.query)) : await
|
|
12978
|
+
if (o.query) runtime2.retain(32 + String(o.query).length * 16);
|
|
12979
|
+
const sql = o.query ? stripWhitespace(String(o.query)) : await runtime2.text();
|
|
12980
12980
|
context.signal.throwIfAborted();
|
|
12981
12981
|
const executionOptions = Object.assign(
|
|
12982
12982
|
/* @__PURE__ */ Object.create(null),
|
|
12983
12983
|
{ no_parameters: true, stream_results: true },
|
|
12984
|
-
sqlOptions(o.execution_option,
|
|
12984
|
+
sqlOptions(o.execution_option, runtime2.step.bind(runtime2))
|
|
12985
12985
|
);
|
|
12986
12986
|
const result = await resources.acquire(() => session.query(sql, [], executionOptions, context.signal), (result2) => result2.close());
|
|
12987
12987
|
if (result.columns !== null) {
|
|
12988
|
-
if (!o.no_header_row) await
|
|
12988
|
+
if (!o.no_header_row) await runtime2.row(result.columns);
|
|
12989
12989
|
let reading;
|
|
12990
12990
|
const iterator = await resources.acquire(async () => result.rows[Symbol.asyncIterator](), async (iterator2) => {
|
|
12991
12991
|
const returning = Promise.resolve().then(() => iterator2.return?.());
|
|
@@ -12994,18 +12994,18 @@ async function query(runtime) {
|
|
|
12994
12994
|
});
|
|
12995
12995
|
let count = 0;
|
|
12996
12996
|
while (true) {
|
|
12997
|
-
|
|
12997
|
+
runtime2.step();
|
|
12998
12998
|
reading = Promise.resolve().then(() => iterator.next());
|
|
12999
12999
|
const next = await reading;
|
|
13000
13000
|
reading = void 0;
|
|
13001
|
-
|
|
13001
|
+
runtime2.step();
|
|
13002
13002
|
if (next.done) break;
|
|
13003
13003
|
if (++count > context.limits.maxDatabaseResultRows) throw new CsvkitBlocked("database result row budget exceeded");
|
|
13004
13004
|
const row = next.value.map((value) => {
|
|
13005
13005
|
if (value instanceof Uint8Array) return bytesRepr(value);
|
|
13006
13006
|
return value;
|
|
13007
13007
|
});
|
|
13008
|
-
await
|
|
13008
|
+
await runtime2.row(row);
|
|
13009
13009
|
}
|
|
13010
13010
|
}
|
|
13011
13011
|
completed = true;
|
|
@@ -13714,7 +13714,7 @@ async function perform(descriptor, parsed, context, retainedSettings = 0) {
|
|
|
13714
13714
|
let failed = false;
|
|
13715
13715
|
let failure;
|
|
13716
13716
|
let status;
|
|
13717
|
-
const
|
|
13717
|
+
const runtime2 = new Runtime({ ...context, registerCleanup: (cleanup, destination) => context.registerCleanup(async () => {
|
|
13718
13718
|
try {
|
|
13719
13719
|
await cleanup();
|
|
13720
13720
|
} catch (cleanupFailure) {
|
|
@@ -13722,16 +13722,16 @@ async function perform(descriptor, parsed, context, retainedSettings = 0) {
|
|
|
13722
13722
|
}
|
|
13723
13723
|
}, destination) }, descriptor, parsed.kind === "parsed" ? parsed.options : {}, parsed.kind === "parsed" ? parsed.matchFiles : void 0);
|
|
13724
13724
|
try {
|
|
13725
|
-
|
|
13725
|
+
runtime2.retain(retainedSettings);
|
|
13726
13726
|
if (parsed.kind === "blocked") throw parsed.failure;
|
|
13727
13727
|
if (parsed.kind === "exit") {
|
|
13728
|
-
if (parsed.stdout) await
|
|
13729
|
-
if (parsed.stderr) await
|
|
13728
|
+
if (parsed.stdout) await runtime2.write(parsed.stdout);
|
|
13729
|
+
if (parsed.stderr) await runtime2.write(parsed.stderr, "stderr");
|
|
13730
13730
|
status = parsed.status;
|
|
13731
13731
|
} else {
|
|
13732
|
-
if (parsed.options.add_bom) await
|
|
13732
|
+
if (parsed.options.add_bom) await runtime2.write("\uFEFF");
|
|
13733
13733
|
if (!descriptor.execute) throw new CsvkitBlocked(`${command} operation engine`);
|
|
13734
|
-
status = await descriptor.execute(
|
|
13734
|
+
status = await descriptor.execute(runtime2);
|
|
13735
13735
|
}
|
|
13736
13736
|
} catch (caught) {
|
|
13737
13737
|
failure = caught;
|
|
@@ -13743,7 +13743,7 @@ async function perform(descriptor, parsed, context, retainedSettings = 0) {
|
|
|
13743
13743
|
parsed.kind === "parsed" && Boolean(parsed.options.verbose),
|
|
13744
13744
|
parsed.kind === "parsed" ? String(parsed.options.encoding) : "utf-8-sig"
|
|
13745
13745
|
);
|
|
13746
|
-
await
|
|
13746
|
+
await runtime2.write(report.stderr, "stderr");
|
|
13747
13747
|
status = report.status;
|
|
13748
13748
|
} catch (sinkFailure) {
|
|
13749
13749
|
failure = sinkFailure;
|
|
@@ -13751,7 +13751,7 @@ async function perform(descriptor, parsed, context, retainedSettings = 0) {
|
|
|
13751
13751
|
}
|
|
13752
13752
|
}
|
|
13753
13753
|
}
|
|
13754
|
-
const results = await Promise.allSettled([
|
|
13754
|
+
const results = await Promise.allSettled([runtime2.close(), ...parsed.kind === "parsed" && parsed.dispose ? [parsed.dispose()] : []]);
|
|
13755
13755
|
context.signal.throwIfAborted();
|
|
13756
13756
|
if (status === void 0) throw failure;
|
|
13757
13757
|
if (!failed) {
|
|
@@ -15467,6 +15467,36 @@ function createCsvpyInterpreter(options) {
|
|
|
15467
15467
|
// packages/safe-bash-command-csvkit/src/command.ts
|
|
15468
15468
|
init_buffer();
|
|
15469
15469
|
|
|
15470
|
+
// packages/safe-bash-command-csvkit/src/default-sqlite.ts
|
|
15471
|
+
init_buffer();
|
|
15472
|
+
var runtime;
|
|
15473
|
+
function createDefaultSqliteDatabaseProvider(clock, limits = {}) {
|
|
15474
|
+
const budget = Object.freeze({ ...limits });
|
|
15475
|
+
return Object.freeze({
|
|
15476
|
+
schemes: Object.freeze(["sqlite", "sqlite+pysqlite"]),
|
|
15477
|
+
profile: "sqlite-wasm-3.50.4-cpython-legacy",
|
|
15478
|
+
async connect(url, options, signal, invocation) {
|
|
15479
|
+
signal.throwIfAborted();
|
|
15480
|
+
runtime ??= import("@sqlite.org/sqlite-wasm").then(({ default: initialize }) => initialize({ print: () => {
|
|
15481
|
+
}, printErr: () => {
|
|
15482
|
+
} }));
|
|
15483
|
+
const sqlite2 = await runtime;
|
|
15484
|
+
signal.throwIfAborted();
|
|
15485
|
+
const provider = createSqliteDatabaseProvider({ sqlite: sqlite2, limits: budget, cwd: invocation?.cwd ?? "/", clock, random: (bytes) => {
|
|
15486
|
+
globalThis.crypto.getRandomValues(bytes);
|
|
15487
|
+
} });
|
|
15488
|
+
try {
|
|
15489
|
+
const session = await provider.connect(url, options, signal, invocation);
|
|
15490
|
+
signal.throwIfAborted();
|
|
15491
|
+
return Object.freeze({ ...session, close: () => provider.dispose() });
|
|
15492
|
+
} catch (error) {
|
|
15493
|
+
await provider.dispose();
|
|
15494
|
+
throw error;
|
|
15495
|
+
}
|
|
15496
|
+
}
|
|
15497
|
+
});
|
|
15498
|
+
}
|
|
15499
|
+
|
|
15470
15500
|
// packages/safe-bash-command-csvkit/src/portable-locale.ts
|
|
15471
15501
|
init_buffer();
|
|
15472
15502
|
var portableLocale = Object.freeze({
|
|
@@ -15514,10 +15544,10 @@ function createCsvkitCommands(options = {}) {
|
|
|
15514
15544
|
const limits = Object.freeze({ ...defaultLimits, ...options.limits });
|
|
15515
15545
|
const codecs = Object.freeze([...options.codecs ?? [utf8Codec, ...pythonCodecs]]);
|
|
15516
15546
|
const compression = Object.freeze([...options.compression ?? [createGzipCompressionProvider(createCompressionCodec())]]);
|
|
15517
|
-
const
|
|
15547
|
+
const clock = options.clock ?? { now: Date.now };
|
|
15548
|
+
const databases2 = Object.freeze([...options.databases ?? [createDefaultSqliteDatabaseProvider(clock, { maxWork: limits.maxWork, maxSqlBytes: limits.maxRetainedBytes, maxValueBytes: limits.maxRetainedBytes, maxResultRows: limits.maxDatabaseResultRows })]]);
|
|
15518
15549
|
const sqlDialects = Object.freeze([...options.sqlDialects ?? databases]);
|
|
15519
15550
|
const locale = options.locale ?? portableLocale;
|
|
15520
|
-
const clock = options.clock ?? { now: Date.now };
|
|
15521
15551
|
const interpreter = options.interpreter, openMatchFile = options.openMatchFile;
|
|
15522
15552
|
const terminal = Object.freeze({ stdinIsTTY: false, stdoutIsTTY: false, stderrIsTTY: false, columns: 80, lines: 24, ...options.terminal });
|
|
15523
15553
|
const sniffing = options.sniffing === void 0 ? Object.freeze({ stream: defaultSniffStreamProfile, suppressWarnings: true }) : Object.freeze({
|