gangtise-openapi-cli 0.41.1 → 0.43.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +45 -39
- package/dist/src/cli.js +50 -1161
- package/dist/src/commands/ai.js +156 -0
- package/dist/src/commands/alternative.js +56 -0
- package/dist/src/commands/auth.js +37 -0
- package/dist/src/commands/bond.js +76 -0
- package/dist/src/commands/fundamental.js +137 -0
- package/dist/src/commands/indicator.js +134 -0
- package/dist/src/commands/insight.js +338 -0
- package/dist/src/commands/quote.js +181 -0
- package/dist/src/commands/raw.js +49 -0
- package/dist/src/commands/reference.js +58 -0
- package/dist/src/commands/shared.js +197 -0
- package/dist/src/commands/tool.js +55 -0
- package/dist/src/commands/vault.js +203 -0
- package/dist/src/core/args.js +13 -9
- package/dist/src/core/asyncContent.js +32 -19
- package/dist/src/core/auth.js +3 -2
- package/dist/src/core/client.js +112 -15
- package/dist/src/core/commandBodies.js +2 -1
- package/dist/src/core/config.js +16 -3
- package/dist/src/core/download.js +9 -5
- package/dist/src/core/endpoints.js +106 -1
- package/dist/src/core/exitStatus.js +17 -0
- package/dist/src/core/fileParse.js +3 -20
- package/dist/src/core/opinionDetail.js +39 -0
- package/dist/src/core/output.js +55 -12
- package/dist/src/core/perSecurity.js +29 -11
- package/dist/src/core/printer.js +7 -8
- package/dist/src/core/quoteSharding.js +19 -15
- package/dist/src/core/rowSink.js +40 -7
- package/dist/src/core/titleCache.js +28 -2
- package/dist/src/core/transport.js +7 -4
- package/dist/src/core/updateCheck.js +51 -0
- package/dist/src/version.js +19 -1
- package/gangtise-openapi/SKILL.md +50 -49
- package/gangtise-openapi/references/commands/ai.md +35 -16
- package/gangtise-openapi/references/commands/alternative.md +2 -2
- package/gangtise-openapi/references/commands/bond.md +1 -1
- package/gangtise-openapi/references/commands/fundamental.md +5 -5
- package/gangtise-openapi/references/commands/indicator.md +29 -27
- package/gangtise-openapi/references/commands/insight.md +39 -31
- package/gangtise-openapi/references/commands/quote.md +19 -19
- package/gangtise-openapi/references/commands/reference-and-lookup.md +18 -15
- package/gangtise-openapi/references/commands/vault.md +10 -7
- package/gangtise-openapi/references/errors.md +27 -25
- package/gangtise-openapi/references/examples.md +38 -36
- package/gangtise-openapi/references/fields.md +8 -6
- package/gangtise-openapi/references/lookup-ids.md +10 -8
- package/gangtise-openapi/references/response-schema.md +7 -7
- package/package.json +1 -1
package/dist/src/core/output.js
CHANGED
|
@@ -1,9 +1,32 @@
|
|
|
1
1
|
import { createHash, randomBytes } from "node:crypto";
|
|
2
|
+
import { rmSync, statSync } from "node:fs";
|
|
2
3
|
import fs from "node:fs/promises";
|
|
3
4
|
import { ConfigError } from "./errors.js";
|
|
4
5
|
const OUTPUT_FORMATS = ["table", "json", "jsonl", "csv", "markdown"];
|
|
5
6
|
export function stagingPath(target, suffix = "part") {
|
|
6
|
-
|
|
7
|
+
const staging = `${target}.${randomBytes(4).toString("hex")}.${suffix}`;
|
|
8
|
+
handedOut.add(staging);
|
|
9
|
+
return staging;
|
|
10
|
+
}
|
|
11
|
+
const handedOut = new Set();
|
|
12
|
+
const claimed = new Set();
|
|
13
|
+
export function trackClaim(name) {
|
|
14
|
+
claimed.add(name);
|
|
15
|
+
}
|
|
16
|
+
export function removeStagingFiles() {
|
|
17
|
+
for (const staging of handedOut) {
|
|
18
|
+
try {
|
|
19
|
+
rmSync(staging, { force: true });
|
|
20
|
+
}
|
|
21
|
+
catch { }
|
|
22
|
+
}
|
|
23
|
+
for (const name of claimed) {
|
|
24
|
+
try {
|
|
25
|
+
if (statSync(name).size === 0)
|
|
26
|
+
rmSync(name, { force: true });
|
|
27
|
+
}
|
|
28
|
+
catch { }
|
|
29
|
+
}
|
|
7
30
|
}
|
|
8
31
|
export function digestBuffer(content) {
|
|
9
32
|
const buf = typeof content === "string" ? Buffer.from(content, "utf8") : content;
|
|
@@ -29,9 +52,13 @@ export function parseOutputFormat(value) {
|
|
|
29
52
|
throw new ConfigError(`Unsupported format: ${format}`);
|
|
30
53
|
}
|
|
31
54
|
function sanitizeCell(value) {
|
|
55
|
+
if (!/[\u0000-\u001f\u007f-\u009f]/.test(value))
|
|
56
|
+
return value;
|
|
32
57
|
return value.replace(/[\r\n]+/g, " ").replace(/[\u0000-\u001f\u007f\u0080-\u009f]/g, "");
|
|
33
58
|
}
|
|
34
59
|
function displayWidth(value) {
|
|
60
|
+
if (/^[\x20-\x7e]*$/.test(value))
|
|
61
|
+
return value.length;
|
|
35
62
|
let width = 0;
|
|
36
63
|
for (const ch of value) {
|
|
37
64
|
const cp = ch.codePointAt(0);
|
|
@@ -97,11 +124,13 @@ function renderTable(rows) {
|
|
|
97
124
|
const columns = Array.from(new Set(rows.flatMap((row) => Object.keys(row))));
|
|
98
125
|
const headerCells = columns.map((column) => clampCell(sanitizeCell(column)));
|
|
99
126
|
const matrix = rows.map((row) => columns.map((column) => clampCell(sanitizeCell(formatScalar(row[column])))));
|
|
100
|
-
const
|
|
101
|
-
const
|
|
102
|
-
const
|
|
103
|
-
const
|
|
104
|
-
const
|
|
127
|
+
const cellWidths = matrix.map((cells) => cells.map(displayWidth));
|
|
128
|
+
const headerWidths = headerCells.map(displayWidth);
|
|
129
|
+
const widths = columns.map((_, c) => cellWidths.reduce((max, cells) => Math.max(max, cells[c]), headerWidths[c]));
|
|
130
|
+
const renderLine = (values, valueWidths) => values.map((value, index) => value + " ".repeat(Math.max(0, widths[index] - valueWidths[index]))).join(" ");
|
|
131
|
+
const header = renderLine(headerCells, headerWidths);
|
|
132
|
+
const divider = renderLine(widths.map((width) => "-".repeat(width)), widths);
|
|
133
|
+
const body = matrix.map((cells, r) => renderLine(cells, cellWidths[r]));
|
|
105
134
|
return [header, divider, ...body].join("\n");
|
|
106
135
|
}
|
|
107
136
|
function renderMarkdown(rows) {
|
|
@@ -170,16 +199,26 @@ export async function streamOutputToFile(value, format, outputPath) {
|
|
|
170
199
|
stream.on("error", () => { });
|
|
171
200
|
try {
|
|
172
201
|
if (format === "jsonl") {
|
|
202
|
+
let chunk = [];
|
|
173
203
|
for (const item of jsonlItems(value)) {
|
|
174
|
-
|
|
204
|
+
chunk.push(JSON.stringify(item));
|
|
205
|
+
if (chunk.length >= LINES_PER_WRITE) {
|
|
206
|
+
await writeLines(stream, chunk);
|
|
207
|
+
chunk = [];
|
|
208
|
+
}
|
|
175
209
|
}
|
|
210
|
+
await writeLines(stream, chunk);
|
|
176
211
|
}
|
|
177
212
|
else {
|
|
178
|
-
|
|
213
|
+
let chunk = ["\ufeff" + csvColumns.map(csvEscape).join(",")];
|
|
179
214
|
for (const row of csvRows) {
|
|
180
|
-
|
|
181
|
-
|
|
215
|
+
chunk.push(csvColumns.map((column) => csvEscape(formatScalar(row[column]))).join(","));
|
|
216
|
+
if (chunk.length >= LINES_PER_WRITE) {
|
|
217
|
+
await writeLines(stream, chunk);
|
|
218
|
+
chunk = [];
|
|
219
|
+
}
|
|
182
220
|
}
|
|
221
|
+
await writeLines(stream, chunk);
|
|
183
222
|
}
|
|
184
223
|
await new Promise((resolve, reject) => {
|
|
185
224
|
stream.end((err) => err ? reject(err) : resolve());
|
|
@@ -215,6 +254,10 @@ export function csvEscape(value) {
|
|
|
215
254
|
return `"${out.replaceAll("\"", "\"\"")}"`;
|
|
216
255
|
return out;
|
|
217
256
|
}
|
|
257
|
+
export const LINES_PER_WRITE = 1000;
|
|
258
|
+
export function writeLines(stream, lines) {
|
|
259
|
+
return lines.length === 0 ? Promise.resolve() : writeLine(stream, lines.join("\n"));
|
|
260
|
+
}
|
|
218
261
|
export function writeLine(stream, line) {
|
|
219
262
|
return new Promise((resolve, reject) => {
|
|
220
263
|
const ok = stream.write(line + "\n", (err) => err ? reject(err) : undefined);
|
|
@@ -228,7 +271,7 @@ export function writeLine(stream, line) {
|
|
|
228
271
|
stream.once("error", onError);
|
|
229
272
|
});
|
|
230
273
|
}
|
|
231
|
-
export async function saveOutputIfNeeded(content, outputPath) {
|
|
274
|
+
export async function saveOutputIfNeeded(content, outputPath, withDigest = true) {
|
|
232
275
|
if (!outputPath) {
|
|
233
276
|
return;
|
|
234
277
|
}
|
|
@@ -242,7 +285,7 @@ export async function saveOutputIfNeeded(content, outputPath) {
|
|
|
242
285
|
else {
|
|
243
286
|
await fs.writeFile(partPath, content);
|
|
244
287
|
}
|
|
245
|
-
const digest = digestBuffer(content);
|
|
288
|
+
const digest = withDigest ? digestBuffer(content) : undefined;
|
|
246
289
|
await fs.rename(partPath, outputPath);
|
|
247
290
|
return digest;
|
|
248
291
|
}
|
|
@@ -17,7 +17,10 @@ export function estimateTradingDays(startDate, endDate) {
|
|
|
17
17
|
}
|
|
18
18
|
return days;
|
|
19
19
|
}
|
|
20
|
-
export async function callPerSecurity(client, endpointKey, securities, makeBody, cap, label) {
|
|
20
|
+
export async function callPerSecurity(client, endpointKey, securities, makeBody, cap, label, groupSize = 1) {
|
|
21
|
+
const groups = [];
|
|
22
|
+
for (let i = 0; i < securities.length; i += groupSize)
|
|
23
|
+
groups.push(securities.slice(i, i + groupSize));
|
|
21
24
|
const sink = client.claimRowSink?.();
|
|
22
25
|
let fieldList;
|
|
23
26
|
let headerSecurity;
|
|
@@ -29,14 +32,14 @@ export async function callPerSecurity(client, endpointKey, securities, makeBody,
|
|
|
29
32
|
const mergePart = async (part, i) => {
|
|
30
33
|
const rec = part;
|
|
31
34
|
if (!(rec && typeof rec === "object" && Array.isArray(rec.list))) {
|
|
32
|
-
throw markStructural(new ApiError(`${label}: ${
|
|
35
|
+
throw markStructural(new ApiError(`${label}: ${groups[i].join(", ")} returned no list payload — the response layout may have changed`, undefined, undefined, rec));
|
|
33
36
|
}
|
|
34
37
|
if (rec.partial === true)
|
|
35
38
|
partial = true;
|
|
36
39
|
if (rec.list.length === 0) {
|
|
37
40
|
if ((typeof rec.total === "number" && rec.total > 0) || rec.partial === true) {
|
|
38
41
|
const claim = typeof rec.total === "number" && rec.total > 0 ? `reported total=${rec.total}` : "carried a partial marker";
|
|
39
|
-
throw markStructural(new ApiError(`${label}: ${
|
|
42
|
+
throw markStructural(new ApiError(`${label}: ${groups[i].join(", ")} ${claim} but delivered no rows — the response is inconsistent`, undefined, undefined, rec));
|
|
40
43
|
}
|
|
41
44
|
if (!emptyFields && Array.isArray(rec.fieldList))
|
|
42
45
|
emptyFields = rec.fieldList;
|
|
@@ -44,26 +47,27 @@ export async function callPerSecurity(client, endpointKey, securities, makeBody,
|
|
|
44
47
|
}
|
|
45
48
|
const fields = Array.isArray(rec.fieldList) && rec.fieldList.length > 0 ? rec.fieldList : undefined;
|
|
46
49
|
if (rec.list.some(Array.isArray) && !columnarSchemaValid(fields, rec.list)) {
|
|
47
|
-
throw markStructural(new ApiError(`${label}: ${
|
|
50
|
+
throw markStructural(new ApiError(`${label}: ${groups[i].join(", ")} returned columnar rows without a usable fieldList (missing, duplicated or mis-sized) — they cannot be read by position`, undefined, undefined, rec));
|
|
48
51
|
}
|
|
49
52
|
if (fields && !fieldList) {
|
|
50
53
|
fieldList = fields;
|
|
51
|
-
headerSecurity =
|
|
54
|
+
headerSecurity = groups[i].join(", ");
|
|
52
55
|
sink?.setFieldList(fieldList);
|
|
53
56
|
}
|
|
54
57
|
if ((fields ?? fieldList) && !sameColumns(fieldList, fields)) {
|
|
55
|
-
throw markStructural(new ApiError(`${label}: ${
|
|
58
|
+
throw markStructural(new ApiError(`${label}: ${groups[i].join(", ")} answered with columns ${JSON.stringify(fields)} while ${headerSecurity} answered ${JSON.stringify(fieldList)} — the parts cannot be merged`, undefined, undefined, rec));
|
|
56
59
|
}
|
|
57
60
|
if (rec.list.length >= cap)
|
|
58
|
-
truncated.push(
|
|
59
|
-
|
|
61
|
+
truncated.push(...groups[i]);
|
|
62
|
+
const rows = inInputOrder(rec.list, fields ?? fieldList, groups[i], label);
|
|
63
|
+
count += rows.length;
|
|
60
64
|
if (sink)
|
|
61
|
-
await sink.push(
|
|
65
|
+
await sink.push(rows);
|
|
62
66
|
else
|
|
63
|
-
for (const row of
|
|
67
|
+
for (const row of rows)
|
|
64
68
|
merged.push(row);
|
|
65
69
|
};
|
|
66
|
-
await runInOrder(
|
|
70
|
+
await runInOrder(groups, PAGE_CONCURRENCY, (codes) => client.call(endpointKey, makeBody(codes)), mergePart);
|
|
67
71
|
const out = { total: count, list: merged };
|
|
68
72
|
if (sink)
|
|
69
73
|
attachRowSink(out, sink);
|
|
@@ -80,6 +84,20 @@ export async function callPerSecurity(client, endpointKey, securities, makeBody,
|
|
|
80
84
|
}
|
|
81
85
|
return out;
|
|
82
86
|
}
|
|
87
|
+
function inInputOrder(rows, fields, group, label) {
|
|
88
|
+
if (group.length < 2)
|
|
89
|
+
return rows;
|
|
90
|
+
const column = fields?.indexOf("securityCode") ?? -1;
|
|
91
|
+
if (rows.some(Array.isArray) && column < 0) {
|
|
92
|
+
throw markStructural(new ApiError(`${label}: ${group.join(", ")} answered without a securityCode column — rows of several securities cannot be told apart`));
|
|
93
|
+
}
|
|
94
|
+
const rank = new Map(group.map((code, i) => [code.toUpperCase(), i]));
|
|
95
|
+
const codeOf = (row) => Array.isArray(row) ? row[column] : row?.securityCode;
|
|
96
|
+
return rows
|
|
97
|
+
.map((row, i) => ({ row, i, at: rank.get(String(codeOf(row)).toUpperCase()) ?? group.length }))
|
|
98
|
+
.sort((a, b) => a.at - b.at || a.i - b.i)
|
|
99
|
+
.map(({ row }) => row);
|
|
100
|
+
}
|
|
83
101
|
function sameColumns(a, b) {
|
|
84
102
|
if (!a || !b)
|
|
85
103
|
return false;
|
package/dist/src/core/printer.js
CHANGED
|
@@ -5,16 +5,16 @@ import { countOutputRows, digestFile, pickList, renderOutput, saveOutputIfNeeded
|
|
|
5
5
|
import { getRowSink } from "./rowSink.js";
|
|
6
6
|
import { extractTitles, writeTitleCache } from "./titleCache.js";
|
|
7
7
|
import { CLI_VERSION } from "../version.js";
|
|
8
|
+
import { decidedExitCode, EXIT_SUPERSEDED, markIncomplete, markSuperseded } from "./exitStatus.js";
|
|
8
9
|
const LARGE_RESULT_ROWS = 50_000;
|
|
9
|
-
export const SUPERSEDED_EXIT =
|
|
10
|
+
export const SUPERSEDED_EXIT = EXIT_SUPERSEDED;
|
|
10
11
|
export async function warnIfSuperseded(output, digest) {
|
|
11
12
|
const actual = await digestFile(output).catch(() => undefined);
|
|
12
13
|
if (!actual || actual.sha256 === digest.sha256)
|
|
13
14
|
return;
|
|
14
15
|
process.stderr.write(`[gangtise] warning: ${output} was replaced by another export to the same path while this one was finishing — the file there is not this command's output `
|
|
15
16
|
+ `(wrote sha256 ${digest.sha256.slice(0, 12)}…, found ${actual.sha256.slice(0, 12)}…). Give concurrent exports distinct --output paths.\n`);
|
|
16
|
-
|
|
17
|
-
process.exitCode = SUPERSEDED_EXIT;
|
|
17
|
+
markSuperseded();
|
|
18
18
|
}
|
|
19
19
|
function warnIfLargeInMemory(items, format) {
|
|
20
20
|
if (items && items.length >= LARGE_RESULT_ROWS) {
|
|
@@ -63,7 +63,7 @@ async function stageExportMeta(output, format, rows, columns, normalized, comple
|
|
|
63
63
|
format,
|
|
64
64
|
rows,
|
|
65
65
|
complete,
|
|
66
|
-
exitCode:
|
|
66
|
+
exitCode: decidedExitCode(),
|
|
67
67
|
command: redactSecrets(process.argv.slice(2)),
|
|
68
68
|
cliVersion: CLI_VERSION,
|
|
69
69
|
fetchedAt: localIso(new Date()),
|
|
@@ -121,11 +121,10 @@ export async function printData(data, format, output, cache) {
|
|
|
121
121
|
if (typeof meta.total === "number" && format !== "json") {
|
|
122
122
|
process.stderr.write(`Total: ${meta.total}, showing: ${showing}\n`);
|
|
123
123
|
}
|
|
124
|
-
if (meta.partial === true)
|
|
125
|
-
|
|
126
|
-
}
|
|
124
|
+
if (meta.partial === true)
|
|
125
|
+
markIncomplete();
|
|
127
126
|
}
|
|
128
|
-
const complete =
|
|
127
|
+
const complete = decidedExitCode() === 0;
|
|
129
128
|
if (output) {
|
|
130
129
|
const staged = format === "csv" || format === "jsonl"
|
|
131
130
|
? await stageExportMeta(output, format, streamed ? streamed.dataRows : countOutputRows(normalized, format), columns, normalized, complete)
|
|
@@ -3,7 +3,8 @@ import { columnarSchemaValid } from "./normalize.js";
|
|
|
3
3
|
import { attachRowSink } from "./rowSink.js";
|
|
4
4
|
import { isVerbose, PAGE_CONCURRENCY, runInOrder } from "./transport.js";
|
|
5
5
|
const DAY_MS = 86_400_000;
|
|
6
|
-
const
|
|
6
|
+
export const QUOTE_MAX_LIMIT = 10_000;
|
|
7
|
+
const ALL_MARKET_LIMIT = QUOTE_MAX_LIMIT;
|
|
7
8
|
function parseDate(value) {
|
|
8
9
|
if (!/^\d{4}-\d{2}-\d{2}$/.test(value))
|
|
9
10
|
return null;
|
|
@@ -42,18 +43,21 @@ function isWeekendUtc(d) {
|
|
|
42
43
|
}
|
|
43
44
|
function buildShards(start, end, shardDays) {
|
|
44
45
|
const shards = [];
|
|
45
|
-
let
|
|
46
|
-
const
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
46
|
+
let group = [];
|
|
47
|
+
const flush = () => {
|
|
48
|
+
if (group.length === 0)
|
|
49
|
+
return;
|
|
50
|
+
shards.push({ startDate: formatDate(new Date(group[0])), endDate: formatDate(new Date(group[group.length - 1])) });
|
|
51
|
+
group = [];
|
|
52
|
+
};
|
|
53
|
+
for (let day = start.getTime(); day <= end.getTime(); day += DAY_MS) {
|
|
54
|
+
if (isWeekendUtc(new Date(day)))
|
|
55
|
+
continue;
|
|
56
|
+
group.push(day);
|
|
57
|
+
if (group.length === shardDays)
|
|
58
|
+
flush();
|
|
59
|
+
}
|
|
60
|
+
flush();
|
|
57
61
|
return shards;
|
|
58
62
|
}
|
|
59
63
|
export async function callKlineWithSharding(client, endpointKey, body, config) {
|
|
@@ -65,7 +69,7 @@ export async function callKlineWithSharding(client, endpointKey, body, config) {
|
|
|
65
69
|
const perShardLimit = allMarketBody.limit ?? ALL_MARKET_LIMIT;
|
|
66
70
|
const truncationHint = perShardLimit < ALL_MARKET_LIMIT
|
|
67
71
|
? "raise or omit --limit to fetch the full market"
|
|
68
|
-
: `a single ${config.shardDays}-
|
|
72
|
+
: `a single ${config.shardDays}-weekday window exceeds the ${ALL_MARKET_LIMIT}-row API cap`;
|
|
69
73
|
const isTruncated = (rec) => Array.isArray(rec.list) && rec.list.length >= perShardLimit;
|
|
70
74
|
const callSingle = async () => {
|
|
71
75
|
const single = await client.call(endpointKey, allMarketBody);
|
|
@@ -92,7 +96,7 @@ export async function callKlineWithSharding(client, endpointKey, body, config) {
|
|
|
92
96
|
if (shards.length === 0)
|
|
93
97
|
return { list: [] };
|
|
94
98
|
if (isVerbose()) {
|
|
95
|
-
process.stderr.write(`[gangtise] sharding ${endpointKey} into ${shards.length} requests (${config.shardDays}
|
|
99
|
+
process.stderr.write(`[gangtise] sharding ${endpointKey} into ${shards.length} requests (${config.shardDays} weekday(s) each)\n`);
|
|
96
100
|
}
|
|
97
101
|
const failedShards = [];
|
|
98
102
|
let firstError = null;
|
package/dist/src/core/rowSink.js
CHANGED
|
@@ -2,7 +2,7 @@ import { createReadStream, createWriteStream } from "node:fs";
|
|
|
2
2
|
import fs from "node:fs/promises";
|
|
3
3
|
import path from "node:path";
|
|
4
4
|
import { assertColumnarHeader, zipFieldRow } from "./normalize.js";
|
|
5
|
-
import { csvEscape, digestFile, formatScalar, stagingPath, writeLine } from "./output.js";
|
|
5
|
+
import { csvEscape, digestFile, formatScalar, stagingPath, writeLine, writeLines, LINES_PER_WRITE } from "./output.js";
|
|
6
6
|
import { extractTitles, MAX_TITLES_PER_ENDPOINT } from "./titleCache.js";
|
|
7
7
|
export const ROW_SINK = Symbol("gangtise.rowSink");
|
|
8
8
|
const STREAM_THRESHOLD = 1000;
|
|
@@ -22,6 +22,7 @@ export class ExportSink {
|
|
|
22
22
|
columns = [];
|
|
23
23
|
columnSet = new Set();
|
|
24
24
|
objectRows = 0;
|
|
25
|
+
earliestDates = new Map();
|
|
25
26
|
partPath;
|
|
26
27
|
rowsPath;
|
|
27
28
|
constructor(outputPath, format = "jsonl", cache) {
|
|
@@ -45,7 +46,27 @@ export class ExportSink {
|
|
|
45
46
|
setFieldList(fields) {
|
|
46
47
|
this.fields = fields;
|
|
47
48
|
}
|
|
49
|
+
watchEarliest(field) {
|
|
50
|
+
if (!this.earliestDates.has(field))
|
|
51
|
+
this.earliestDates.set(field, undefined);
|
|
52
|
+
}
|
|
53
|
+
earliest(field) {
|
|
54
|
+
return this.earliestDates.get(field);
|
|
55
|
+
}
|
|
48
56
|
async push(rows) {
|
|
57
|
+
for (const [field, current] of this.earliestDates) {
|
|
58
|
+
const index = this.fields ? this.fields.indexOf(field) : -1;
|
|
59
|
+
let earliest = current;
|
|
60
|
+
for (const row of rows) {
|
|
61
|
+
const value = Array.isArray(row) ? (index >= 0 ? row[index] : undefined) : row && typeof row === "object" ? row[field] : undefined;
|
|
62
|
+
if (typeof value !== "string")
|
|
63
|
+
continue;
|
|
64
|
+
const day = value.slice(0, 10);
|
|
65
|
+
if (earliest === undefined || day < earliest)
|
|
66
|
+
earliest = day;
|
|
67
|
+
}
|
|
68
|
+
this.earliestDates.set(field, earliest);
|
|
69
|
+
}
|
|
49
70
|
if (this.stream) {
|
|
50
71
|
await this.write(rows);
|
|
51
72
|
return;
|
|
@@ -100,6 +121,7 @@ export class ExportSink {
|
|
|
100
121
|
}
|
|
101
122
|
async write(rows) {
|
|
102
123
|
const stream = this.stream;
|
|
124
|
+
let chunk = [];
|
|
103
125
|
for (const row of rows) {
|
|
104
126
|
let item = row;
|
|
105
127
|
if (Array.isArray(row)) {
|
|
@@ -118,7 +140,11 @@ export class ExportSink {
|
|
|
118
140
|
}
|
|
119
141
|
}
|
|
120
142
|
}
|
|
121
|
-
|
|
143
|
+
chunk.push(JSON.stringify(item));
|
|
144
|
+
if (chunk.length >= LINES_PER_WRITE) {
|
|
145
|
+
await writeLines(stream, chunk);
|
|
146
|
+
chunk = [];
|
|
147
|
+
}
|
|
122
148
|
this.rows++;
|
|
123
149
|
if (this.cache && this.titleCount < MAX_TITLES_PER_ENDPOINT) {
|
|
124
150
|
for (const [id, title] of Object.entries(extractTitles([item], this.cache))) {
|
|
@@ -128,6 +154,7 @@ export class ExportSink {
|
|
|
128
154
|
}
|
|
129
155
|
}
|
|
130
156
|
}
|
|
157
|
+
await writeLines(stream, chunk);
|
|
131
158
|
}
|
|
132
159
|
async assembleCsv() {
|
|
133
160
|
const out = createWriteStream(this.partPath, { encoding: "utf8" });
|
|
@@ -139,17 +166,18 @@ export class ExportSink {
|
|
|
139
166
|
const input = createReadStream(this.rowsPath, { encoding: "utf8" });
|
|
140
167
|
try {
|
|
141
168
|
let index = 0;
|
|
142
|
-
|
|
169
|
+
let csvLines = [];
|
|
170
|
+
const emit = (line) => {
|
|
143
171
|
if (!line)
|
|
144
172
|
return;
|
|
145
173
|
const item = JSON.parse(line);
|
|
146
174
|
if (objectMode) {
|
|
147
175
|
if (!isObjectRow(item))
|
|
148
176
|
return;
|
|
149
|
-
|
|
177
|
+
csvLines.push(columns.map((column) => csvEscape(formatScalar(item[column]))).join(","));
|
|
150
178
|
}
|
|
151
179
|
else {
|
|
152
|
-
|
|
180
|
+
csvLines.push(`${csvEscape(String(index++))},${csvEscape(formatScalar(item))}`);
|
|
153
181
|
}
|
|
154
182
|
};
|
|
155
183
|
let rest = "";
|
|
@@ -157,9 +185,14 @@ export class ExportSink {
|
|
|
157
185
|
const lines = (rest + String(chunk)).split("\n");
|
|
158
186
|
rest = lines.pop() ?? "";
|
|
159
187
|
for (const line of lines)
|
|
160
|
-
|
|
188
|
+
emit(line);
|
|
189
|
+
if (csvLines.length >= LINES_PER_WRITE) {
|
|
190
|
+
await writeLines(out, csvLines);
|
|
191
|
+
csvLines = [];
|
|
192
|
+
}
|
|
161
193
|
}
|
|
162
|
-
|
|
194
|
+
emit(rest);
|
|
195
|
+
await writeLines(out, csvLines);
|
|
163
196
|
await endStream(out);
|
|
164
197
|
}
|
|
165
198
|
catch (error) {
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import fs from "node:fs/promises";
|
|
2
2
|
import os from "node:os";
|
|
3
3
|
import path from "node:path";
|
|
4
|
+
import { stagingPath } from "./output.js";
|
|
4
5
|
export const DEFAULT_TITLE_CACHE_PATH = path.join(os.homedir(), ".config", "gangtise", "title-cache.json");
|
|
5
6
|
export const TITLE_LOOKUP_SIZE = 200;
|
|
6
7
|
const TITLE_CACHE_TTL_MS = 24 * 60 * 60 * 1000;
|
|
@@ -43,6 +44,29 @@ async function loadInto(filePath) {
|
|
|
43
44
|
export async function readTitleCache(filePath = DEFAULT_TITLE_CACHE_PATH) {
|
|
44
45
|
return loadInto(filePath);
|
|
45
46
|
}
|
|
47
|
+
async function mergeFromDisk(filePath, data) {
|
|
48
|
+
let disk;
|
|
49
|
+
try {
|
|
50
|
+
disk = JSON.parse(await fs.readFile(filePath, "utf8"));
|
|
51
|
+
}
|
|
52
|
+
catch {
|
|
53
|
+
return;
|
|
54
|
+
}
|
|
55
|
+
if (!disk || typeof disk !== "object" || Array.isArray(disk))
|
|
56
|
+
return;
|
|
57
|
+
for (const [key, entry] of Object.entries(disk)) {
|
|
58
|
+
if (!entry?.titles || typeof entry.titles !== "object" || typeof entry.ts !== "number")
|
|
59
|
+
continue;
|
|
60
|
+
const mine = data[key];
|
|
61
|
+
if (!mine) {
|
|
62
|
+
data[key] = { titles: entry.titles, ts: entry.ts };
|
|
63
|
+
continue;
|
|
64
|
+
}
|
|
65
|
+
const titles = { ...entry.titles, ...mine.titles };
|
|
66
|
+
data[key] = { titles: capTitles(titles, Object.keys(mine.titles), MAX_TITLES_PER_ENDPOINT), ts: Math.max(entry.ts, mine.ts) };
|
|
67
|
+
}
|
|
68
|
+
pruneCache(data, "", []);
|
|
69
|
+
}
|
|
46
70
|
async function flush(filePath) {
|
|
47
71
|
for (;;) {
|
|
48
72
|
if (!dirty || !memoryCache) {
|
|
@@ -50,9 +74,11 @@ async function flush(filePath) {
|
|
|
50
74
|
return;
|
|
51
75
|
}
|
|
52
76
|
dirty = false;
|
|
53
|
-
const
|
|
77
|
+
const data = memoryCache;
|
|
78
|
+
await mergeFromDisk(filePath, data);
|
|
79
|
+
const snapshot = JSON.stringify(data);
|
|
54
80
|
await fs.mkdir(path.dirname(filePath), { recursive: true });
|
|
55
|
-
const tmp =
|
|
81
|
+
const tmp = stagingPath(filePath, "tmp");
|
|
56
82
|
await fs.writeFile(tmp, snapshot, { encoding: "utf8", mode: 0o600 });
|
|
57
83
|
try {
|
|
58
84
|
await fs.rename(tmp, filePath);
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
+
import { createRequire } from "node:module";
|
|
1
2
|
import { gunzipSync } from "node:zlib";
|
|
2
|
-
import { Agent } from "undici";
|
|
3
3
|
import { ApiError } from "./errors.js";
|
|
4
4
|
export function decodeResponseBody(buf, contentEncoding) {
|
|
5
5
|
const enc = (Array.isArray(contentEncoding) ? contentEncoding[0] : contentEncoding)?.toLowerCase().trim();
|
|
@@ -34,12 +34,14 @@ function retryAfterFromError(error) {
|
|
|
34
34
|
return undefined;
|
|
35
35
|
}
|
|
36
36
|
let cachedDispatcher = null;
|
|
37
|
+
const requireLazily = createRequire(import.meta.url);
|
|
37
38
|
export function getDispatcher() {
|
|
38
39
|
if (!cachedDispatcher) {
|
|
40
|
+
const { Agent } = requireLazily("undici");
|
|
39
41
|
cachedDispatcher = new Agent({
|
|
40
42
|
keepAliveTimeout: 60_000,
|
|
41
43
|
keepAliveMaxTimeout: 600_000,
|
|
42
|
-
connections: 16,
|
|
44
|
+
connections: Math.max(16, PAGE_CONCURRENCY),
|
|
43
45
|
pipelining: 1,
|
|
44
46
|
});
|
|
45
47
|
}
|
|
@@ -62,8 +64,9 @@ export async function runWithConcurrency(items, concurrency, fn) {
|
|
|
62
64
|
await Promise.all(Array.from({ length: limit }, () => worker()));
|
|
63
65
|
return results;
|
|
64
66
|
}
|
|
65
|
-
export async function runInOrder(items, concurrency, fn, consume) {
|
|
67
|
+
export async function runInOrder(items, concurrency, fn, consume, window = concurrency) {
|
|
66
68
|
const width = Math.max(1, concurrency);
|
|
69
|
+
const maxPending = Math.max(width, window);
|
|
67
70
|
const pending = new Map();
|
|
68
71
|
let next = 0;
|
|
69
72
|
let failure = null;
|
|
@@ -104,7 +107,7 @@ export async function runInOrder(items, concurrency, fn, consume) {
|
|
|
104
107
|
return;
|
|
105
108
|
pending.set(index, result);
|
|
106
109
|
drain();
|
|
107
|
-
while (!failure && pending.size >=
|
|
110
|
+
while (!failure && pending.size >= maxPending) {
|
|
108
111
|
await new Promise((resolve) => waiters.push(resolve));
|
|
109
112
|
}
|
|
110
113
|
});
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
|
2
|
+
import os from "node:os";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { isVersionNewer } from "./args.js";
|
|
5
|
+
import { stagingPath } from "./output.js";
|
|
6
|
+
export const UPDATE_CHECK_TTL_MS = 24 * 60 * 60 * 1000;
|
|
7
|
+
export const DEFAULT_UPDATE_CHECK_PATH = path.join(os.homedir(), ".config", "gangtise", "update-check.json");
|
|
8
|
+
function saveLatest(cachePath, latest, now) {
|
|
9
|
+
const staging = stagingPath(cachePath, "tmp");
|
|
10
|
+
try {
|
|
11
|
+
mkdirSync(path.dirname(cachePath), { recursive: true });
|
|
12
|
+
writeFileSync(staging, JSON.stringify({ checkedAt: now, latest }));
|
|
13
|
+
renameSync(staging, cachePath);
|
|
14
|
+
}
|
|
15
|
+
catch {
|
|
16
|
+
try {
|
|
17
|
+
rmSync(staging, { force: true });
|
|
18
|
+
}
|
|
19
|
+
catch { }
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
export async function latestVersion(cachePath = DEFAULT_UPDATE_CHECK_PATH, timeoutMs = 2000, now = Date.now()) {
|
|
23
|
+
const cacheable = path.isAbsolute(cachePath);
|
|
24
|
+
if (cacheable) {
|
|
25
|
+
try {
|
|
26
|
+
const cached = JSON.parse(readFileSync(cachePath, "utf8"));
|
|
27
|
+
const age = typeof cached.checkedAt === "number" ? now - cached.checkedAt : -1;
|
|
28
|
+
if (typeof cached.latest === "string" && age >= 0 && age < UPDATE_CHECK_TTL_MS)
|
|
29
|
+
return cached.latest;
|
|
30
|
+
}
|
|
31
|
+
catch { }
|
|
32
|
+
}
|
|
33
|
+
try {
|
|
34
|
+
const response = await fetch("https://registry.npmjs.org/gangtise-openapi-cli/latest", { signal: AbortSignal.timeout(timeoutMs) });
|
|
35
|
+
const latest = (await response.json()).version;
|
|
36
|
+
if (typeof latest !== "string")
|
|
37
|
+
return undefined;
|
|
38
|
+
if (cacheable)
|
|
39
|
+
saveLatest(cachePath, latest, now);
|
|
40
|
+
return latest;
|
|
41
|
+
}
|
|
42
|
+
catch {
|
|
43
|
+
return undefined;
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
export async function checkForUpdate(current) {
|
|
47
|
+
const latest = await latestVersion();
|
|
48
|
+
if (latest && isVersionNewer(latest, current)) {
|
|
49
|
+
process.stderr.write(`Update available: ${current} → ${latest}\nRun: npm update -g gangtise-openapi-cli\n`);
|
|
50
|
+
}
|
|
51
|
+
}
|
package/dist/src/version.js
CHANGED
|
@@ -1 +1,19 @@
|
|
|
1
|
-
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { fileURLToPath } from "node:url";
|
|
4
|
+
function packageVersion() {
|
|
5
|
+
let dir = path.dirname(fileURLToPath(import.meta.url));
|
|
6
|
+
for (;;) {
|
|
7
|
+
const file = path.join(dir, "package.json");
|
|
8
|
+
if (fs.existsSync(file)) {
|
|
9
|
+
const pkg = JSON.parse(fs.readFileSync(file, "utf8"));
|
|
10
|
+
if (pkg.name === "gangtise-openapi-cli" && pkg.version)
|
|
11
|
+
return pkg.version;
|
|
12
|
+
}
|
|
13
|
+
const parent = path.dirname(dir);
|
|
14
|
+
if (parent === dir)
|
|
15
|
+
throw new Error("gangtise-openapi-cli: its package.json was not found above " + fileURLToPath(import.meta.url));
|
|
16
|
+
dir = parent;
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
export const CLI_VERSION = packageVersion();
|