gangtise-openapi-cli 0.43.1 → 0.44.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +86 -15
- package/dist/src/cli.js +29 -18
- package/dist/src/commands/ai.js +20 -29
- package/dist/src/commands/auth.js +5 -4
- package/dist/src/commands/bond.js +5 -4
- package/dist/src/commands/fund.js +100 -0
- package/dist/src/commands/fundamental.js +2 -0
- package/dist/src/commands/groups.js +14 -0
- package/dist/src/commands/indicator.js +5 -5
- package/dist/src/commands/insight.js +3 -4
- package/dist/src/commands/raw.js +4 -3
- package/dist/src/commands/shared.js +35 -6
- package/dist/src/commands/tool.js +2 -2
- package/dist/src/commands/vault.js +30 -30
- package/dist/src/core/args.js +23 -7
- package/dist/src/core/asyncContent.js +1 -1
- package/dist/src/core/auth.js +2 -10
- package/dist/src/core/client.js +67 -19
- package/dist/src/core/commandBodies.js +1 -27
- package/dist/src/core/config.js +9 -5
- package/dist/src/core/download.js +47 -10
- package/dist/src/core/endpoints.js +195 -0
- package/dist/src/core/errors.js +1 -1
- package/dist/src/core/output.js +103 -72
- package/dist/src/core/printer.js +10 -5
- package/dist/src/core/quoteSharding.js +28 -4
- package/dist/src/core/rowSink.js +24 -10
- package/dist/src/core/titleCache.js +2 -10
- package/dist/src/core/transport.js +25 -7
- package/dist/src/core/updateCheck.js +3 -9
- package/gangtise-openapi/SKILL.md +52 -16
- package/gangtise-openapi/references/commands/fund.md +192 -0
- package/gangtise-openapi/references/commands/fundamental.md +2 -2
- package/gangtise-openapi/references/commands/insight.md +2 -2
- package/gangtise-openapi/references/commands/quote.md +1 -1
- package/gangtise-openapi/references/commands/tool.md +1 -1
- package/gangtise-openapi/references/commands/vault.md +1 -1
- package/gangtise-openapi/references/errors.md +4 -4
- package/gangtise-openapi/references/response-schema.md +10 -0
- package/package.json +1 -1
package/dist/src/core/output.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { createHash, randomBytes } from "node:crypto";
|
|
2
|
-
import { rmSync, statSync } from "node:fs";
|
|
2
|
+
import { renameSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
3
3
|
import fs from "node:fs/promises";
|
|
4
|
+
import path from "node:path";
|
|
4
5
|
import { ConfigError } from "./errors.js";
|
|
5
6
|
const OUTPUT_FORMATS = ["table", "json", "jsonl", "csv", "markdown"];
|
|
6
7
|
export function stagingPath(target, suffix = "part") {
|
|
@@ -8,6 +9,46 @@ export function stagingPath(target, suffix = "part") {
|
|
|
8
9
|
handedOut.add(staging);
|
|
9
10
|
return staging;
|
|
10
11
|
}
|
|
12
|
+
export async function writeFileAtomic(target, content, options = {}) {
|
|
13
|
+
const staging = stagingPath(target, options.suffix);
|
|
14
|
+
try {
|
|
15
|
+
await fs.writeFile(staging, content, options.mode === undefined ? undefined : { mode: options.mode });
|
|
16
|
+
await publishFile(staging, target);
|
|
17
|
+
}
|
|
18
|
+
catch (error) {
|
|
19
|
+
await fs.unlink(staging).catch(() => { });
|
|
20
|
+
throw error;
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
const publishedIdentity = new Map();
|
|
24
|
+
export async function publishFile(staging, target) {
|
|
25
|
+
const { dev, ino } = await fs.stat(staging);
|
|
26
|
+
await fs.rename(staging, target);
|
|
27
|
+
publishedIdentity.set(path.resolve(target), { dev, ino });
|
|
28
|
+
}
|
|
29
|
+
export async function stillPublished(target) {
|
|
30
|
+
const mine = publishedIdentity.get(path.resolve(target));
|
|
31
|
+
if (!mine || !mine.ino)
|
|
32
|
+
return undefined;
|
|
33
|
+
const now = await fs.stat(target).catch(() => undefined);
|
|
34
|
+
if (!now)
|
|
35
|
+
return undefined;
|
|
36
|
+
return now.dev === mine.dev && now.ino === mine.ino;
|
|
37
|
+
}
|
|
38
|
+
export function writeFileAtomicSync(target, content, options = {}) {
|
|
39
|
+
const staging = stagingPath(target, options.suffix);
|
|
40
|
+
try {
|
|
41
|
+
writeFileSync(staging, content);
|
|
42
|
+
renameSync(staging, target);
|
|
43
|
+
}
|
|
44
|
+
catch (error) {
|
|
45
|
+
try {
|
|
46
|
+
rmSync(staging, { force: true });
|
|
47
|
+
}
|
|
48
|
+
catch { }
|
|
49
|
+
throw error;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
11
52
|
const handedOut = new Set();
|
|
12
53
|
const claimed = new Set();
|
|
13
54
|
export function trackClaim(name) {
|
|
@@ -117,13 +158,14 @@ function clampCell(value) {
|
|
|
117
158
|
}
|
|
118
159
|
return out + "…";
|
|
119
160
|
}
|
|
120
|
-
function renderTable(rows) {
|
|
161
|
+
function renderTable(rows, clamp) {
|
|
121
162
|
if (rows.length === 0) {
|
|
122
163
|
return "(empty)";
|
|
123
164
|
}
|
|
165
|
+
const cell = clamp ? clampCell : (value) => value;
|
|
124
166
|
const columns = Array.from(new Set(rows.flatMap((row) => Object.keys(row))));
|
|
125
|
-
const headerCells = columns.map((column) =>
|
|
126
|
-
const matrix = rows.map((row) => columns.map((column) =>
|
|
167
|
+
const headerCells = columns.map((column) => cell(sanitizeCell(column)));
|
|
168
|
+
const matrix = rows.map((row) => columns.map((column) => cell(sanitizeCell(formatScalar(row[column])))));
|
|
127
169
|
const cellWidths = matrix.map((cells) => cells.map(displayWidth));
|
|
128
170
|
const headerWidths = headerCells.map(displayWidth);
|
|
129
171
|
const widths = columns.map((_, c) => cellWidths.reduce((max, cells) => Math.max(max, cells[c]), headerWidths[c]));
|
|
@@ -144,14 +186,25 @@ function renderMarkdown(rows) {
|
|
|
144
186
|
const body = rows.map((row) => `| ${columns.map((column) => mdEscape(sanitizeCell(formatScalar(row[column])))).join(" | ")} |`);
|
|
145
187
|
return [header, divider, ...body].join("\n");
|
|
146
188
|
}
|
|
147
|
-
function
|
|
148
|
-
|
|
149
|
-
|
|
189
|
+
export function csvHeader(columns) {
|
|
190
|
+
return columns.map(csvEscape).join(",");
|
|
191
|
+
}
|
|
192
|
+
export function csvRow(columns, row) {
|
|
193
|
+
return columns.map((column) => csvEscape(formatScalar(row[column]))).join(",");
|
|
194
|
+
}
|
|
195
|
+
export function* outputLines(value, format, columns) {
|
|
196
|
+
if (format === "jsonl") {
|
|
197
|
+
for (const item of jsonlItems(value))
|
|
198
|
+
yield JSON.stringify(item);
|
|
199
|
+
return;
|
|
150
200
|
}
|
|
151
|
-
const
|
|
152
|
-
const header =
|
|
153
|
-
|
|
154
|
-
|
|
201
|
+
const rows = toRows(value);
|
|
202
|
+
const header = rows.length > 0 ? Array.from(new Set(rows.flatMap((row) => Object.keys(row)))) : (columns ?? []).map(String);
|
|
203
|
+
if (header.length === 0)
|
|
204
|
+
return;
|
|
205
|
+
yield csvHeader(header);
|
|
206
|
+
for (const row of rows)
|
|
207
|
+
yield csvRow(header, row);
|
|
155
208
|
}
|
|
156
209
|
export function jsonlItems(value) {
|
|
157
210
|
const list = value && typeof value === "object" && !Array.isArray(value) ? value.list : undefined;
|
|
@@ -160,71 +213,61 @@ export function jsonlItems(value) {
|
|
|
160
213
|
export function countOutputRows(value, format) {
|
|
161
214
|
return format === "jsonl" ? jsonlItems(value).length : toRows(value).length;
|
|
162
215
|
}
|
|
163
|
-
export function renderOutput(value, format) {
|
|
216
|
+
export function renderOutput(value, format, options = {}) {
|
|
164
217
|
switch (format) {
|
|
165
218
|
case "json":
|
|
166
219
|
return JSON.stringify(value, null, 2);
|
|
167
220
|
case "jsonl":
|
|
168
|
-
return jsonlItems(value).map((item) => JSON.stringify(item)).join("\n");
|
|
169
221
|
case "csv":
|
|
170
|
-
return
|
|
222
|
+
return [...outputLines(value, format, options.columns)].join("\n");
|
|
171
223
|
case "markdown":
|
|
172
224
|
return renderMarkdown(toRows(value));
|
|
173
225
|
case "table":
|
|
174
226
|
default:
|
|
175
|
-
return renderTable(toRows(value));
|
|
227
|
+
return renderTable(toRows(value), options.clampCells ?? true);
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
export async function writeOutputLines(stream, value, format, options = {}) {
|
|
231
|
+
const hash = options.digest ? createHash("sha256") : undefined;
|
|
232
|
+
let bytes = 0;
|
|
233
|
+
const flush = async (lines) => {
|
|
234
|
+
if (lines.length === 0)
|
|
235
|
+
return;
|
|
236
|
+
const text = lines.join("\n");
|
|
237
|
+
if (hash) {
|
|
238
|
+
const buf = Buffer.from(`${text}\n`, "utf8");
|
|
239
|
+
hash.update(buf);
|
|
240
|
+
bytes += buf.byteLength;
|
|
241
|
+
}
|
|
242
|
+
await writeLine(stream, text);
|
|
243
|
+
};
|
|
244
|
+
let chunk = [];
|
|
245
|
+
let first = true;
|
|
246
|
+
for (const line of outputLines(value, format, options.columns)) {
|
|
247
|
+
chunk.push(first && options.bom && format === "csv" ? `\ufeff${line}` : line);
|
|
248
|
+
first = false;
|
|
249
|
+
if (chunk.length >= LINES_PER_WRITE) {
|
|
250
|
+
await flush(chunk);
|
|
251
|
+
chunk = [];
|
|
252
|
+
}
|
|
176
253
|
}
|
|
254
|
+
await flush(chunk);
|
|
255
|
+
return hash ? { bytes, sha256: hash.digest("hex") } : undefined;
|
|
177
256
|
}
|
|
178
|
-
export async function streamOutputToFile(value, format, outputPath) {
|
|
257
|
+
export async function streamOutputToFile(value, format, outputPath, columns) {
|
|
179
258
|
if (format !== "jsonl" && format !== "csv")
|
|
180
259
|
return null;
|
|
181
|
-
const list = pickList(value);
|
|
182
|
-
if (!list)
|
|
183
|
-
return null;
|
|
184
|
-
if (list.length < 1000)
|
|
185
|
-
return null;
|
|
186
|
-
let csvRows = [];
|
|
187
|
-
let csvColumns = [];
|
|
188
|
-
if (format === "csv") {
|
|
189
|
-
csvRows = list.filter((row) => Boolean(row && typeof row === "object" && !Array.isArray(row)));
|
|
190
|
-
if (csvRows.length === 0)
|
|
191
|
-
return null;
|
|
192
|
-
csvColumns = Array.from(new Set(csvRows.flatMap((row) => Object.keys(row))));
|
|
193
|
-
}
|
|
194
|
-
const { dirname } = await import("node:path");
|
|
195
260
|
const { createWriteStream } = await import("node:fs");
|
|
196
|
-
await fs.mkdir(dirname(outputPath), { recursive: true });
|
|
261
|
+
await fs.mkdir(path.dirname(outputPath), { recursive: true });
|
|
197
262
|
const partPath = stagingPath(outputPath);
|
|
198
263
|
const stream = createWriteStream(partPath, { encoding: "utf8" });
|
|
199
264
|
stream.on("error", () => { });
|
|
200
265
|
try {
|
|
201
|
-
|
|
202
|
-
let chunk = [];
|
|
203
|
-
for (const item of jsonlItems(value)) {
|
|
204
|
-
chunk.push(JSON.stringify(item));
|
|
205
|
-
if (chunk.length >= LINES_PER_WRITE) {
|
|
206
|
-
await writeLines(stream, chunk);
|
|
207
|
-
chunk = [];
|
|
208
|
-
}
|
|
209
|
-
}
|
|
210
|
-
await writeLines(stream, chunk);
|
|
211
|
-
}
|
|
212
|
-
else {
|
|
213
|
-
let chunk = ["\ufeff" + csvColumns.map(csvEscape).join(",")];
|
|
214
|
-
for (const row of csvRows) {
|
|
215
|
-
chunk.push(csvColumns.map((column) => csvEscape(formatScalar(row[column]))).join(","));
|
|
216
|
-
if (chunk.length >= LINES_PER_WRITE) {
|
|
217
|
-
await writeLines(stream, chunk);
|
|
218
|
-
chunk = [];
|
|
219
|
-
}
|
|
220
|
-
}
|
|
221
|
-
await writeLines(stream, chunk);
|
|
222
|
-
}
|
|
266
|
+
const digest = await writeOutputLines(stream, value, format, { columns, bom: true, digest: true });
|
|
223
267
|
await new Promise((resolve, reject) => {
|
|
224
268
|
stream.end((err) => err ? reject(err) : resolve());
|
|
225
269
|
});
|
|
226
|
-
|
|
227
|
-
await fs.rename(partPath, outputPath);
|
|
270
|
+
await publishFile(partPath, outputPath);
|
|
228
271
|
return digest;
|
|
229
272
|
}
|
|
230
273
|
catch (error) {
|
|
@@ -246,9 +289,10 @@ export function pickList(value) {
|
|
|
246
289
|
}
|
|
247
290
|
return null;
|
|
248
291
|
}
|
|
292
|
+
const NUMERIC_LOOKING = /^[+\-](?:(?=[^\d]*\d)[\d.,%eE]+)?$/;
|
|
249
293
|
export function csvEscape(value) {
|
|
250
294
|
let out = value;
|
|
251
|
-
if (/^[=@\t\r]/.test(out) || (/^[+\-]/.test(out) && !
|
|
295
|
+
if (/^[=@\t\r]/.test(out) || (/^[+\-]/.test(out) && !NUMERIC_LOOKING.test(out)))
|
|
252
296
|
out = "'" + out;
|
|
253
297
|
if (/[",\n\r]/.test(out))
|
|
254
298
|
return `"${out.replaceAll("\"", "\"\"")}"`;
|
|
@@ -277,20 +321,7 @@ export async function saveOutputIfNeeded(content, outputPath, withDigest = true)
|
|
|
277
321
|
}
|
|
278
322
|
const { dirname } = await import("node:path");
|
|
279
323
|
await fs.mkdir(dirname(outputPath), { recursive: true });
|
|
280
|
-
const
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
await fs.writeFile(partPath, content, "utf8");
|
|
284
|
-
}
|
|
285
|
-
else {
|
|
286
|
-
await fs.writeFile(partPath, content);
|
|
287
|
-
}
|
|
288
|
-
const digest = withDigest ? digestBuffer(content) : undefined;
|
|
289
|
-
await fs.rename(partPath, outputPath);
|
|
290
|
-
return digest;
|
|
291
|
-
}
|
|
292
|
-
catch (error) {
|
|
293
|
-
await fs.unlink(partPath).catch(() => { });
|
|
294
|
-
throw error;
|
|
295
|
-
}
|
|
324
|
+
const digest = withDigest ? digestBuffer(content) : undefined;
|
|
325
|
+
await writeFileAtomic(outputPath, content);
|
|
326
|
+
return digest;
|
|
296
327
|
}
|
package/dist/src/core/printer.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import fs from "node:fs/promises";
|
|
2
2
|
import path from "node:path";
|
|
3
3
|
import { normalizeRows } from "./normalize.js";
|
|
4
|
-
import { countOutputRows, digestFile, pickList, renderOutput, saveOutputIfNeeded, stagingPath, streamOutputToFile } from "./output.js";
|
|
4
|
+
import { countOutputRows, digestFile, pickList, renderOutput, saveOutputIfNeeded, stagingPath, stillPublished, streamOutputToFile, writeOutputLines } from "./output.js";
|
|
5
5
|
import { getRowSink } from "./rowSink.js";
|
|
6
6
|
import { extractTitles, writeTitleCache } from "./titleCache.js";
|
|
7
7
|
import { CLI_VERSION } from "../version.js";
|
|
@@ -9,11 +9,13 @@ import { decidedExitCode, EXIT_SUPERSEDED, markIncomplete, markSuperseded } from
|
|
|
9
9
|
const LARGE_RESULT_ROWS = 50_000;
|
|
10
10
|
export const SUPERSEDED_EXIT = EXIT_SUPERSEDED;
|
|
11
11
|
export async function warnIfSuperseded(output, digest) {
|
|
12
|
+
if (await stillPublished(output) === true)
|
|
13
|
+
return;
|
|
12
14
|
const actual = await digestFile(output).catch(() => undefined);
|
|
13
15
|
if (!actual || actual.sha256 === digest.sha256)
|
|
14
16
|
return;
|
|
15
17
|
process.stderr.write(`[gangtise] warning: ${output} was replaced by another export to the same path while this one was finishing — the file there is not this command's output `
|
|
16
|
-
+ `(wrote sha256 ${digest.sha256.slice(0, 12)}
|
|
18
|
+
+ `(this run wrote sha256 ${digest.sha256.slice(0, 12)}…). Give concurrent exports distinct --output paths.\n`);
|
|
17
19
|
markSuperseded();
|
|
18
20
|
}
|
|
19
21
|
function warnIfLargeInMemory(items, format) {
|
|
@@ -135,10 +137,9 @@ export async function printData(data, format, output, cache) {
|
|
|
135
137
|
await streamed.finish();
|
|
136
138
|
digest = streamed.digest;
|
|
137
139
|
}
|
|
138
|
-
else if (!(digest = (await streamOutputToFile(normalized, format, output)) ?? undefined)) {
|
|
140
|
+
else if (!(digest = (await streamOutputToFile(normalized, format, output, columns)) ?? undefined)) {
|
|
139
141
|
warnIfLargeInMemory(items, format);
|
|
140
|
-
|
|
141
|
-
digest = await saveOutputIfNeeded(format === "csv" ? `\ufeff${content}` : content, output);
|
|
142
|
+
digest = await saveOutputIfNeeded(renderOutput(normalized, format, { clampCells: false }), output);
|
|
142
143
|
}
|
|
143
144
|
}
|
|
144
145
|
catch (error) {
|
|
@@ -151,6 +152,10 @@ export async function printData(data, format, output, cache) {
|
|
|
151
152
|
process.stdout.write(`${output}\n`);
|
|
152
153
|
return;
|
|
153
154
|
}
|
|
155
|
+
if (format === "jsonl" || format === "csv") {
|
|
156
|
+
await writeOutputLines(process.stdout, normalized, format, { columns });
|
|
157
|
+
return;
|
|
158
|
+
}
|
|
154
159
|
warnIfLargeInMemory(items, format);
|
|
155
160
|
const rendered = renderOutput(normalized, format);
|
|
156
161
|
if (rendered)
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { isStructuralError } from "./errors.js";
|
|
1
|
+
import { ApiError, isStructuralError } from "./errors.js";
|
|
2
2
|
import { columnarSchemaValid } from "./normalize.js";
|
|
3
3
|
import { attachRowSink } from "./rowSink.js";
|
|
4
4
|
import { isVerbose, PAGE_CONCURRENCY, runInOrder } from "./transport.js";
|
|
@@ -60,6 +60,12 @@ function buildShards(start, end, shardDays) {
|
|
|
60
60
|
flush();
|
|
61
61
|
return shards;
|
|
62
62
|
}
|
|
63
|
+
const OUT_OF_WINDOW_CODE = "110003";
|
|
64
|
+
function describeError(error) {
|
|
65
|
+
if (error instanceof ApiError)
|
|
66
|
+
return `${error.code ? `(${error.code}) ` : ""}${error.message}`;
|
|
67
|
+
return error instanceof Error ? error.message : String(error);
|
|
68
|
+
}
|
|
63
69
|
export async function callKlineWithSharding(client, endpointKey, body, config) {
|
|
64
70
|
const fullMarketValue = config.fullMarketValue ?? "all";
|
|
65
71
|
if (!isFullMarket(body, fullMarketValue)) {
|
|
@@ -101,9 +107,13 @@ export async function callKlineWithSharding(client, endpointKey, body, config) {
|
|
|
101
107
|
const failedShards = [];
|
|
102
108
|
let firstError = null;
|
|
103
109
|
let aborted = false;
|
|
110
|
+
let unsent = 0;
|
|
111
|
+
const outOfWindow = [];
|
|
112
|
+
let windowError = null;
|
|
104
113
|
const fetchShard = async (shard) => {
|
|
105
114
|
if (aborted) {
|
|
106
115
|
failedShards.push(shard);
|
|
116
|
+
unsent++;
|
|
107
117
|
return null;
|
|
108
118
|
}
|
|
109
119
|
try {
|
|
@@ -115,6 +125,12 @@ export async function callKlineWithSharding(client, endpointKey, body, config) {
|
|
|
115
125
|
return res;
|
|
116
126
|
}
|
|
117
127
|
catch (error) {
|
|
128
|
+
if (error instanceof ApiError && error.code === OUT_OF_WINDOW_CODE) {
|
|
129
|
+
outOfWindow.push(shard);
|
|
130
|
+
if (!windowError)
|
|
131
|
+
windowError = error;
|
|
132
|
+
return null;
|
|
133
|
+
}
|
|
118
134
|
if (!firstError)
|
|
119
135
|
firstError = error;
|
|
120
136
|
if (!isStructuralError(error))
|
|
@@ -193,13 +209,19 @@ export async function callKlineWithSharding(client, endpointKey, body, config) {
|
|
|
193
209
|
await keep(map ? rows.map((item) => (Array.isArray(item) ? map.map((k) => item[k]) : item)) : rows);
|
|
194
210
|
};
|
|
195
211
|
await runInOrder(shards, config.concurrency ?? PAGE_CONCURRENCY, fetchShard, mergeShard);
|
|
196
|
-
if (failedShards.length === shards.length) {
|
|
197
|
-
throw firstError ?? new Error(`All ${shards.length} kline shards failed`);
|
|
212
|
+
if (failedShards.length + outOfWindow.length === shards.length) {
|
|
213
|
+
throw firstError ?? windowError ?? new Error(`All ${shards.length} kline shards failed`);
|
|
214
|
+
}
|
|
215
|
+
const skipped = [...outOfWindow].sort((a, b) => a.startDate.localeCompare(b.startDate));
|
|
216
|
+
if (skipped.length > 0) {
|
|
217
|
+
process.stderr.write(`[gangtise] note: ${skipped.length}/${shards.length} shard(s) between ${skipped[0].startDate} and ${skipped[skipped.length - 1].endDate} lie outside your account's history window (110003) and were skipped; the result covers the rest of the range (see outOfWindowShards).\n`);
|
|
198
218
|
}
|
|
199
219
|
const base = header ?? fallback ?? {};
|
|
200
220
|
const out = { ...base, total: count, list: merged };
|
|
201
221
|
if (sink)
|
|
202
222
|
attachRowSink(out, sink);
|
|
223
|
+
if (skipped.length > 0)
|
|
224
|
+
out.outOfWindowShards = skipped;
|
|
203
225
|
if (fieldList) {
|
|
204
226
|
out.fieldList = fieldList;
|
|
205
227
|
}
|
|
@@ -215,7 +237,9 @@ export async function callKlineWithSharding(client, endpointKey, body, config) {
|
|
|
215
237
|
if (failedShards.length > 0) {
|
|
216
238
|
out.partial = true;
|
|
217
239
|
out.failedShards = failedShards;
|
|
218
|
-
|
|
240
|
+
const notSent = unsent > 0 ? ` (${unsent} of them not sent: the first error stopped the rest)` : "";
|
|
241
|
+
const cause = firstError ? `; first error: ${describeError(firstError)}` : "";
|
|
242
|
+
process.stderr.write(`[gangtise] warning: ${failedShards.length}/${shards.length} shards failed${notSent}${cause}; results are partial (see failedShards)\n`);
|
|
219
243
|
}
|
|
220
244
|
if (partialShards.length > 0) {
|
|
221
245
|
out.partial = true;
|
package/dist/src/core/rowSink.js
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
1
2
|
import { createReadStream, createWriteStream } from "node:fs";
|
|
2
3
|
import fs from "node:fs/promises";
|
|
3
4
|
import path from "node:path";
|
|
4
5
|
import { assertColumnarHeader, zipFieldRow } from "./normalize.js";
|
|
5
|
-
import {
|
|
6
|
+
import { csvHeader, csvRow, publishFile, stagingPath, writeLine, LINES_PER_WRITE } from "./output.js";
|
|
6
7
|
import { extractTitles, MAX_TITLES_PER_ENDPOINT } from "./titleCache.js";
|
|
7
8
|
export const ROW_SINK = Symbol("gangtise.rowSink");
|
|
8
9
|
const STREAM_THRESHOLD = 1000;
|
|
@@ -25,6 +26,8 @@ export class ExportSink {
|
|
|
25
26
|
earliestDates = new Map();
|
|
26
27
|
partPath;
|
|
27
28
|
rowsPath;
|
|
29
|
+
hash = createHash("sha256");
|
|
30
|
+
bytes = 0;
|
|
28
31
|
constructor(outputPath, format = "jsonl", cache) {
|
|
29
32
|
this.outputPath = outputPath;
|
|
30
33
|
this.format = format;
|
|
@@ -93,8 +96,8 @@ export class ExportSink {
|
|
|
93
96
|
await endStream(stream);
|
|
94
97
|
if (this.format === "csv")
|
|
95
98
|
await this.assembleCsv();
|
|
96
|
-
this.digest =
|
|
97
|
-
await
|
|
99
|
+
this.digest = { bytes: this.bytes, sha256: this.hash.digest("hex") };
|
|
100
|
+
await publishFile(this.partPath, this.outputPath);
|
|
98
101
|
this.finished = true;
|
|
99
102
|
}
|
|
100
103
|
catch (error) {
|
|
@@ -142,7 +145,7 @@ export class ExportSink {
|
|
|
142
145
|
}
|
|
143
146
|
chunk.push(JSON.stringify(item));
|
|
144
147
|
if (chunk.length >= LINES_PER_WRITE) {
|
|
145
|
-
await
|
|
148
|
+
await this.writeChunk(stream, chunk, this.format === "jsonl");
|
|
146
149
|
chunk = [];
|
|
147
150
|
}
|
|
148
151
|
this.rows++;
|
|
@@ -154,7 +157,18 @@ export class ExportSink {
|
|
|
154
157
|
}
|
|
155
158
|
}
|
|
156
159
|
}
|
|
157
|
-
await
|
|
160
|
+
await this.writeChunk(stream, chunk, this.format === "jsonl");
|
|
161
|
+
}
|
|
162
|
+
async writeChunk(stream, lines, published) {
|
|
163
|
+
if (lines.length === 0)
|
|
164
|
+
return;
|
|
165
|
+
const text = lines.join("\n");
|
|
166
|
+
if (published) {
|
|
167
|
+
const buf = Buffer.from(`${text}\n`, "utf8");
|
|
168
|
+
this.hash.update(buf);
|
|
169
|
+
this.bytes += buf.byteLength;
|
|
170
|
+
}
|
|
171
|
+
await writeLine(stream, text);
|
|
158
172
|
}
|
|
159
173
|
async assembleCsv() {
|
|
160
174
|
const out = createWriteStream(this.partPath, { encoding: "utf8" });
|
|
@@ -162,7 +176,7 @@ export class ExportSink {
|
|
|
162
176
|
this.stream = out;
|
|
163
177
|
const objectMode = this.objectRows > 0;
|
|
164
178
|
const columns = objectMode ? this.columns : ["index", "value"];
|
|
165
|
-
await
|
|
179
|
+
await this.writeChunk(out, [`\ufeff${csvHeader(columns)}`], true);
|
|
166
180
|
const input = createReadStream(this.rowsPath, { encoding: "utf8" });
|
|
167
181
|
try {
|
|
168
182
|
let index = 0;
|
|
@@ -174,10 +188,10 @@ export class ExportSink {
|
|
|
174
188
|
if (objectMode) {
|
|
175
189
|
if (!isObjectRow(item))
|
|
176
190
|
return;
|
|
177
|
-
csvLines.push(columns
|
|
191
|
+
csvLines.push(csvRow(columns, item));
|
|
178
192
|
}
|
|
179
193
|
else {
|
|
180
|
-
csvLines.push(
|
|
194
|
+
csvLines.push(csvRow(columns, { index: index++, value: item }));
|
|
181
195
|
}
|
|
182
196
|
};
|
|
183
197
|
let rest = "";
|
|
@@ -187,12 +201,12 @@ export class ExportSink {
|
|
|
187
201
|
for (const line of lines)
|
|
188
202
|
emit(line);
|
|
189
203
|
if (csvLines.length >= LINES_PER_WRITE) {
|
|
190
|
-
await
|
|
204
|
+
await this.writeChunk(out, csvLines, true);
|
|
191
205
|
csvLines = [];
|
|
192
206
|
}
|
|
193
207
|
}
|
|
194
208
|
emit(rest);
|
|
195
|
-
await
|
|
209
|
+
await this.writeChunk(out, csvLines, true);
|
|
196
210
|
await endStream(out);
|
|
197
211
|
}
|
|
198
212
|
catch (error) {
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import fs from "node:fs/promises";
|
|
2
2
|
import os from "node:os";
|
|
3
3
|
import path from "node:path";
|
|
4
|
-
import {
|
|
4
|
+
import { writeFileAtomic } from "./output.js";
|
|
5
5
|
export const DEFAULT_TITLE_CACHE_PATH = path.join(os.homedir(), ".config", "gangtise", "title-cache.json");
|
|
6
6
|
export const TITLE_LOOKUP_SIZE = 200;
|
|
7
7
|
const TITLE_CACHE_TTL_MS = 24 * 60 * 60 * 1000;
|
|
@@ -78,15 +78,7 @@ async function flush(filePath) {
|
|
|
78
78
|
await mergeFromDisk(filePath, data);
|
|
79
79
|
const snapshot = JSON.stringify(data);
|
|
80
80
|
await fs.mkdir(path.dirname(filePath), { recursive: true });
|
|
81
|
-
|
|
82
|
-
await fs.writeFile(tmp, snapshot, { encoding: "utf8", mode: 0o600 });
|
|
83
|
-
try {
|
|
84
|
-
await fs.rename(tmp, filePath);
|
|
85
|
-
}
|
|
86
|
-
catch (error) {
|
|
87
|
-
await fs.unlink(tmp).catch(() => { });
|
|
88
|
-
throw error;
|
|
89
|
-
}
|
|
81
|
+
await writeFileAtomic(filePath, snapshot, { mode: 0o600, suffix: "tmp" });
|
|
90
82
|
}
|
|
91
83
|
}
|
|
92
84
|
function capTitles(merged, freshKeys, cap) {
|
|
@@ -53,12 +53,19 @@ export async function runWithConcurrency(items, concurrency, fn) {
|
|
|
53
53
|
const limit = Math.max(1, Math.min(concurrency, items.length));
|
|
54
54
|
const results = new Array(items.length);
|
|
55
55
|
let next = 0;
|
|
56
|
+
let failed = false;
|
|
56
57
|
async function worker() {
|
|
57
|
-
while (
|
|
58
|
+
while (!failed) {
|
|
58
59
|
const index = next++;
|
|
59
60
|
if (index >= items.length)
|
|
60
61
|
return;
|
|
61
|
-
|
|
62
|
+
try {
|
|
63
|
+
results[index] = await fn(items[index], index);
|
|
64
|
+
}
|
|
65
|
+
catch (error) {
|
|
66
|
+
failed = true;
|
|
67
|
+
throw error;
|
|
68
|
+
}
|
|
62
69
|
}
|
|
63
70
|
}
|
|
64
71
|
await Promise.all(Array.from({ length: limit }, () => worker()));
|
|
@@ -122,6 +129,8 @@ export function resolvePageConcurrency(raw, fallback = 5, max = 32) {
|
|
|
122
129
|
return fallback;
|
|
123
130
|
return Math.min(parsed, max);
|
|
124
131
|
}
|
|
132
|
+
export const JSON_TOTAL_FACTOR = 2;
|
|
133
|
+
export const TRANSFER_TOTAL_FACTOR = 60;
|
|
125
134
|
export const PAGE_CONCURRENCY = resolvePageConcurrency(process.env.GANGTISE_PAGE_CONCURRENCY);
|
|
126
135
|
const RETRYABLE_HTTP_STATUS = new Set([429, 500, 502, 503, 504]);
|
|
127
136
|
const RETRYABLE_NETWORK_CODES = new Set(["ECONNREFUSED", "ECONNRESET", "ETIMEDOUT", "ENOTFOUND", "EAI_AGAIN", "UND_ERR_CONNECT_TIMEOUT", "UND_ERR_SOCKET", "UND_ERR_HEADERS_TIMEOUT", "UND_ERR_BODY_TIMEOUT"]);
|
|
@@ -129,9 +138,6 @@ const RETRYABLE_API_CODES = new Set(["999999"]);
|
|
|
129
138
|
const TERMINAL_API_CODES = new Set(["999011", "140002"]);
|
|
130
139
|
const NO_REPLAY_NETWORK_CODES = new Set(["ECONNREFUSED", "ENOTFOUND", "EAI_AGAIN", "UND_ERR_CONNECT_TIMEOUT"]);
|
|
131
140
|
function isRetryableError(error, policy) {
|
|
132
|
-
if (error && typeof error === "object" && error.__retryable === true) {
|
|
133
|
-
return true;
|
|
134
|
-
}
|
|
135
141
|
if (error instanceof ApiError) {
|
|
136
142
|
if (error.code && TERMINAL_API_CODES.has(error.code))
|
|
137
143
|
return false;
|
|
@@ -157,8 +163,12 @@ function isRetryableError(error, policy) {
|
|
|
157
163
|
}
|
|
158
164
|
return false;
|
|
159
165
|
}
|
|
160
|
-
|
|
161
|
-
|
|
166
|
+
const MAX_AUTH_REPLAYS = 2;
|
|
167
|
+
export function markAuthReplay(error) {
|
|
168
|
+
return Object.assign(error, { __authReplay: true });
|
|
169
|
+
}
|
|
170
|
+
function isAuthReplay(error) {
|
|
171
|
+
return Boolean(error && typeof error === "object" && error.__authReplay === true);
|
|
162
172
|
}
|
|
163
173
|
export function isTransientError(error) {
|
|
164
174
|
return isRetryableError(error, "default");
|
|
@@ -168,11 +178,19 @@ export async function withRetry(fn, options = {}) {
|
|
|
168
178
|
const baseDelay = options.baseDelayMs ?? 400;
|
|
169
179
|
const maxDelay = options.maxDelayMs ?? 4_000;
|
|
170
180
|
let attempt = 0;
|
|
181
|
+
let authReplays = 0;
|
|
171
182
|
while (true) {
|
|
172
183
|
try {
|
|
173
184
|
return await fn();
|
|
174
185
|
}
|
|
175
186
|
catch (error) {
|
|
187
|
+
if (isAuthReplay(error)) {
|
|
188
|
+
if (authReplays++ >= MAX_AUTH_REPLAYS)
|
|
189
|
+
throw error;
|
|
190
|
+
if (verboseEnabled)
|
|
191
|
+
process.stderr.write(`[gangtise] replaying with a recovered token (${authReplays}/${MAX_AUTH_REPLAYS}) after: ${error instanceof Error ? error.message.slice(0, 120) : String(error)}\n`);
|
|
192
|
+
continue;
|
|
193
|
+
}
|
|
176
194
|
if (attempt >= retries || !isRetryableError(error, options.policy ?? "default"))
|
|
177
195
|
throw error;
|
|
178
196
|
const retryAfter = retryAfterFromError(error);
|
|
@@ -1,22 +1,16 @@
|
|
|
1
|
-
import { mkdirSync, readFileSync
|
|
1
|
+
import { mkdirSync, readFileSync } from "node:fs";
|
|
2
2
|
import os from "node:os";
|
|
3
3
|
import path from "node:path";
|
|
4
4
|
import { isVersionNewer } from "./args.js";
|
|
5
|
-
import {
|
|
5
|
+
import { writeFileAtomicSync } from "./output.js";
|
|
6
6
|
export const UPDATE_CHECK_TTL_MS = 24 * 60 * 60 * 1000;
|
|
7
7
|
export const DEFAULT_UPDATE_CHECK_PATH = path.join(os.homedir(), ".config", "gangtise", "update-check.json");
|
|
8
8
|
function saveLatest(cachePath, latest, now) {
|
|
9
|
-
const staging = stagingPath(cachePath, "tmp");
|
|
10
9
|
try {
|
|
11
10
|
mkdirSync(path.dirname(cachePath), { recursive: true });
|
|
12
|
-
|
|
13
|
-
renameSync(staging, cachePath);
|
|
11
|
+
writeFileAtomicSync(cachePath, JSON.stringify({ checkedAt: now, latest }), { suffix: "tmp" });
|
|
14
12
|
}
|
|
15
13
|
catch {
|
|
16
|
-
try {
|
|
17
|
-
rmSync(staging, { force: true });
|
|
18
|
-
}
|
|
19
|
-
catch { }
|
|
20
14
|
}
|
|
21
15
|
}
|
|
22
16
|
export async function latestVersion(cachePath = DEFAULT_UPDATE_CHECK_PATH, timeoutMs = 2000, now = Date.now()) {
|