gangtise-openapi-cli 0.43.1 → 0.44.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/README.md +86 -15
  2. package/dist/src/cli.js +29 -18
  3. package/dist/src/commands/ai.js +20 -29
  4. package/dist/src/commands/auth.js +5 -4
  5. package/dist/src/commands/bond.js +5 -4
  6. package/dist/src/commands/fund.js +100 -0
  7. package/dist/src/commands/fundamental.js +2 -0
  8. package/dist/src/commands/groups.js +14 -0
  9. package/dist/src/commands/indicator.js +5 -5
  10. package/dist/src/commands/insight.js +3 -4
  11. package/dist/src/commands/raw.js +4 -3
  12. package/dist/src/commands/shared.js +35 -6
  13. package/dist/src/commands/tool.js +2 -2
  14. package/dist/src/commands/vault.js +30 -30
  15. package/dist/src/core/args.js +23 -7
  16. package/dist/src/core/asyncContent.js +1 -1
  17. package/dist/src/core/auth.js +2 -10
  18. package/dist/src/core/client.js +67 -19
  19. package/dist/src/core/commandBodies.js +1 -27
  20. package/dist/src/core/config.js +9 -5
  21. package/dist/src/core/download.js +47 -10
  22. package/dist/src/core/endpoints.js +195 -0
  23. package/dist/src/core/errors.js +1 -1
  24. package/dist/src/core/output.js +103 -72
  25. package/dist/src/core/printer.js +10 -5
  26. package/dist/src/core/quoteSharding.js +28 -4
  27. package/dist/src/core/rowSink.js +24 -10
  28. package/dist/src/core/titleCache.js +2 -10
  29. package/dist/src/core/transport.js +25 -7
  30. package/dist/src/core/updateCheck.js +3 -9
  31. package/gangtise-openapi/SKILL.md +52 -16
  32. package/gangtise-openapi/references/commands/fund.md +192 -0
  33. package/gangtise-openapi/references/commands/fundamental.md +2 -2
  34. package/gangtise-openapi/references/commands/insight.md +2 -2
  35. package/gangtise-openapi/references/commands/quote.md +1 -1
  36. package/gangtise-openapi/references/commands/tool.md +1 -1
  37. package/gangtise-openapi/references/commands/vault.md +1 -1
  38. package/gangtise-openapi/references/errors.md +4 -4
  39. package/gangtise-openapi/references/response-schema.md +10 -0
  40. package/package.json +1 -1
@@ -1,6 +1,7 @@
1
1
  import { createHash, randomBytes } from "node:crypto";
2
- import { rmSync, statSync } from "node:fs";
2
+ import { renameSync, rmSync, statSync, writeFileSync } from "node:fs";
3
3
  import fs from "node:fs/promises";
4
+ import path from "node:path";
4
5
  import { ConfigError } from "./errors.js";
5
6
  const OUTPUT_FORMATS = ["table", "json", "jsonl", "csv", "markdown"];
6
7
  export function stagingPath(target, suffix = "part") {
@@ -8,6 +9,46 @@ export function stagingPath(target, suffix = "part") {
8
9
  handedOut.add(staging);
9
10
  return staging;
10
11
  }
12
+ export async function writeFileAtomic(target, content, options = {}) {
13
+ const staging = stagingPath(target, options.suffix);
14
+ try {
15
+ await fs.writeFile(staging, content, options.mode === undefined ? undefined : { mode: options.mode });
16
+ await publishFile(staging, target);
17
+ }
18
+ catch (error) {
19
+ await fs.unlink(staging).catch(() => { });
20
+ throw error;
21
+ }
22
+ }
23
+ const publishedIdentity = new Map();
24
+ export async function publishFile(staging, target) {
25
+ const { dev, ino } = await fs.stat(staging);
26
+ await fs.rename(staging, target);
27
+ publishedIdentity.set(path.resolve(target), { dev, ino });
28
+ }
29
+ export async function stillPublished(target) {
30
+ const mine = publishedIdentity.get(path.resolve(target));
31
+ if (!mine || !mine.ino)
32
+ return undefined;
33
+ const now = await fs.stat(target).catch(() => undefined);
34
+ if (!now)
35
+ return undefined;
36
+ return now.dev === mine.dev && now.ino === mine.ino;
37
+ }
38
+ export function writeFileAtomicSync(target, content, options = {}) {
39
+ const staging = stagingPath(target, options.suffix);
40
+ try {
41
+ writeFileSync(staging, content);
42
+ renameSync(staging, target);
43
+ }
44
+ catch (error) {
45
+ try {
46
+ rmSync(staging, { force: true });
47
+ }
48
+ catch { }
49
+ throw error;
50
+ }
51
+ }
11
52
  const handedOut = new Set();
12
53
  const claimed = new Set();
13
54
  export function trackClaim(name) {
@@ -117,13 +158,14 @@ function clampCell(value) {
117
158
  }
118
159
  return out + "…";
119
160
  }
120
- function renderTable(rows) {
161
+ function renderTable(rows, clamp) {
121
162
  if (rows.length === 0) {
122
163
  return "(empty)";
123
164
  }
165
+ const cell = clamp ? clampCell : (value) => value;
124
166
  const columns = Array.from(new Set(rows.flatMap((row) => Object.keys(row))));
125
- const headerCells = columns.map((column) => clampCell(sanitizeCell(column)));
126
- const matrix = rows.map((row) => columns.map((column) => clampCell(sanitizeCell(formatScalar(row[column])))));
167
+ const headerCells = columns.map((column) => cell(sanitizeCell(column)));
168
+ const matrix = rows.map((row) => columns.map((column) => cell(sanitizeCell(formatScalar(row[column])))));
127
169
  const cellWidths = matrix.map((cells) => cells.map(displayWidth));
128
170
  const headerWidths = headerCells.map(displayWidth);
129
171
  const widths = columns.map((_, c) => cellWidths.reduce((max, cells) => Math.max(max, cells[c]), headerWidths[c]));
@@ -144,14 +186,25 @@ function renderMarkdown(rows) {
144
186
  const body = rows.map((row) => `| ${columns.map((column) => mdEscape(sanitizeCell(formatScalar(row[column])))).join(" | ")} |`);
145
187
  return [header, divider, ...body].join("\n");
146
188
  }
147
- function renderCsv(rows) {
148
- if (rows.length === 0) {
149
- return "";
189
+ export function csvHeader(columns) {
190
+ return columns.map(csvEscape).join(",");
191
+ }
192
+ export function csvRow(columns, row) {
193
+ return columns.map((column) => csvEscape(formatScalar(row[column]))).join(",");
194
+ }
195
+ export function* outputLines(value, format, columns) {
196
+ if (format === "jsonl") {
197
+ for (const item of jsonlItems(value))
198
+ yield JSON.stringify(item);
199
+ return;
150
200
  }
151
- const columns = Array.from(new Set(rows.flatMap((row) => Object.keys(row))));
152
- const header = columns.map(csvEscape).join(",");
153
- const body = rows.map((row) => columns.map((column) => csvEscape(formatScalar(row[column]))).join(","));
154
- return [header, ...body].join("\n");
201
+ const rows = toRows(value);
202
+ const header = rows.length > 0 ? Array.from(new Set(rows.flatMap((row) => Object.keys(row)))) : (columns ?? []).map(String);
203
+ if (header.length === 0)
204
+ return;
205
+ yield csvHeader(header);
206
+ for (const row of rows)
207
+ yield csvRow(header, row);
155
208
  }
156
209
  export function jsonlItems(value) {
157
210
  const list = value && typeof value === "object" && !Array.isArray(value) ? value.list : undefined;
@@ -160,71 +213,61 @@ export function jsonlItems(value) {
160
213
  export function countOutputRows(value, format) {
161
214
  return format === "jsonl" ? jsonlItems(value).length : toRows(value).length;
162
215
  }
163
- export function renderOutput(value, format) {
216
+ export function renderOutput(value, format, options = {}) {
164
217
  switch (format) {
165
218
  case "json":
166
219
  return JSON.stringify(value, null, 2);
167
220
  case "jsonl":
168
- return jsonlItems(value).map((item) => JSON.stringify(item)).join("\n");
169
221
  case "csv":
170
- return renderCsv(toRows(value));
222
+ return [...outputLines(value, format, options.columns)].join("\n");
171
223
  case "markdown":
172
224
  return renderMarkdown(toRows(value));
173
225
  case "table":
174
226
  default:
175
- return renderTable(toRows(value));
227
+ return renderTable(toRows(value), options.clampCells ?? true);
228
+ }
229
+ }
230
+ export async function writeOutputLines(stream, value, format, options = {}) {
231
+ const hash = options.digest ? createHash("sha256") : undefined;
232
+ let bytes = 0;
233
+ const flush = async (lines) => {
234
+ if (lines.length === 0)
235
+ return;
236
+ const text = lines.join("\n");
237
+ if (hash) {
238
+ const buf = Buffer.from(`${text}\n`, "utf8");
239
+ hash.update(buf);
240
+ bytes += buf.byteLength;
241
+ }
242
+ await writeLine(stream, text);
243
+ };
244
+ let chunk = [];
245
+ let first = true;
246
+ for (const line of outputLines(value, format, options.columns)) {
247
+ chunk.push(first && options.bom && format === "csv" ? `\ufeff${line}` : line);
248
+ first = false;
249
+ if (chunk.length >= LINES_PER_WRITE) {
250
+ await flush(chunk);
251
+ chunk = [];
252
+ }
176
253
  }
254
+ await flush(chunk);
255
+ return hash ? { bytes, sha256: hash.digest("hex") } : undefined;
177
256
  }
178
- export async function streamOutputToFile(value, format, outputPath) {
257
+ export async function streamOutputToFile(value, format, outputPath, columns) {
179
258
  if (format !== "jsonl" && format !== "csv")
180
259
  return null;
181
- const list = pickList(value);
182
- if (!list)
183
- return null;
184
- if (list.length < 1000)
185
- return null;
186
- let csvRows = [];
187
- let csvColumns = [];
188
- if (format === "csv") {
189
- csvRows = list.filter((row) => Boolean(row && typeof row === "object" && !Array.isArray(row)));
190
- if (csvRows.length === 0)
191
- return null;
192
- csvColumns = Array.from(new Set(csvRows.flatMap((row) => Object.keys(row))));
193
- }
194
- const { dirname } = await import("node:path");
195
260
  const { createWriteStream } = await import("node:fs");
196
- await fs.mkdir(dirname(outputPath), { recursive: true });
261
+ await fs.mkdir(path.dirname(outputPath), { recursive: true });
197
262
  const partPath = stagingPath(outputPath);
198
263
  const stream = createWriteStream(partPath, { encoding: "utf8" });
199
264
  stream.on("error", () => { });
200
265
  try {
201
- if (format === "jsonl") {
202
- let chunk = [];
203
- for (const item of jsonlItems(value)) {
204
- chunk.push(JSON.stringify(item));
205
- if (chunk.length >= LINES_PER_WRITE) {
206
- await writeLines(stream, chunk);
207
- chunk = [];
208
- }
209
- }
210
- await writeLines(stream, chunk);
211
- }
212
- else {
213
- let chunk = ["\ufeff" + csvColumns.map(csvEscape).join(",")];
214
- for (const row of csvRows) {
215
- chunk.push(csvColumns.map((column) => csvEscape(formatScalar(row[column]))).join(","));
216
- if (chunk.length >= LINES_PER_WRITE) {
217
- await writeLines(stream, chunk);
218
- chunk = [];
219
- }
220
- }
221
- await writeLines(stream, chunk);
222
- }
266
+ const digest = await writeOutputLines(stream, value, format, { columns, bom: true, digest: true });
223
267
  await new Promise((resolve, reject) => {
224
268
  stream.end((err) => err ? reject(err) : resolve());
225
269
  });
226
- const digest = await digestFile(partPath);
227
- await fs.rename(partPath, outputPath);
270
+ await publishFile(partPath, outputPath);
228
271
  return digest;
229
272
  }
230
273
  catch (error) {
@@ -246,9 +289,10 @@ export function pickList(value) {
246
289
  }
247
290
  return null;
248
291
  }
292
+ const NUMERIC_LOOKING = /^[+\-](?:(?=[^\d]*\d)[\d.,%eE]+)?$/;
249
293
  export function csvEscape(value) {
250
294
  let out = value;
251
- if (/^[=@\t\r]/.test(out) || (/^[+\-]/.test(out) && !Number.isFinite(Number(out))))
295
+ if (/^[=@\t\r]/.test(out) || (/^[+\-]/.test(out) && !NUMERIC_LOOKING.test(out)))
252
296
  out = "'" + out;
253
297
  if (/[",\n\r]/.test(out))
254
298
  return `"${out.replaceAll("\"", "\"\"")}"`;
@@ -277,20 +321,7 @@ export async function saveOutputIfNeeded(content, outputPath, withDigest = true)
277
321
  }
278
322
  const { dirname } = await import("node:path");
279
323
  await fs.mkdir(dirname(outputPath), { recursive: true });
280
- const partPath = stagingPath(outputPath);
281
- try {
282
- if (typeof content === "string") {
283
- await fs.writeFile(partPath, content, "utf8");
284
- }
285
- else {
286
- await fs.writeFile(partPath, content);
287
- }
288
- const digest = withDigest ? digestBuffer(content) : undefined;
289
- await fs.rename(partPath, outputPath);
290
- return digest;
291
- }
292
- catch (error) {
293
- await fs.unlink(partPath).catch(() => { });
294
- throw error;
295
- }
324
+ const digest = withDigest ? digestBuffer(content) : undefined;
325
+ await writeFileAtomic(outputPath, content);
326
+ return digest;
296
327
  }
@@ -1,7 +1,7 @@
1
1
  import fs from "node:fs/promises";
2
2
  import path from "node:path";
3
3
  import { normalizeRows } from "./normalize.js";
4
- import { countOutputRows, digestFile, pickList, renderOutput, saveOutputIfNeeded, stagingPath, streamOutputToFile } from "./output.js";
4
+ import { countOutputRows, digestFile, pickList, renderOutput, saveOutputIfNeeded, stagingPath, stillPublished, streamOutputToFile, writeOutputLines } from "./output.js";
5
5
  import { getRowSink } from "./rowSink.js";
6
6
  import { extractTitles, writeTitleCache } from "./titleCache.js";
7
7
  import { CLI_VERSION } from "../version.js";
@@ -9,11 +9,13 @@ import { decidedExitCode, EXIT_SUPERSEDED, markIncomplete, markSuperseded } from
9
9
  const LARGE_RESULT_ROWS = 50_000;
10
10
  export const SUPERSEDED_EXIT = EXIT_SUPERSEDED;
11
11
  export async function warnIfSuperseded(output, digest) {
12
+ if (await stillPublished(output) === true)
13
+ return;
12
14
  const actual = await digestFile(output).catch(() => undefined);
13
15
  if (!actual || actual.sha256 === digest.sha256)
14
16
  return;
15
17
  process.stderr.write(`[gangtise] warning: ${output} was replaced by another export to the same path while this one was finishing — the file there is not this command's output `
16
- + `(wrote sha256 ${digest.sha256.slice(0, 12)}…, found ${actual.sha256.slice(0, 12)}…). Give concurrent exports distinct --output paths.\n`);
18
+ + `(this run wrote sha256 ${digest.sha256.slice(0, 12)}…). Give concurrent exports distinct --output paths.\n`);
17
19
  markSuperseded();
18
20
  }
19
21
  function warnIfLargeInMemory(items, format) {
@@ -135,10 +137,9 @@ export async function printData(data, format, output, cache) {
135
137
  await streamed.finish();
136
138
  digest = streamed.digest;
137
139
  }
138
- else if (!(digest = (await streamOutputToFile(normalized, format, output)) ?? undefined)) {
140
+ else if (!(digest = (await streamOutputToFile(normalized, format, output, columns)) ?? undefined)) {
139
141
  warnIfLargeInMemory(items, format);
140
- const content = renderOutput(normalized, format);
141
- digest = await saveOutputIfNeeded(format === "csv" ? `\ufeff${content}` : content, output);
142
+ digest = await saveOutputIfNeeded(renderOutput(normalized, format, { clampCells: false }), output);
142
143
  }
143
144
  }
144
145
  catch (error) {
@@ -151,6 +152,10 @@ export async function printData(data, format, output, cache) {
151
152
  process.stdout.write(`${output}\n`);
152
153
  return;
153
154
  }
155
+ if (format === "jsonl" || format === "csv") {
156
+ await writeOutputLines(process.stdout, normalized, format, { columns });
157
+ return;
158
+ }
154
159
  warnIfLargeInMemory(items, format);
155
160
  const rendered = renderOutput(normalized, format);
156
161
  if (rendered)
@@ -1,4 +1,4 @@
1
- import { isStructuralError } from "./errors.js";
1
+ import { ApiError, isStructuralError } from "./errors.js";
2
2
  import { columnarSchemaValid } from "./normalize.js";
3
3
  import { attachRowSink } from "./rowSink.js";
4
4
  import { isVerbose, PAGE_CONCURRENCY, runInOrder } from "./transport.js";
@@ -60,6 +60,12 @@ function buildShards(start, end, shardDays) {
60
60
  flush();
61
61
  return shards;
62
62
  }
63
+ const OUT_OF_WINDOW_CODE = "110003";
64
+ function describeError(error) {
65
+ if (error instanceof ApiError)
66
+ return `${error.code ? `(${error.code}) ` : ""}${error.message}`;
67
+ return error instanceof Error ? error.message : String(error);
68
+ }
63
69
  export async function callKlineWithSharding(client, endpointKey, body, config) {
64
70
  const fullMarketValue = config.fullMarketValue ?? "all";
65
71
  if (!isFullMarket(body, fullMarketValue)) {
@@ -101,9 +107,13 @@ export async function callKlineWithSharding(client, endpointKey, body, config) {
101
107
  const failedShards = [];
102
108
  let firstError = null;
103
109
  let aborted = false;
110
+ let unsent = 0;
111
+ const outOfWindow = [];
112
+ let windowError = null;
104
113
  const fetchShard = async (shard) => {
105
114
  if (aborted) {
106
115
  failedShards.push(shard);
116
+ unsent++;
107
117
  return null;
108
118
  }
109
119
  try {
@@ -115,6 +125,12 @@ export async function callKlineWithSharding(client, endpointKey, body, config) {
115
125
  return res;
116
126
  }
117
127
  catch (error) {
128
+ if (error instanceof ApiError && error.code === OUT_OF_WINDOW_CODE) {
129
+ outOfWindow.push(shard);
130
+ if (!windowError)
131
+ windowError = error;
132
+ return null;
133
+ }
118
134
  if (!firstError)
119
135
  firstError = error;
120
136
  if (!isStructuralError(error))
@@ -193,13 +209,19 @@ export async function callKlineWithSharding(client, endpointKey, body, config) {
193
209
  await keep(map ? rows.map((item) => (Array.isArray(item) ? map.map((k) => item[k]) : item)) : rows);
194
210
  };
195
211
  await runInOrder(shards, config.concurrency ?? PAGE_CONCURRENCY, fetchShard, mergeShard);
196
- if (failedShards.length === shards.length) {
197
- throw firstError ?? new Error(`All ${shards.length} kline shards failed`);
212
+ if (failedShards.length + outOfWindow.length === shards.length) {
213
+ throw firstError ?? windowError ?? new Error(`All ${shards.length} kline shards failed`);
214
+ }
215
+ const skipped = [...outOfWindow].sort((a, b) => a.startDate.localeCompare(b.startDate));
216
+ if (skipped.length > 0) {
217
+ process.stderr.write(`[gangtise] note: ${skipped.length}/${shards.length} shard(s) between ${skipped[0].startDate} and ${skipped[skipped.length - 1].endDate} lie outside your account's history window (110003) and were skipped; the result covers the rest of the range (see outOfWindowShards).\n`);
198
218
  }
199
219
  const base = header ?? fallback ?? {};
200
220
  const out = { ...base, total: count, list: merged };
201
221
  if (sink)
202
222
  attachRowSink(out, sink);
223
+ if (skipped.length > 0)
224
+ out.outOfWindowShards = skipped;
203
225
  if (fieldList) {
204
226
  out.fieldList = fieldList;
205
227
  }
@@ -215,7 +237,9 @@ export async function callKlineWithSharding(client, endpointKey, body, config) {
215
237
  if (failedShards.length > 0) {
216
238
  out.partial = true;
217
239
  out.failedShards = failedShards;
218
- process.stderr.write(`[gangtise] warning: ${failedShards.length}/${shards.length} shards failed; results are partial (see failedShards)\n`);
240
+ const notSent = unsent > 0 ? ` (${unsent} of them not sent: the first error stopped the rest)` : "";
241
+ const cause = firstError ? `; first error: ${describeError(firstError)}` : "";
242
+ process.stderr.write(`[gangtise] warning: ${failedShards.length}/${shards.length} shards failed${notSent}${cause}; results are partial (see failedShards)\n`);
219
243
  }
220
244
  if (partialShards.length > 0) {
221
245
  out.partial = true;
@@ -1,8 +1,9 @@
1
+ import { createHash } from "node:crypto";
1
2
  import { createReadStream, createWriteStream } from "node:fs";
2
3
  import fs from "node:fs/promises";
3
4
  import path from "node:path";
4
5
  import { assertColumnarHeader, zipFieldRow } from "./normalize.js";
5
- import { csvEscape, digestFile, formatScalar, stagingPath, writeLine, writeLines, LINES_PER_WRITE } from "./output.js";
6
+ import { csvHeader, csvRow, publishFile, stagingPath, writeLine, LINES_PER_WRITE } from "./output.js";
6
7
  import { extractTitles, MAX_TITLES_PER_ENDPOINT } from "./titleCache.js";
7
8
  export const ROW_SINK = Symbol("gangtise.rowSink");
8
9
  const STREAM_THRESHOLD = 1000;
@@ -25,6 +26,8 @@ export class ExportSink {
25
26
  earliestDates = new Map();
26
27
  partPath;
27
28
  rowsPath;
29
+ hash = createHash("sha256");
30
+ bytes = 0;
28
31
  constructor(outputPath, format = "jsonl", cache) {
29
32
  this.outputPath = outputPath;
30
33
  this.format = format;
@@ -93,8 +96,8 @@ export class ExportSink {
93
96
  await endStream(stream);
94
97
  if (this.format === "csv")
95
98
  await this.assembleCsv();
96
- this.digest = await digestFile(this.partPath);
97
- await fs.rename(this.partPath, this.outputPath);
99
+ this.digest = { bytes: this.bytes, sha256: this.hash.digest("hex") };
100
+ await publishFile(this.partPath, this.outputPath);
98
101
  this.finished = true;
99
102
  }
100
103
  catch (error) {
@@ -142,7 +145,7 @@ export class ExportSink {
142
145
  }
143
146
  chunk.push(JSON.stringify(item));
144
147
  if (chunk.length >= LINES_PER_WRITE) {
145
- await writeLines(stream, chunk);
148
+ await this.writeChunk(stream, chunk, this.format === "jsonl");
146
149
  chunk = [];
147
150
  }
148
151
  this.rows++;
@@ -154,7 +157,18 @@ export class ExportSink {
154
157
  }
155
158
  }
156
159
  }
157
- await writeLines(stream, chunk);
160
+ await this.writeChunk(stream, chunk, this.format === "jsonl");
161
+ }
162
+ async writeChunk(stream, lines, published) {
163
+ if (lines.length === 0)
164
+ return;
165
+ const text = lines.join("\n");
166
+ if (published) {
167
+ const buf = Buffer.from(`${text}\n`, "utf8");
168
+ this.hash.update(buf);
169
+ this.bytes += buf.byteLength;
170
+ }
171
+ await writeLine(stream, text);
158
172
  }
159
173
  async assembleCsv() {
160
174
  const out = createWriteStream(this.partPath, { encoding: "utf8" });
@@ -162,7 +176,7 @@ export class ExportSink {
162
176
  this.stream = out;
163
177
  const objectMode = this.objectRows > 0;
164
178
  const columns = objectMode ? this.columns : ["index", "value"];
165
- await writeLine(out, "\ufeff" + columns.map(csvEscape).join(","));
179
+ await this.writeChunk(out, [`\ufeff${csvHeader(columns)}`], true);
166
180
  const input = createReadStream(this.rowsPath, { encoding: "utf8" });
167
181
  try {
168
182
  let index = 0;
@@ -174,10 +188,10 @@ export class ExportSink {
174
188
  if (objectMode) {
175
189
  if (!isObjectRow(item))
176
190
  return;
177
- csvLines.push(columns.map((column) => csvEscape(formatScalar(item[column]))).join(","));
191
+ csvLines.push(csvRow(columns, item));
178
192
  }
179
193
  else {
180
- csvLines.push(`${csvEscape(String(index++))},${csvEscape(formatScalar(item))}`);
194
+ csvLines.push(csvRow(columns, { index: index++, value: item }));
181
195
  }
182
196
  };
183
197
  let rest = "";
@@ -187,12 +201,12 @@ export class ExportSink {
187
201
  for (const line of lines)
188
202
  emit(line);
189
203
  if (csvLines.length >= LINES_PER_WRITE) {
190
- await writeLines(out, csvLines);
204
+ await this.writeChunk(out, csvLines, true);
191
205
  csvLines = [];
192
206
  }
193
207
  }
194
208
  emit(rest);
195
- await writeLines(out, csvLines);
209
+ await this.writeChunk(out, csvLines, true);
196
210
  await endStream(out);
197
211
  }
198
212
  catch (error) {
@@ -1,7 +1,7 @@
1
1
  import fs from "node:fs/promises";
2
2
  import os from "node:os";
3
3
  import path from "node:path";
4
- import { stagingPath } from "./output.js";
4
+ import { writeFileAtomic } from "./output.js";
5
5
  export const DEFAULT_TITLE_CACHE_PATH = path.join(os.homedir(), ".config", "gangtise", "title-cache.json");
6
6
  export const TITLE_LOOKUP_SIZE = 200;
7
7
  const TITLE_CACHE_TTL_MS = 24 * 60 * 60 * 1000;
@@ -78,15 +78,7 @@ async function flush(filePath) {
78
78
  await mergeFromDisk(filePath, data);
79
79
  const snapshot = JSON.stringify(data);
80
80
  await fs.mkdir(path.dirname(filePath), { recursive: true });
81
- const tmp = stagingPath(filePath, "tmp");
82
- await fs.writeFile(tmp, snapshot, { encoding: "utf8", mode: 0o600 });
83
- try {
84
- await fs.rename(tmp, filePath);
85
- }
86
- catch (error) {
87
- await fs.unlink(tmp).catch(() => { });
88
- throw error;
89
- }
81
+ await writeFileAtomic(filePath, snapshot, { mode: 0o600, suffix: "tmp" });
90
82
  }
91
83
  }
92
84
  function capTitles(merged, freshKeys, cap) {
@@ -53,12 +53,19 @@ export async function runWithConcurrency(items, concurrency, fn) {
53
53
  const limit = Math.max(1, Math.min(concurrency, items.length));
54
54
  const results = new Array(items.length);
55
55
  let next = 0;
56
+ let failed = false;
56
57
  async function worker() {
57
- while (true) {
58
+ while (!failed) {
58
59
  const index = next++;
59
60
  if (index >= items.length)
60
61
  return;
61
- results[index] = await fn(items[index], index);
62
+ try {
63
+ results[index] = await fn(items[index], index);
64
+ }
65
+ catch (error) {
66
+ failed = true;
67
+ throw error;
68
+ }
62
69
  }
63
70
  }
64
71
  await Promise.all(Array.from({ length: limit }, () => worker()));
@@ -122,6 +129,8 @@ export function resolvePageConcurrency(raw, fallback = 5, max = 32) {
122
129
  return fallback;
123
130
  return Math.min(parsed, max);
124
131
  }
132
+ export const JSON_TOTAL_FACTOR = 2;
133
+ export const TRANSFER_TOTAL_FACTOR = 60;
125
134
  export const PAGE_CONCURRENCY = resolvePageConcurrency(process.env.GANGTISE_PAGE_CONCURRENCY);
126
135
  const RETRYABLE_HTTP_STATUS = new Set([429, 500, 502, 503, 504]);
127
136
  const RETRYABLE_NETWORK_CODES = new Set(["ECONNREFUSED", "ECONNRESET", "ETIMEDOUT", "ENOTFOUND", "EAI_AGAIN", "UND_ERR_CONNECT_TIMEOUT", "UND_ERR_SOCKET", "UND_ERR_HEADERS_TIMEOUT", "UND_ERR_BODY_TIMEOUT"]);
@@ -129,9 +138,6 @@ const RETRYABLE_API_CODES = new Set(["999999"]);
129
138
  const TERMINAL_API_CODES = new Set(["999011", "140002"]);
130
139
  const NO_REPLAY_NETWORK_CODES = new Set(["ECONNREFUSED", "ENOTFOUND", "EAI_AGAIN", "UND_ERR_CONNECT_TIMEOUT"]);
131
140
  function isRetryableError(error, policy) {
132
- if (error && typeof error === "object" && error.__retryable === true) {
133
- return true;
134
- }
135
141
  if (error instanceof ApiError) {
136
142
  if (error.code && TERMINAL_API_CODES.has(error.code))
137
143
  return false;
@@ -157,8 +163,12 @@ function isRetryableError(error, policy) {
157
163
  }
158
164
  return false;
159
165
  }
160
- export function markRetryable(error) {
161
- return Object.assign(error, { __retryable: true });
166
+ const MAX_AUTH_REPLAYS = 2;
167
+ export function markAuthReplay(error) {
168
+ return Object.assign(error, { __authReplay: true });
169
+ }
170
+ function isAuthReplay(error) {
171
+ return Boolean(error && typeof error === "object" && error.__authReplay === true);
162
172
  }
163
173
  export function isTransientError(error) {
164
174
  return isRetryableError(error, "default");
@@ -168,11 +178,19 @@ export async function withRetry(fn, options = {}) {
168
178
  const baseDelay = options.baseDelayMs ?? 400;
169
179
  const maxDelay = options.maxDelayMs ?? 4_000;
170
180
  let attempt = 0;
181
+ let authReplays = 0;
171
182
  while (true) {
172
183
  try {
173
184
  return await fn();
174
185
  }
175
186
  catch (error) {
187
+ if (isAuthReplay(error)) {
188
+ if (authReplays++ >= MAX_AUTH_REPLAYS)
189
+ throw error;
190
+ if (verboseEnabled)
191
+ process.stderr.write(`[gangtise] replaying with a recovered token (${authReplays}/${MAX_AUTH_REPLAYS}) after: ${error instanceof Error ? error.message.slice(0, 120) : String(error)}\n`);
192
+ continue;
193
+ }
176
194
  if (attempt >= retries || !isRetryableError(error, options.policy ?? "default"))
177
195
  throw error;
178
196
  const retryAfter = retryAfterFromError(error);
@@ -1,22 +1,16 @@
1
- import { mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
1
+ import { mkdirSync, readFileSync } from "node:fs";
2
2
  import os from "node:os";
3
3
  import path from "node:path";
4
4
  import { isVersionNewer } from "./args.js";
5
- import { stagingPath } from "./output.js";
5
+ import { writeFileAtomicSync } from "./output.js";
6
6
  export const UPDATE_CHECK_TTL_MS = 24 * 60 * 60 * 1000;
7
7
  export const DEFAULT_UPDATE_CHECK_PATH = path.join(os.homedir(), ".config", "gangtise", "update-check.json");
8
8
  function saveLatest(cachePath, latest, now) {
9
- const staging = stagingPath(cachePath, "tmp");
10
9
  try {
11
10
  mkdirSync(path.dirname(cachePath), { recursive: true });
12
- writeFileSync(staging, JSON.stringify({ checkedAt: now, latest }));
13
- renameSync(staging, cachePath);
11
+ writeFileAtomicSync(cachePath, JSON.stringify({ checkedAt: now, latest }), { suffix: "tmp" });
14
12
  }
15
13
  catch {
16
- try {
17
- rmSync(staging, { force: true });
18
- }
19
- catch { }
20
14
  }
21
15
  }
22
16
  export async function latestVersion(cachePath = DEFAULT_UPDATE_CHECK_PATH, timeoutMs = 2000, now = Date.now()) {