gangtise-openapi-cli 0.41.1 → 0.43.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/README.md +45 -39
  2. package/dist/src/cli.js +50 -1161
  3. package/dist/src/commands/ai.js +156 -0
  4. package/dist/src/commands/alternative.js +56 -0
  5. package/dist/src/commands/auth.js +37 -0
  6. package/dist/src/commands/bond.js +76 -0
  7. package/dist/src/commands/fundamental.js +137 -0
  8. package/dist/src/commands/indicator.js +134 -0
  9. package/dist/src/commands/insight.js +338 -0
  10. package/dist/src/commands/quote.js +181 -0
  11. package/dist/src/commands/raw.js +49 -0
  12. package/dist/src/commands/reference.js +58 -0
  13. package/dist/src/commands/shared.js +197 -0
  14. package/dist/src/commands/tool.js +55 -0
  15. package/dist/src/commands/vault.js +203 -0
  16. package/dist/src/core/args.js +13 -9
  17. package/dist/src/core/asyncContent.js +32 -19
  18. package/dist/src/core/auth.js +3 -2
  19. package/dist/src/core/client.js +112 -15
  20. package/dist/src/core/commandBodies.js +2 -1
  21. package/dist/src/core/config.js +16 -3
  22. package/dist/src/core/download.js +9 -5
  23. package/dist/src/core/endpoints.js +106 -1
  24. package/dist/src/core/exitStatus.js +17 -0
  25. package/dist/src/core/fileParse.js +3 -20
  26. package/dist/src/core/opinionDetail.js +39 -0
  27. package/dist/src/core/output.js +55 -12
  28. package/dist/src/core/perSecurity.js +29 -11
  29. package/dist/src/core/printer.js +7 -8
  30. package/dist/src/core/quoteSharding.js +19 -15
  31. package/dist/src/core/rowSink.js +40 -7
  32. package/dist/src/core/titleCache.js +28 -2
  33. package/dist/src/core/transport.js +7 -4
  34. package/dist/src/core/updateCheck.js +51 -0
  35. package/dist/src/version.js +19 -1
  36. package/gangtise-openapi/SKILL.md +50 -49
  37. package/gangtise-openapi/references/commands/ai.md +35 -16
  38. package/gangtise-openapi/references/commands/alternative.md +2 -2
  39. package/gangtise-openapi/references/commands/bond.md +1 -1
  40. package/gangtise-openapi/references/commands/fundamental.md +5 -5
  41. package/gangtise-openapi/references/commands/indicator.md +29 -27
  42. package/gangtise-openapi/references/commands/insight.md +39 -31
  43. package/gangtise-openapi/references/commands/quote.md +19 -19
  44. package/gangtise-openapi/references/commands/reference-and-lookup.md +18 -15
  45. package/gangtise-openapi/references/commands/vault.md +10 -7
  46. package/gangtise-openapi/references/errors.md +27 -25
  47. package/gangtise-openapi/references/examples.md +38 -36
  48. package/gangtise-openapi/references/fields.md +8 -6
  49. package/gangtise-openapi/references/lookup-ids.md +10 -8
  50. package/gangtise-openapi/references/response-schema.md +7 -7
  51. package/package.json +1 -1
@@ -1,9 +1,32 @@
1
1
  import { createHash, randomBytes } from "node:crypto";
2
+ import { rmSync, statSync } from "node:fs";
2
3
  import fs from "node:fs/promises";
3
4
  import { ConfigError } from "./errors.js";
4
5
  const OUTPUT_FORMATS = ["table", "json", "jsonl", "csv", "markdown"];
5
6
  export function stagingPath(target, suffix = "part") {
6
- return `${target}.${randomBytes(4).toString("hex")}.${suffix}`;
7
+ const staging = `${target}.${randomBytes(4).toString("hex")}.${suffix}`;
8
+ handedOut.add(staging);
9
+ return staging;
10
+ }
11
+ const handedOut = new Set();
12
+ const claimed = new Set();
13
+ export function trackClaim(name) {
14
+ claimed.add(name);
15
+ }
16
+ export function removeStagingFiles() {
17
+ for (const staging of handedOut) {
18
+ try {
19
+ rmSync(staging, { force: true });
20
+ }
21
+ catch { }
22
+ }
23
+ for (const name of claimed) {
24
+ try {
25
+ if (statSync(name).size === 0)
26
+ rmSync(name, { force: true });
27
+ }
28
+ catch { }
29
+ }
7
30
  }
8
31
  export function digestBuffer(content) {
9
32
  const buf = typeof content === "string" ? Buffer.from(content, "utf8") : content;
@@ -29,9 +52,13 @@ export function parseOutputFormat(value) {
29
52
  throw new ConfigError(`Unsupported format: ${format}`);
30
53
  }
31
54
  function sanitizeCell(value) {
55
+ if (!/[\u0000-\u001f\u007f-\u009f]/.test(value))
56
+ return value;
32
57
  return value.replace(/[\r\n]+/g, " ").replace(/[\u0000-\u001f\u007f\u0080-\u009f]/g, "");
33
58
  }
34
59
  function displayWidth(value) {
60
+ if (/^[\x20-\x7e]*$/.test(value))
61
+ return value.length;
35
62
  let width = 0;
36
63
  for (const ch of value) {
37
64
  const cp = ch.codePointAt(0);
@@ -97,11 +124,13 @@ function renderTable(rows) {
97
124
  const columns = Array.from(new Set(rows.flatMap((row) => Object.keys(row))));
98
125
  const headerCells = columns.map((column) => clampCell(sanitizeCell(column)));
99
126
  const matrix = rows.map((row) => columns.map((column) => clampCell(sanitizeCell(formatScalar(row[column])))));
100
- const widths = columns.map((_, c) => matrix.reduce((max, cells) => Math.max(max, displayWidth(cells[c])), displayWidth(headerCells[c])));
101
- const renderLine = (values) => values.map((value, index) => value + " ".repeat(Math.max(0, widths[index] - displayWidth(value)))).join(" ");
102
- const header = renderLine(headerCells);
103
- const divider = renderLine(widths.map((width) => "-".repeat(width)));
104
- const body = matrix.map((cells) => renderLine(cells));
127
+ const cellWidths = matrix.map((cells) => cells.map(displayWidth));
128
+ const headerWidths = headerCells.map(displayWidth);
129
+ const widths = columns.map((_, c) => cellWidths.reduce((max, cells) => Math.max(max, cells[c]), headerWidths[c]));
130
+ const renderLine = (values, valueWidths) => values.map((value, index) => value + " ".repeat(Math.max(0, widths[index] - valueWidths[index]))).join(" ");
131
+ const header = renderLine(headerCells, headerWidths);
132
+ const divider = renderLine(widths.map((width) => "-".repeat(width)), widths);
133
+ const body = matrix.map((cells, r) => renderLine(cells, cellWidths[r]));
105
134
  return [header, divider, ...body].join("\n");
106
135
  }
107
136
  function renderMarkdown(rows) {
@@ -170,16 +199,26 @@ export async function streamOutputToFile(value, format, outputPath) {
170
199
  stream.on("error", () => { });
171
200
  try {
172
201
  if (format === "jsonl") {
202
+ let chunk = [];
173
203
  for (const item of jsonlItems(value)) {
174
- await writeLine(stream, JSON.stringify(item));
204
+ chunk.push(JSON.stringify(item));
205
+ if (chunk.length >= LINES_PER_WRITE) {
206
+ await writeLines(stream, chunk);
207
+ chunk = [];
208
+ }
175
209
  }
210
+ await writeLines(stream, chunk);
176
211
  }
177
212
  else {
178
- await writeLine(stream, "\ufeff" + csvColumns.map(csvEscape).join(","));
213
+ let chunk = ["\ufeff" + csvColumns.map(csvEscape).join(",")];
179
214
  for (const row of csvRows) {
180
- const cells = csvColumns.map((column) => csvEscape(formatScalar(row[column])));
181
- await writeLine(stream, cells.join(","));
215
+ chunk.push(csvColumns.map((column) => csvEscape(formatScalar(row[column]))).join(","));
216
+ if (chunk.length >= LINES_PER_WRITE) {
217
+ await writeLines(stream, chunk);
218
+ chunk = [];
219
+ }
182
220
  }
221
+ await writeLines(stream, chunk);
183
222
  }
184
223
  await new Promise((resolve, reject) => {
185
224
  stream.end((err) => err ? reject(err) : resolve());
@@ -215,6 +254,10 @@ export function csvEscape(value) {
215
254
  return `"${out.replaceAll("\"", "\"\"")}"`;
216
255
  return out;
217
256
  }
257
+ export const LINES_PER_WRITE = 1000;
258
+ export function writeLines(stream, lines) {
259
+ return lines.length === 0 ? Promise.resolve() : writeLine(stream, lines.join("\n"));
260
+ }
218
261
  export function writeLine(stream, line) {
219
262
  return new Promise((resolve, reject) => {
220
263
  const ok = stream.write(line + "\n", (err) => err ? reject(err) : undefined);
@@ -228,7 +271,7 @@ export function writeLine(stream, line) {
228
271
  stream.once("error", onError);
229
272
  });
230
273
  }
231
- export async function saveOutputIfNeeded(content, outputPath) {
274
+ export async function saveOutputIfNeeded(content, outputPath, withDigest = true) {
232
275
  if (!outputPath) {
233
276
  return;
234
277
  }
@@ -242,7 +285,7 @@ export async function saveOutputIfNeeded(content, outputPath) {
242
285
  else {
243
286
  await fs.writeFile(partPath, content);
244
287
  }
245
- const digest = digestBuffer(content);
288
+ const digest = withDigest ? digestBuffer(content) : undefined;
246
289
  await fs.rename(partPath, outputPath);
247
290
  return digest;
248
291
  }
@@ -17,7 +17,10 @@ export function estimateTradingDays(startDate, endDate) {
17
17
  }
18
18
  return days;
19
19
  }
20
- export async function callPerSecurity(client, endpointKey, securities, makeBody, cap, label) {
20
+ export async function callPerSecurity(client, endpointKey, securities, makeBody, cap, label, groupSize = 1) {
21
+ const groups = [];
22
+ for (let i = 0; i < securities.length; i += groupSize)
23
+ groups.push(securities.slice(i, i + groupSize));
21
24
  const sink = client.claimRowSink?.();
22
25
  let fieldList;
23
26
  let headerSecurity;
@@ -29,14 +32,14 @@ export async function callPerSecurity(client, endpointKey, securities, makeBody,
29
32
  const mergePart = async (part, i) => {
30
33
  const rec = part;
31
34
  if (!(rec && typeof rec === "object" && Array.isArray(rec.list))) {
32
- throw markStructural(new ApiError(`${label}: ${securities[i]} returned no list payload — the response layout may have changed`, undefined, undefined, rec));
35
+ throw markStructural(new ApiError(`${label}: ${groups[i].join(", ")} returned no list payload — the response layout may have changed`, undefined, undefined, rec));
33
36
  }
34
37
  if (rec.partial === true)
35
38
  partial = true;
36
39
  if (rec.list.length === 0) {
37
40
  if ((typeof rec.total === "number" && rec.total > 0) || rec.partial === true) {
38
41
  const claim = typeof rec.total === "number" && rec.total > 0 ? `reported total=${rec.total}` : "carried a partial marker";
39
- throw markStructural(new ApiError(`${label}: ${securities[i]} ${claim} but delivered no rows — the response is inconsistent`, undefined, undefined, rec));
42
+ throw markStructural(new ApiError(`${label}: ${groups[i].join(", ")} ${claim} but delivered no rows — the response is inconsistent`, undefined, undefined, rec));
40
43
  }
41
44
  if (!emptyFields && Array.isArray(rec.fieldList))
42
45
  emptyFields = rec.fieldList;
@@ -44,26 +47,27 @@ export async function callPerSecurity(client, endpointKey, securities, makeBody,
44
47
  }
45
48
  const fields = Array.isArray(rec.fieldList) && rec.fieldList.length > 0 ? rec.fieldList : undefined;
46
49
  if (rec.list.some(Array.isArray) && !columnarSchemaValid(fields, rec.list)) {
47
- throw markStructural(new ApiError(`${label}: ${securities[i]} returned columnar rows without a usable fieldList (missing, duplicated or mis-sized) — they cannot be read by position`, undefined, undefined, rec));
50
+ throw markStructural(new ApiError(`${label}: ${groups[i].join(", ")} returned columnar rows without a usable fieldList (missing, duplicated or mis-sized) — they cannot be read by position`, undefined, undefined, rec));
48
51
  }
49
52
  if (fields && !fieldList) {
50
53
  fieldList = fields;
51
- headerSecurity = securities[i];
54
+ headerSecurity = groups[i].join(", ");
52
55
  sink?.setFieldList(fieldList);
53
56
  }
54
57
  if ((fields ?? fieldList) && !sameColumns(fieldList, fields)) {
55
- throw markStructural(new ApiError(`${label}: ${securities[i]} answered with columns ${JSON.stringify(fields)} while ${headerSecurity} answered ${JSON.stringify(fieldList)} — the parts cannot be merged`, undefined, undefined, rec));
58
+ throw markStructural(new ApiError(`${label}: ${groups[i].join(", ")} answered with columns ${JSON.stringify(fields)} while ${headerSecurity} answered ${JSON.stringify(fieldList)} — the parts cannot be merged`, undefined, undefined, rec));
56
59
  }
57
60
  if (rec.list.length >= cap)
58
- truncated.push(securities[i]);
59
- count += rec.list.length;
61
+ truncated.push(...groups[i]);
62
+ const rows = inInputOrder(rec.list, fields ?? fieldList, groups[i], label);
63
+ count += rows.length;
60
64
  if (sink)
61
- await sink.push(rec.list);
65
+ await sink.push(rows);
62
66
  else
63
- for (const row of rec.list)
67
+ for (const row of rows)
64
68
  merged.push(row);
65
69
  };
66
- await runInOrder(securities, PAGE_CONCURRENCY, (code) => client.call(endpointKey, makeBody(code)), mergePart);
70
+ await runInOrder(groups, PAGE_CONCURRENCY, (codes) => client.call(endpointKey, makeBody(codes)), mergePart);
67
71
  const out = { total: count, list: merged };
68
72
  if (sink)
69
73
  attachRowSink(out, sink);
@@ -80,6 +84,20 @@ export async function callPerSecurity(client, endpointKey, securities, makeBody,
80
84
  }
81
85
  return out;
82
86
  }
87
+ function inInputOrder(rows, fields, group, label) {
88
+ if (group.length < 2)
89
+ return rows;
90
+ const column = fields?.indexOf("securityCode") ?? -1;
91
+ if (rows.some(Array.isArray) && column < 0) {
92
+ throw markStructural(new ApiError(`${label}: ${group.join(", ")} answered without a securityCode column — rows of several securities cannot be told apart`));
93
+ }
94
+ const rank = new Map(group.map((code, i) => [code.toUpperCase(), i]));
95
+ const codeOf = (row) => Array.isArray(row) ? row[column] : row?.securityCode;
96
+ return rows
97
+ .map((row, i) => ({ row, i, at: rank.get(String(codeOf(row)).toUpperCase()) ?? group.length }))
98
+ .sort((a, b) => a.at - b.at || a.i - b.i)
99
+ .map(({ row }) => row);
100
+ }
83
101
  function sameColumns(a, b) {
84
102
  if (!a || !b)
85
103
  return false;
@@ -5,16 +5,16 @@ import { countOutputRows, digestFile, pickList, renderOutput, saveOutputIfNeeded
5
5
  import { getRowSink } from "./rowSink.js";
6
6
  import { extractTitles, writeTitleCache } from "./titleCache.js";
7
7
  import { CLI_VERSION } from "../version.js";
8
+ import { decidedExitCode, EXIT_SUPERSEDED, markIncomplete, markSuperseded } from "./exitStatus.js";
8
9
  const LARGE_RESULT_ROWS = 50_000;
9
- export const SUPERSEDED_EXIT = 4;
10
+ export const SUPERSEDED_EXIT = EXIT_SUPERSEDED;
10
11
  export async function warnIfSuperseded(output, digest) {
11
12
  const actual = await digestFile(output).catch(() => undefined);
12
13
  if (!actual || actual.sha256 === digest.sha256)
13
14
  return;
14
15
  process.stderr.write(`[gangtise] warning: ${output} was replaced by another export to the same path while this one was finishing — the file there is not this command's output `
15
16
  + `(wrote sha256 ${digest.sha256.slice(0, 12)}…, found ${actual.sha256.slice(0, 12)}…). Give concurrent exports distinct --output paths.\n`);
16
- if (process.exitCode !== 3)
17
- process.exitCode = SUPERSEDED_EXIT;
17
+ markSuperseded();
18
18
  }
19
19
  function warnIfLargeInMemory(items, format) {
20
20
  if (items && items.length >= LARGE_RESULT_ROWS) {
@@ -63,7 +63,7 @@ async function stageExportMeta(output, format, rows, columns, normalized, comple
63
63
  format,
64
64
  rows,
65
65
  complete,
66
- exitCode: complete ? 0 : 3,
66
+ exitCode: decidedExitCode(),
67
67
  command: redactSecrets(process.argv.slice(2)),
68
68
  cliVersion: CLI_VERSION,
69
69
  fetchedAt: localIso(new Date()),
@@ -121,11 +121,10 @@ export async function printData(data, format, output, cache) {
121
121
  if (typeof meta.total === "number" && format !== "json") {
122
122
  process.stderr.write(`Total: ${meta.total}, showing: ${showing}\n`);
123
123
  }
124
- if (meta.partial === true) {
125
- process.exitCode = 3;
126
- }
124
+ if (meta.partial === true)
125
+ markIncomplete();
127
126
  }
128
- const complete = process.exitCode !== 3;
127
+ const complete = decidedExitCode() === 0;
129
128
  if (output) {
130
129
  const staged = format === "csv" || format === "jsonl"
131
130
  ? await stageExportMeta(output, format, streamed ? streamed.dataRows : countOutputRows(normalized, format), columns, normalized, complete)
@@ -3,7 +3,8 @@ import { columnarSchemaValid } from "./normalize.js";
3
3
  import { attachRowSink } from "./rowSink.js";
4
4
  import { isVerbose, PAGE_CONCURRENCY, runInOrder } from "./transport.js";
5
5
  const DAY_MS = 86_400_000;
6
- const ALL_MARKET_LIMIT = 10_000;
6
+ export const QUOTE_MAX_LIMIT = 10_000;
7
+ const ALL_MARKET_LIMIT = QUOTE_MAX_LIMIT;
7
8
  function parseDate(value) {
8
9
  if (!/^\d{4}-\d{2}-\d{2}$/.test(value))
9
10
  return null;
@@ -42,18 +43,21 @@ function isWeekendUtc(d) {
42
43
  }
43
44
  function buildShards(start, end, shardDays) {
44
45
  const shards = [];
45
- let cursor = start.getTime();
46
- const endTime = end.getTime();
47
- while (cursor <= endTime) {
48
- const shardEnd = Math.min(cursor + (shardDays - 1) * DAY_MS, endTime);
49
- if (!(shardDays === 1 && isWeekendUtc(new Date(cursor)))) {
50
- shards.push({
51
- startDate: formatDate(new Date(cursor)),
52
- endDate: formatDate(new Date(shardEnd)),
53
- });
54
- }
55
- cursor = shardEnd + DAY_MS;
56
- }
46
+ let group = [];
47
+ const flush = () => {
48
+ if (group.length === 0)
49
+ return;
50
+ shards.push({ startDate: formatDate(new Date(group[0])), endDate: formatDate(new Date(group[group.length - 1])) });
51
+ group = [];
52
+ };
53
+ for (let day = start.getTime(); day <= end.getTime(); day += DAY_MS) {
54
+ if (isWeekendUtc(new Date(day)))
55
+ continue;
56
+ group.push(day);
57
+ if (group.length === shardDays)
58
+ flush();
59
+ }
60
+ flush();
57
61
  return shards;
58
62
  }
59
63
  export async function callKlineWithSharding(client, endpointKey, body, config) {
@@ -65,7 +69,7 @@ export async function callKlineWithSharding(client, endpointKey, body, config) {
65
69
  const perShardLimit = allMarketBody.limit ?? ALL_MARKET_LIMIT;
66
70
  const truncationHint = perShardLimit < ALL_MARKET_LIMIT
67
71
  ? "raise or omit --limit to fetch the full market"
68
- : `a single ${config.shardDays}-day window exceeds the ${ALL_MARKET_LIMIT}-row API cap`;
72
+ : `a single ${config.shardDays}-weekday window exceeds the ${ALL_MARKET_LIMIT}-row API cap`;
69
73
  const isTruncated = (rec) => Array.isArray(rec.list) && rec.list.length >= perShardLimit;
70
74
  const callSingle = async () => {
71
75
  const single = await client.call(endpointKey, allMarketBody);
@@ -92,7 +96,7 @@ export async function callKlineWithSharding(client, endpointKey, body, config) {
92
96
  if (shards.length === 0)
93
97
  return { list: [] };
94
98
  if (isVerbose()) {
95
- process.stderr.write(`[gangtise] sharding ${endpointKey} into ${shards.length} requests (${config.shardDays} day(s) each)\n`);
99
+ process.stderr.write(`[gangtise] sharding ${endpointKey} into ${shards.length} requests (${config.shardDays} weekday(s) each)\n`);
96
100
  }
97
101
  const failedShards = [];
98
102
  let firstError = null;
@@ -2,7 +2,7 @@ import { createReadStream, createWriteStream } from "node:fs";
2
2
  import fs from "node:fs/promises";
3
3
  import path from "node:path";
4
4
  import { assertColumnarHeader, zipFieldRow } from "./normalize.js";
5
- import { csvEscape, digestFile, formatScalar, stagingPath, writeLine } from "./output.js";
5
+ import { csvEscape, digestFile, formatScalar, stagingPath, writeLine, writeLines, LINES_PER_WRITE } from "./output.js";
6
6
  import { extractTitles, MAX_TITLES_PER_ENDPOINT } from "./titleCache.js";
7
7
  export const ROW_SINK = Symbol("gangtise.rowSink");
8
8
  const STREAM_THRESHOLD = 1000;
@@ -22,6 +22,7 @@ export class ExportSink {
22
22
  columns = [];
23
23
  columnSet = new Set();
24
24
  objectRows = 0;
25
+ earliestDates = new Map();
25
26
  partPath;
26
27
  rowsPath;
27
28
  constructor(outputPath, format = "jsonl", cache) {
@@ -45,7 +46,27 @@ export class ExportSink {
45
46
  setFieldList(fields) {
46
47
  this.fields = fields;
47
48
  }
49
+ watchEarliest(field) {
50
+ if (!this.earliestDates.has(field))
51
+ this.earliestDates.set(field, undefined);
52
+ }
53
+ earliest(field) {
54
+ return this.earliestDates.get(field);
55
+ }
48
56
  async push(rows) {
57
+ for (const [field, current] of this.earliestDates) {
58
+ const index = this.fields ? this.fields.indexOf(field) : -1;
59
+ let earliest = current;
60
+ for (const row of rows) {
61
+ const value = Array.isArray(row) ? (index >= 0 ? row[index] : undefined) : row && typeof row === "object" ? row[field] : undefined;
62
+ if (typeof value !== "string")
63
+ continue;
64
+ const day = value.slice(0, 10);
65
+ if (earliest === undefined || day < earliest)
66
+ earliest = day;
67
+ }
68
+ this.earliestDates.set(field, earliest);
69
+ }
49
70
  if (this.stream) {
50
71
  await this.write(rows);
51
72
  return;
@@ -100,6 +121,7 @@ export class ExportSink {
100
121
  }
101
122
  async write(rows) {
102
123
  const stream = this.stream;
124
+ let chunk = [];
103
125
  for (const row of rows) {
104
126
  let item = row;
105
127
  if (Array.isArray(row)) {
@@ -118,7 +140,11 @@ export class ExportSink {
118
140
  }
119
141
  }
120
142
  }
121
- await writeLine(stream, JSON.stringify(item));
143
+ chunk.push(JSON.stringify(item));
144
+ if (chunk.length >= LINES_PER_WRITE) {
145
+ await writeLines(stream, chunk);
146
+ chunk = [];
147
+ }
122
148
  this.rows++;
123
149
  if (this.cache && this.titleCount < MAX_TITLES_PER_ENDPOINT) {
124
150
  for (const [id, title] of Object.entries(extractTitles([item], this.cache))) {
@@ -128,6 +154,7 @@ export class ExportSink {
128
154
  }
129
155
  }
130
156
  }
157
+ await writeLines(stream, chunk);
131
158
  }
132
159
  async assembleCsv() {
133
160
  const out = createWriteStream(this.partPath, { encoding: "utf8" });
@@ -139,17 +166,18 @@ export class ExportSink {
139
166
  const input = createReadStream(this.rowsPath, { encoding: "utf8" });
140
167
  try {
141
168
  let index = 0;
142
- const emit = async (line) => {
169
+ let csvLines = [];
170
+ const emit = (line) => {
143
171
  if (!line)
144
172
  return;
145
173
  const item = JSON.parse(line);
146
174
  if (objectMode) {
147
175
  if (!isObjectRow(item))
148
176
  return;
149
- await writeLine(out, columns.map((column) => csvEscape(formatScalar(item[column]))).join(","));
177
+ csvLines.push(columns.map((column) => csvEscape(formatScalar(item[column]))).join(","));
150
178
  }
151
179
  else {
152
- await writeLine(out, `${csvEscape(String(index++))},${csvEscape(formatScalar(item))}`);
180
+ csvLines.push(`${csvEscape(String(index++))},${csvEscape(formatScalar(item))}`);
153
181
  }
154
182
  };
155
183
  let rest = "";
@@ -157,9 +185,14 @@ export class ExportSink {
157
185
  const lines = (rest + String(chunk)).split("\n");
158
186
  rest = lines.pop() ?? "";
159
187
  for (const line of lines)
160
- await emit(line);
188
+ emit(line);
189
+ if (csvLines.length >= LINES_PER_WRITE) {
190
+ await writeLines(out, csvLines);
191
+ csvLines = [];
192
+ }
161
193
  }
162
- await emit(rest);
194
+ emit(rest);
195
+ await writeLines(out, csvLines);
163
196
  await endStream(out);
164
197
  }
165
198
  catch (error) {
@@ -1,6 +1,7 @@
1
1
  import fs from "node:fs/promises";
2
2
  import os from "node:os";
3
3
  import path from "node:path";
4
+ import { stagingPath } from "./output.js";
4
5
  export const DEFAULT_TITLE_CACHE_PATH = path.join(os.homedir(), ".config", "gangtise", "title-cache.json");
5
6
  export const TITLE_LOOKUP_SIZE = 200;
6
7
  const TITLE_CACHE_TTL_MS = 24 * 60 * 60 * 1000;
@@ -43,6 +44,29 @@ async function loadInto(filePath) {
43
44
  export async function readTitleCache(filePath = DEFAULT_TITLE_CACHE_PATH) {
44
45
  return loadInto(filePath);
45
46
  }
47
+ async function mergeFromDisk(filePath, data) {
48
+ let disk;
49
+ try {
50
+ disk = JSON.parse(await fs.readFile(filePath, "utf8"));
51
+ }
52
+ catch {
53
+ return;
54
+ }
55
+ if (!disk || typeof disk !== "object" || Array.isArray(disk))
56
+ return;
57
+ for (const [key, entry] of Object.entries(disk)) {
58
+ if (!entry?.titles || typeof entry.titles !== "object" || typeof entry.ts !== "number")
59
+ continue;
60
+ const mine = data[key];
61
+ if (!mine) {
62
+ data[key] = { titles: entry.titles, ts: entry.ts };
63
+ continue;
64
+ }
65
+ const titles = { ...entry.titles, ...mine.titles };
66
+ data[key] = { titles: capTitles(titles, Object.keys(mine.titles), MAX_TITLES_PER_ENDPOINT), ts: Math.max(entry.ts, mine.ts) };
67
+ }
68
+ pruneCache(data, "", []);
69
+ }
46
70
  async function flush(filePath) {
47
71
  for (;;) {
48
72
  if (!dirty || !memoryCache) {
@@ -50,9 +74,11 @@ async function flush(filePath) {
50
74
  return;
51
75
  }
52
76
  dirty = false;
53
- const snapshot = JSON.stringify(memoryCache);
77
+ const data = memoryCache;
78
+ await mergeFromDisk(filePath, data);
79
+ const snapshot = JSON.stringify(data);
54
80
  await fs.mkdir(path.dirname(filePath), { recursive: true });
55
- const tmp = `${filePath}.tmp-${process.pid}-${Date.now()}`;
81
+ const tmp = stagingPath(filePath, "tmp");
56
82
  await fs.writeFile(tmp, snapshot, { encoding: "utf8", mode: 0o600 });
57
83
  try {
58
84
  await fs.rename(tmp, filePath);
@@ -1,5 +1,5 @@
1
+ import { createRequire } from "node:module";
1
2
  import { gunzipSync } from "node:zlib";
2
- import { Agent } from "undici";
3
3
  import { ApiError } from "./errors.js";
4
4
  export function decodeResponseBody(buf, contentEncoding) {
5
5
  const enc = (Array.isArray(contentEncoding) ? contentEncoding[0] : contentEncoding)?.toLowerCase().trim();
@@ -34,12 +34,14 @@ function retryAfterFromError(error) {
34
34
  return undefined;
35
35
  }
36
36
  let cachedDispatcher = null;
37
+ const requireLazily = createRequire(import.meta.url);
37
38
  export function getDispatcher() {
38
39
  if (!cachedDispatcher) {
40
+ const { Agent } = requireLazily("undici");
39
41
  cachedDispatcher = new Agent({
40
42
  keepAliveTimeout: 60_000,
41
43
  keepAliveMaxTimeout: 600_000,
42
- connections: 16,
44
+ connections: Math.max(16, PAGE_CONCURRENCY),
43
45
  pipelining: 1,
44
46
  });
45
47
  }
@@ -62,8 +64,9 @@ export async function runWithConcurrency(items, concurrency, fn) {
62
64
  await Promise.all(Array.from({ length: limit }, () => worker()));
63
65
  return results;
64
66
  }
65
- export async function runInOrder(items, concurrency, fn, consume) {
67
+ export async function runInOrder(items, concurrency, fn, consume, window = concurrency) {
66
68
  const width = Math.max(1, concurrency);
69
+ const maxPending = Math.max(width, window);
67
70
  const pending = new Map();
68
71
  let next = 0;
69
72
  let failure = null;
@@ -104,7 +107,7 @@ export async function runInOrder(items, concurrency, fn, consume) {
104
107
  return;
105
108
  pending.set(index, result);
106
109
  drain();
107
- while (!failure && pending.size >= width) {
110
+ while (!failure && pending.size >= maxPending) {
108
111
  await new Promise((resolve) => waiters.push(resolve));
109
112
  }
110
113
  });
@@ -0,0 +1,51 @@
1
+ import { mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
2
+ import os from "node:os";
3
+ import path from "node:path";
4
+ import { isVersionNewer } from "./args.js";
5
+ import { stagingPath } from "./output.js";
6
+ export const UPDATE_CHECK_TTL_MS = 24 * 60 * 60 * 1000;
7
+ export const DEFAULT_UPDATE_CHECK_PATH = path.join(os.homedir(), ".config", "gangtise", "update-check.json");
8
+ function saveLatest(cachePath, latest, now) {
9
+ const staging = stagingPath(cachePath, "tmp");
10
+ try {
11
+ mkdirSync(path.dirname(cachePath), { recursive: true });
12
+ writeFileSync(staging, JSON.stringify({ checkedAt: now, latest }));
13
+ renameSync(staging, cachePath);
14
+ }
15
+ catch {
16
+ try {
17
+ rmSync(staging, { force: true });
18
+ }
19
+ catch { }
20
+ }
21
+ }
22
+ export async function latestVersion(cachePath = DEFAULT_UPDATE_CHECK_PATH, timeoutMs = 2000, now = Date.now()) {
23
+ const cacheable = path.isAbsolute(cachePath);
24
+ if (cacheable) {
25
+ try {
26
+ const cached = JSON.parse(readFileSync(cachePath, "utf8"));
27
+ const age = typeof cached.checkedAt === "number" ? now - cached.checkedAt : -1;
28
+ if (typeof cached.latest === "string" && age >= 0 && age < UPDATE_CHECK_TTL_MS)
29
+ return cached.latest;
30
+ }
31
+ catch { }
32
+ }
33
+ try {
34
+ const response = await fetch("https://registry.npmjs.org/gangtise-openapi-cli/latest", { signal: AbortSignal.timeout(timeoutMs) });
35
+ const latest = (await response.json()).version;
36
+ if (typeof latest !== "string")
37
+ return undefined;
38
+ if (cacheable)
39
+ saveLatest(cachePath, latest, now);
40
+ return latest;
41
+ }
42
+ catch {
43
+ return undefined;
44
+ }
45
+ }
46
+ export async function checkForUpdate(current) {
47
+ const latest = await latestVersion();
48
+ if (latest && isVersionNewer(latest, current)) {
49
+ process.stderr.write(`Update available: ${current} → ${latest}\nRun: npm update -g gangtise-openapi-cli\n`);
50
+ }
51
+ }
@@ -1 +1,19 @@
1
- export const CLI_VERSION = "0.41.1";
1
+ import fs from "node:fs";
2
+ import path from "node:path";
3
+ import { fileURLToPath } from "node:url";
4
+ function packageVersion() {
5
+ let dir = path.dirname(fileURLToPath(import.meta.url));
6
+ for (;;) {
7
+ const file = path.join(dir, "package.json");
8
+ if (fs.existsSync(file)) {
9
+ const pkg = JSON.parse(fs.readFileSync(file, "utf8"));
10
+ if (pkg.name === "gangtise-openapi-cli" && pkg.version)
11
+ return pkg.version;
12
+ }
13
+ const parent = path.dirname(dir);
14
+ if (parent === dir)
15
+ throw new Error("gangtise-openapi-cli: its package.json was not found above " + fileURLToPath(import.meta.url));
16
+ dir = parent;
17
+ }
18
+ }
19
+ export const CLI_VERSION = packageVersion();