gangtise-openapi-cli 0.37.1 → 0.39.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -131,6 +131,7 @@ function epochMillis(value) {
131
131
  return Number(value) * 1000;
132
132
  return undefined;
133
133
  }
134
+ const BEIJING_OFFSET = "+08:00";
134
135
  export function toTimestamp13(value) {
135
136
  if (value === undefined)
136
137
  return undefined;
@@ -140,13 +141,9 @@ export function toTimestamp13(value) {
140
141
  if (!datetimeFieldsValid(value))
141
142
  return undefined;
142
143
  const parts = LOCAL_DATETIME.exec(value);
143
- const [, y, , mo, d, hh = "0", mi = "0", ss = "0"] = parts;
144
- const year = Number(y);
145
- const dt = new Date(year, Number(mo) - 1, Number(d), Number(hh), Number(mi), Number(ss));
146
- const faithful = dt.getFullYear() === year && dt.getMonth() === Number(mo) - 1
147
- && dt.getDate() === Number(d) && dt.getHours() === Number(hh)
148
- && dt.getMinutes() === Number(mi) && dt.getSeconds() === Number(ss);
149
- return faithful ? dt.getTime() : undefined;
144
+ const [, y, , mo, d, hh = "00", mi = "00", ss = "00"] = parts;
145
+ const instant = new Date(`${y}-${mo}-${d}T${hh}:${mi}:${ss}${BEIJING_OFFSET}`).getTime();
146
+ return Number.isNaN(instant) ? undefined : instant;
150
147
  }
151
148
  export function parseTimestamp13(value, optionName) {
152
149
  if (value === undefined)
@@ -4,18 +4,29 @@ import path from "node:path";
4
4
  import { pipeline } from "node:stream/promises";
5
5
  import { FormData, request } from "undici";
6
6
  import { isTokenCacheValid, normalizeToken, readTokenCache, requireAccessCredentials, writeTokenCache } from "./auth.js";
7
- import { ApiError, attachEnvelopeTraceId, ValidationError } from "./errors.js";
7
+ import { ApiError, attachEnvelopeTraceId, markStructural, ValidationError } from "./errors.js";
8
8
  import { ENDPOINTS, resolveTimeoutMs } from "./endpoints.js";
9
9
  import { getLookupData } from "./lookupData/index.js";
10
- import { decodeResponseBody, getDispatcher, isVerbose, logTiming, markRetryable, PAGE_CONCURRENCY, parseRetryAfterMs, quoteBigIntFields, runWithConcurrency, withRetry } from "./transport.js";
10
+ import { stagingPath } from "./output.js";
11
+ import { decodeResponseBody, getDispatcher, isVerbose, logTiming, markRetryable, PAGE_CONCURRENCY, parseRetryAfterMs, quoteBigIntFields, runInOrder, withRetry } from "./transport.js";
12
+ import { attachRowSink } from "./rowSink.js";
11
13
  const AUTH_RETRY_CODES = new Set(["8000014", "8000015", "0000001008", "999002"]);
12
14
  export class GangtiseClient {
13
15
  config;
16
+ rowSink;
14
17
  refreshPromise = null;
15
18
  memoCache = null;
16
19
  envTokenInvalidated = false;
17
- constructor(config) {
20
+ rowSinkClaimed = false;
21
+ constructor(config, rowSink) {
18
22
  this.config = config;
23
+ this.rowSink = rowSink;
24
+ }
25
+ claimRowSink() {
26
+ if (!this.rowSink || this.rowSinkClaimed)
27
+ return undefined;
28
+ this.rowSinkClaimed = true;
29
+ return this.rowSink;
19
30
  }
20
31
  async getAuthorizationHeader(forceRefresh = false) {
21
32
  if (this.config.token && !this.envTokenInvalidated && !forceRefresh) {
@@ -151,27 +162,42 @@ export class GangtiseClient {
151
162
  return firstPage;
152
163
  }
153
164
  const total = firstPage.total;
154
- const collected = [...firstPage.list];
155
165
  const available = Math.max(total - startFrom, 0);
156
166
  const target = requestedSize === undefined ? available : Math.min(requestedSize, available);
167
+ const sink = this.claimRowSink();
168
+ if (sink && Array.isArray(firstPage.fieldList))
169
+ sink.setFieldList(firstPage.fieldList);
170
+ const collected = [];
171
+ let count = 0;
172
+ const keep = async (rows) => {
173
+ const kept = requestedSize === undefined ? rows : rows.slice(0, Math.max(0, requestedSize - count));
174
+ if (kept.length === 0)
175
+ return;
176
+ count += kept.length;
177
+ if (sink)
178
+ await sink.push(kept);
179
+ else
180
+ for (const row of kept)
181
+ collected.push(row);
182
+ };
183
+ const result = () => {
184
+ const out = { ...firstPage, total, list: collected };
185
+ return sink ? attachRowSink(out, sink) : out;
186
+ };
187
+ await keep(firstPage.list);
157
188
  if (firstPage.list.length < firstPageSize) {
158
- const out = {
159
- ...firstPage,
160
- total,
161
- list: requestedSize === undefined ? collected : collected.slice(0, requestedSize),
162
- };
163
- if (collected.length < target) {
164
- process.stderr.write(`[gangtise] warning: server returned a short page (${collected.length} rows) but reported total=${total}; treating it as the end of data — results may be incomplete\n`);
189
+ const out = result();
190
+ if (count < target) {
191
+ process.stderr.write(`[gangtise] warning: server returned a short page (${count} rows) but reported total=${total}; treating it as the end of data — results may be incomplete\n`);
165
192
  out.partial = true;
166
193
  }
167
194
  return out;
168
195
  }
169
- if (collected.length >= target) {
170
- return {
171
- ...firstPage,
172
- total,
173
- list: requestedSize === undefined ? collected : collected.slice(0, requestedSize),
174
- };
196
+ if (count >= target) {
197
+ const out = result();
198
+ if (requestedSize === undefined && total > firstPageSize)
199
+ await this.flagIfTotalCapped(endpoint, initialBody, total, out, count);
200
+ return out;
175
201
  }
176
202
  const MAX_PAGES = 1000;
177
203
  let truncatedByPageCap = false;
@@ -190,10 +216,11 @@ export class GangtiseClient {
190
216
  }
191
217
  let unexpectedShape = false;
192
218
  let totalDrift = false;
219
+ let laterPartial = false;
193
220
  const failedPages = [];
194
221
  let firstError = null;
195
222
  let aborted = false;
196
- const pages = await runWithConcurrency(pageRequests, PAGE_CONCURRENCY, async (req) => {
223
+ await runInOrder(pageRequests, PAGE_CONCURRENCY, async (req) => {
197
224
  if (aborted) {
198
225
  failedPages.push(req);
199
226
  return [];
@@ -211,6 +238,8 @@ export class GangtiseClient {
211
238
  }
212
239
  if (page.total !== total)
213
240
  totalDrift = true;
241
+ if (page.partial === true)
242
+ laterPartial = true;
214
243
  return page.list;
215
244
  }
216
245
  catch (error) {
@@ -220,12 +249,7 @@ export class GangtiseClient {
220
249
  failedPages.push(req);
221
250
  return [];
222
251
  }
223
- });
224
- for (const list of pages) {
225
- if (list.length === 0)
226
- continue;
227
- collected.push(...list);
228
- }
252
+ }, (list) => keep(list));
229
253
  if (unexpectedShape) {
230
254
  process.stderr.write(`[gangtise] warning: a page response had unexpected shape; its rows are missing (counted in failedPages)\n`);
231
255
  }
@@ -233,44 +257,45 @@ export class GangtiseClient {
233
257
  process.stderr.write(`[gangtise] warning: 'total' changed across pages (data shifted during fetch); rows may be duplicated or missing\n`);
234
258
  }
235
259
  if (truncatedByPageCap) {
236
- process.stderr.write(`[gangtise] warning: hit the ${MAX_PAGES}-page safety cap; fetched ${collected.length} of ${total} rows. Narrow the query (e.g. a shorter date range) or pass --size to fetch a bounded subset.\n`);
260
+ process.stderr.write(`[gangtise] warning: hit the ${MAX_PAGES}-page safety cap; fetched ${count} of ${total} rows. Narrow the query (e.g. a shorter date range) or pass --size to fetch a bounded subset.\n`);
237
261
  }
238
- const short = collected.length < target;
239
- let totalCapped = false;
262
+ const short = count < target;
263
+ const out = result();
240
264
  if (requestedSize === undefined && total > 0 && !short && !totalDrift && !truncatedByPageCap && failedPages.length === 0) {
241
- try {
242
- const probe = await this.requestJson(endpoint, { ...initialBody, from: total, size: 1 });
243
- if (this.isPaginatedListResponse(probe) && probe.list.length > 0)
244
- totalCapped = true;
245
- }
246
- catch {
247
- }
248
- }
249
- if (totalCapped) {
250
- process.stderr.write(`[gangtise] warning: ${endpoint.key} reported total=${total} but rows exist past that offset — 'total' is a server-side cap, not the real count. This export is TRUNCATED at ${collected.length} rows. Narrow the query (date range / filters) and fetch in slices.\n`);
251
- }
252
- const out = {
253
- ...firstPage,
254
- total,
255
- list: requestedSize === undefined ? collected : collected.slice(0, requestedSize),
256
- };
257
- if (totalCapped) {
258
- out.partial = true;
259
- out.totalCapped = true;
265
+ await this.flagIfTotalCapped(endpoint, initialBody, total, out, count);
260
266
  }
261
267
  if (short || totalDrift || failedPages.length > 0)
262
268
  out.partial = true;
269
+ if (laterPartial) {
270
+ out.partial = true;
271
+ process.stderr.write(`[gangtise] warning: a later page reported itself partial; the merged result is marked partial\n`);
272
+ }
263
273
  if (failedPages.length > 0) {
264
274
  out.failedPages = failedPages.map((p) => ({ from: p.from, size: p.size }));
265
275
  const detail = firstError instanceof Error ? `: ${firstError.message}` : "";
266
276
  const skippedHint = aborted ? " A page hit a non-retryable error (e.g. rate limit); remaining pages were skipped." : "";
267
- process.stderr.write(`[gangtise] warning: ${failedPages.length}/${pageRequests.length} pages not fetched${detail}; results are partial — got ${collected.length}/${total} rows (see failedPages).${skippedHint}\n`);
277
+ process.stderr.write(`[gangtise] warning: ${failedPages.length}/${pageRequests.length} pages not fetched${detail}; results are partial — got ${count}/${total} rows (see failedPages).${skippedHint}\n`);
268
278
  }
269
279
  else if (short && !truncatedByPageCap && !totalDrift) {
270
- process.stderr.write(`[gangtise] warning: server returned ${collected.length} of ${total} rows (a later page came back short); results may be incomplete\n`);
280
+ process.stderr.write(`[gangtise] warning: server returned ${count} of ${total} rows (a later page came back short); results may be incomplete\n`);
271
281
  }
272
282
  return out;
273
283
  }
284
+ async flagIfTotalCapped(endpoint, initialBody, total, out, collectedLength) {
285
+ let totalCapped = false;
286
+ try {
287
+ const probe = await this.requestJson(endpoint, { ...initialBody, from: total, size: 1 });
288
+ if (this.isPaginatedListResponse(probe) && probe.list.length > 0)
289
+ totalCapped = true;
290
+ }
291
+ catch {
292
+ }
293
+ if (!totalCapped)
294
+ return;
295
+ process.stderr.write(`[gangtise] warning: ${endpoint.key} reported total=${total} but rows exist past that offset — 'total' is a server-side cap, not the real count. This export is TRUNCATED at ${collectedLength} rows. Narrow the query (date range / filters) and fetch in slices.\n`);
296
+ out.partial = true;
297
+ out.totalCapped = true;
298
+ }
274
299
  async login() {
275
300
  const authorization = await this.getAuthorizationHeader();
276
301
  const cache = await readTokenCache(this.config.tokenCachePath);
@@ -340,7 +365,11 @@ export class GangtiseClient {
340
365
  if (response.statusCode >= 400) {
341
366
  this.throwHttpError(parsed, response.statusCode, retryAfterMs);
342
367
  }
343
- return this.unwrapEnvelope(parsed, response.statusCode, retryAfterMs);
368
+ const payload = this.unwrapEnvelope(parsed, response.statusCode, retryAfterMs);
369
+ if (endpoint.expects === "list" && !(payload && typeof payload === "object" && Array.isArray(payload.list))) {
370
+ throw markStructural(new ApiError(`${endpoint.key} returned no list payload (got ${payload === null ? "null" : Array.isArray(payload) ? "an array" : typeof payload}) — the response layout may have changed`, undefined, response.statusCode, parsed));
371
+ }
372
+ return payload;
344
373
  }
345
374
  catch (error) {
346
375
  await this.refreshAuthIfRecoverable(error, useAuth, authState, usedAuthorization);
@@ -476,7 +505,7 @@ export class GangtiseClient {
476
505
  }
477
506
  if (options?.streamTo) {
478
507
  await fs.mkdir(path.dirname(options.streamTo), { recursive: true });
479
- const partPath = `${options.streamTo}.part`;
508
+ const partPath = stagingPath(options.streamTo);
480
509
  try {
481
510
  await pipeline(response.body, createWriteStream(partPath));
482
511
  await fs.rename(partPath, options.streamTo);
@@ -519,6 +548,6 @@ export class GangtiseClient {
519
548
  if (endpoint.kind === 'json' && endpoint.pagination?.enabled) {
520
549
  return this.requestPaginated(endpoint, body);
521
550
  }
522
- return this.requestJson(endpoint, body);
551
+ return this.requestJson(endpoint, body, endpoint.key !== 'auth.login');
523
552
  }
524
553
  }
@@ -1,7 +1,7 @@
1
1
  import fs from "node:fs/promises";
2
- import { extname } from "node:path";
2
+ import { dirname, extname } from "node:path";
3
3
  import { DownloadError } from "./errors.js";
4
- import { saveOutputIfNeeded } from "./output.js";
4
+ import { saveOutputIfNeeded, stagingPath } from "./output.js";
5
5
  import { extractTitles, lookupTitleCache, readTitleCache, TITLE_LOOKUP_SIZE, writeTitleCache } from "./titleCache.js";
6
6
  function sanitizeFilename(name) {
7
7
  return name.replace(/[/\\:*?"<>|\u0000-\u001f]/g, "_");
@@ -16,18 +16,38 @@ function truncateFilename(name, maxBytes = 200) {
16
16
  return stem.join("") + ext;
17
17
  }
18
18
  export async function uniquePath(p) {
19
- const exists = (f) => fs.access(f).then(() => true, () => false);
20
- if (!(await exists(p)))
19
+ const claim = async (f) => {
20
+ try {
21
+ const handle = await fs.open(f, "wx");
22
+ await handle.close();
23
+ return true;
24
+ }
25
+ catch (error) {
26
+ if (error.code === "EEXIST")
27
+ return false;
28
+ throw error;
29
+ }
30
+ };
31
+ await fs.mkdir(dirname(p) || ".", { recursive: true });
32
+ if (await claim(p))
21
33
  return p;
22
34
  const ext = extname(p);
23
35
  const stem = p.slice(0, p.length - ext.length);
24
36
  for (let i = 1; i <= 99; i++) {
25
37
  const candidate = `${stem}-${i}${ext}`;
26
- if (!(await exists(candidate)))
38
+ if (await claim(candidate))
27
39
  return candidate;
28
40
  }
29
41
  throw new DownloadError(`Refusing to overwrite: 100 files already share the name "${p}" — pass --output or clean up the directory`);
30
42
  }
43
+ export async function releaseClaim(p) {
44
+ try {
45
+ if ((await fs.stat(p)).size === 0)
46
+ await fs.unlink(p);
47
+ }
48
+ catch {
49
+ }
50
+ }
31
51
  const MIME_EXT = {
32
52
  "application/pdf": ".pdf",
33
53
  "application/msword": ".doc",
@@ -119,7 +139,7 @@ async function downloadUrlTo(url, outputPath) {
119
139
  return "signed-url";
120
140
  }
121
141
  };
122
- const partPath = `${outputPath}.part`;
142
+ const partPath = stagingPath(outputPath);
123
143
  await withRetry(async () => {
124
144
  const startedAt = Date.now();
125
145
  const requestOptions = {
@@ -178,13 +198,27 @@ export async function saveDownloadResult(result, fallbackName, output) {
178
198
  if (file.data instanceof Uint8Array) {
179
199
  const autoName = (file.filename ? sanitizeFilename(file.filename) : undefined) ?? (safeFallback + extFromContentType(file.contentType));
180
200
  const outputPath = output ?? await uniquePath(truncateFilename(autoName));
181
- await saveOutputIfNeeded(file.data, outputPath);
201
+ try {
202
+ await saveOutputIfNeeded(file.data, outputPath);
203
+ }
204
+ catch (error) {
205
+ if (!output)
206
+ await releaseClaim(outputPath);
207
+ throw error;
208
+ }
182
209
  process.stdout.write(`${outputPath}\n`);
183
210
  return;
184
211
  }
185
212
  if (typeof file.text === "string") {
186
213
  const outputPath = output ?? await uniquePath(truncateFilename(`${safeFallback}.txt`));
187
- await saveOutputIfNeeded(file.text, outputPath);
214
+ try {
215
+ await saveOutputIfNeeded(file.text, outputPath);
216
+ }
217
+ catch (error) {
218
+ if (!output)
219
+ await releaseClaim(outputPath);
220
+ throw error;
221
+ }
188
222
  process.stdout.write(`${outputPath}\n`);
189
223
  return;
190
224
  }
@@ -273,42 +273,49 @@ const ENDPOINT_DEFS = {
273
273
  path: "/application/open-quote/kline/daily",
274
274
  kind: "json",
275
275
  description: "Query A-share historical daily kline (SH/SZ/BJ)",
276
+ expects: "list",
276
277
  },
277
278
  "quote.day-kline-hk": {
278
279
  method: "POST",
279
280
  path: "/application/open-quote/kline-hk/daily",
280
281
  kind: "json",
281
282
  description: "Query HK stock historical daily kline (HK)",
283
+ expects: "list",
282
284
  },
283
285
  "quote.day-kline-us": {
284
286
  method: "POST",
285
287
  path: "/application/open-quote/kline-us/daily",
286
288
  kind: "json",
287
289
  description: "Query US stock historical daily kline (NYSE/NASDAQ/AMEX)",
290
+ expects: "list",
288
291
  },
289
292
  "quote.index-day-kline": {
290
293
  method: "POST",
291
294
  path: "/application/open-quote/index/kline/daily",
292
295
  kind: "json",
293
296
  description: "Query SH/SZ/BJ index daily kline",
297
+ expects: "list",
294
298
  },
295
299
  "quote.minute-kline": {
296
300
  method: "POST",
297
301
  path: "/application/open-quote/kline/minute",
298
302
  kind: "json",
299
303
  description: "Query A-share minute kline (SH/SZ/BJ)",
304
+ expects: "list",
300
305
  },
301
306
  "quote.realtime": {
302
307
  method: "POST",
303
308
  path: "/application/open-quote/quote/realtime",
304
309
  kind: "json",
305
310
  description: "Query realtime quote snapshot (A-share / HK / US)",
311
+ expects: "list",
306
312
  },
307
313
  "quote.fund-flow": {
308
314
  method: "POST",
309
315
  path: "/application/open-quote/fund-flow/daily",
310
316
  kind: "json",
311
317
  description: "Query A-share daily fund flow (SH/SZ/BJ; small/medium/large/xlarge orders + main net inflow)",
318
+ expects: "list",
312
319
  },
313
320
  "fundamental.income-statement": {
314
321
  method: "POST",
@@ -17,6 +17,14 @@ export function attachEnvelopeTraceId(payload, traceId) {
17
17
  }
18
18
  return payload;
19
19
  }
20
+ const STRUCTURAL = Symbol("gangtise.structural");
21
+ export function markStructural(error) {
22
+ Object.defineProperty(error, STRUCTURAL, { value: true, enumerable: false, configurable: true });
23
+ return error;
24
+ }
25
+ export function isStructuralError(error) {
26
+ return Boolean(error && typeof error === "object" && error[STRUCTURAL] === true);
27
+ }
20
28
  const ERROR_HINTS = {
21
29
  "999001": "检查 GANGTISE_TOKEN 或 AK/SK 是否已 export。",
22
30
  "999002": "有 AK/SK 时 CLI 会自动重新登录重试一次,否则请重新登录。",
@@ -14,6 +14,37 @@ export function zipFieldRow(fields, row, source) {
14
14
  return acc;
15
15
  }, {});
16
16
  }
17
+ export function columnarSchemaValid(fields, list) {
18
+ if (!fields)
19
+ return false;
20
+ if (new Set(fields.map(String)).size !== fields.length)
21
+ return false;
22
+ return list.every((row) => !Array.isArray(row) || row.length === fields.length);
23
+ }
24
+ export function flagMissingFields(data, requested, label) {
25
+ if (!requested?.length || !data || typeof data !== "object" || Array.isArray(data))
26
+ return;
27
+ const rec = data;
28
+ if (!Array.isArray(rec.fieldList))
29
+ return;
30
+ const returned = new Set(rec.fieldList.map(String));
31
+ const missing = requested.filter((field) => !returned.has(field));
32
+ if (missing.length === 0)
33
+ return;
34
+ rec.partial = true;
35
+ rec.missingFields = missing;
36
+ process.stderr.write(`[gangtise] warning: ${label} returned no column for ${missing.join(", ")} — the server drops a field name it does not recognise (or no longer serves) without an error. Check the name against references/fields.md; result marked partial (exit 3).\n`);
37
+ }
38
+ export function assertColumnarHeader(fields, source) {
39
+ if (!fields) {
40
+ throw new ValidationError(`响应包含数组形式的行但没有 fieldList,无法确定各列的含义,已拒绝输出。这是上游响应结构异常,请报障${traceSuffix(source)}。`);
41
+ }
42
+ const names = fields.map(String);
43
+ const dupes = [...new Set(names.filter((name, i) => names.indexOf(name) !== i))];
44
+ if (dupes.length > 0) {
45
+ throw new ValidationError(`响应 fieldList 有重复列名(${dupes.join(", ")})——按位置拍平时后一列会覆盖前一列,已拒绝输出。这是上游响应结构异常,请报障${traceSuffix(source)}。`);
46
+ }
47
+ }
17
48
  export function normalizeRows(value) {
18
49
  if (!value || typeof value !== "object") {
19
50
  return value;
@@ -24,12 +55,16 @@ export function normalizeRows(value) {
24
55
  const record = value;
25
56
  if (Array.isArray(record.fieldList) && Array.isArray(record.list)) {
26
57
  const fields = record.fieldList;
58
+ if (record.list.some(Array.isArray))
59
+ assertColumnarHeader(fields, record);
27
60
  const normalizedList = record.list.map((row) => (Array.isArray(row) ? zipFieldRow(fields, row, record) : row));
28
61
  const { fieldList, list, ...meta } = record;
29
62
  const hasMeta = Object.keys(meta).length > 0;
30
63
  return hasMeta ? { ...meta, list: normalizedList } : normalizedList;
31
64
  }
32
65
  if (Array.isArray(record.list)) {
66
+ if (record.list.some(Array.isArray))
67
+ assertColumnarHeader(undefined, record);
33
68
  const { list, ...meta } = record;
34
69
  const hasMeta = Object.keys(meta).length > 0;
35
70
  return hasMeta ? { ...meta, list } : list;
@@ -1,6 +1,26 @@
1
+ import { createHash, randomBytes } from "node:crypto";
1
2
  import fs from "node:fs/promises";
2
3
  import { ConfigError } from "./errors.js";
3
4
  const OUTPUT_FORMATS = ["table", "json", "jsonl", "csv", "markdown"];
5
+ export function stagingPath(target, suffix = "part") {
6
+ return `${target}.${randomBytes(4).toString("hex")}.${suffix}`;
7
+ }
8
+ export function digestBuffer(content) {
9
+ const buf = typeof content === "string" ? Buffer.from(content, "utf8") : content;
10
+ return { bytes: buf.byteLength, sha256: createHash("sha256").update(buf).digest("hex") };
11
+ }
12
+ export async function digestFile(filePath) {
13
+ const { createReadStream } = await import("node:fs");
14
+ const hash = createHash("sha256");
15
+ let bytes = 0;
16
+ const stream = createReadStream(filePath);
17
+ for await (const chunk of stream) {
18
+ const buf = chunk;
19
+ bytes += buf.byteLength;
20
+ hash.update(buf);
21
+ }
22
+ return { bytes, sha256: hash.digest("hex") };
23
+ }
4
24
  export function parseOutputFormat(value) {
5
25
  const format = value ?? "table";
6
26
  if (OUTPUT_FORMATS.includes(format)) {
@@ -24,7 +44,7 @@ function displayWidth(value) {
24
44
  }
25
45
  return width;
26
46
  }
27
- function formatScalar(value) {
47
+ export function formatScalar(value) {
28
48
  if (value === null || value === undefined) {
29
49
  return "";
30
50
  }
@@ -104,16 +124,19 @@ function renderCsv(rows) {
104
124
  const body = rows.map((row) => columns.map((column) => csvEscape(formatScalar(row[column]))).join(","));
105
125
  return [header, ...body].join("\n");
106
126
  }
127
+ export function jsonlItems(value) {
128
+ const list = value && typeof value === "object" && !Array.isArray(value) ? value.list : undefined;
129
+ return Array.isArray(list) ? list : toRows(value);
130
+ }
131
+ export function countOutputRows(value, format) {
132
+ return format === "jsonl" ? jsonlItems(value).length : toRows(value).length;
133
+ }
107
134
  export function renderOutput(value, format) {
108
135
  switch (format) {
109
136
  case "json":
110
137
  return JSON.stringify(value, null, 2);
111
- case "jsonl": {
112
- const items = value && typeof value === "object" && !Array.isArray(value) && Array.isArray(value.list)
113
- ? value.list
114
- : null;
115
- return (items ?? toRows(value)).map((item) => JSON.stringify(item)).join("\n");
116
- }
138
+ case "jsonl":
139
+ return jsonlItems(value).map((item) => JSON.stringify(item)).join("\n");
117
140
  case "csv":
118
141
  return renderCsv(toRows(value));
119
142
  case "markdown":
@@ -125,29 +148,29 @@ export function renderOutput(value, format) {
125
148
  }
126
149
  export async function streamOutputToFile(value, format, outputPath) {
127
150
  if (format !== "jsonl" && format !== "csv")
128
- return false;
151
+ return null;
129
152
  const list = pickList(value);
130
153
  if (!list)
131
- return false;
154
+ return null;
132
155
  if (list.length < 1000)
133
- return false;
156
+ return null;
134
157
  let csvRows = [];
135
158
  let csvColumns = [];
136
159
  if (format === "csv") {
137
160
  csvRows = list.filter((row) => Boolean(row && typeof row === "object" && !Array.isArray(row)));
138
161
  if (csvRows.length === 0)
139
- return false;
162
+ return null;
140
163
  csvColumns = Array.from(new Set(csvRows.flatMap((row) => Object.keys(row))));
141
164
  }
142
165
  const { dirname } = await import("node:path");
143
166
  const { createWriteStream } = await import("node:fs");
144
167
  await fs.mkdir(dirname(outputPath), { recursive: true });
145
- const partPath = `${outputPath}.part`;
168
+ const partPath = stagingPath(outputPath);
146
169
  const stream = createWriteStream(partPath, { encoding: "utf8" });
147
170
  stream.on("error", () => { });
148
171
  try {
149
172
  if (format === "jsonl") {
150
- for (const item of list) {
173
+ for (const item of jsonlItems(value)) {
151
174
  await writeLine(stream, JSON.stringify(item));
152
175
  }
153
176
  }
@@ -161,7 +184,9 @@ export async function streamOutputToFile(value, format, outputPath) {
161
184
  await new Promise((resolve, reject) => {
162
185
  stream.end((err) => err ? reject(err) : resolve());
163
186
  });
187
+ const digest = await digestFile(partPath);
164
188
  await fs.rename(partPath, outputPath);
189
+ return digest;
165
190
  }
166
191
  catch (error) {
167
192
  stream.destroy();
@@ -171,7 +196,6 @@ export async function streamOutputToFile(value, format, outputPath) {
171
196
  await fs.unlink(partPath).catch(() => { });
172
197
  throw error;
173
198
  }
174
- return true;
175
199
  }
176
200
  export function pickList(value) {
177
201
  if (Array.isArray(value))
@@ -183,7 +207,7 @@ export function pickList(value) {
183
207
  }
184
208
  return null;
185
209
  }
186
- function csvEscape(value) {
210
+ export function csvEscape(value) {
187
211
  let out = value;
188
212
  if (/^[=@\t\r]/.test(out) || (/^[+\-]/.test(out) && !Number.isFinite(Number(out))))
189
213
  out = "'" + out;
@@ -191,7 +215,7 @@ function csvEscape(value) {
191
215
  return `"${out.replaceAll("\"", "\"\"")}"`;
192
216
  return out;
193
217
  }
194
- function writeLine(stream, line) {
218
+ export function writeLine(stream, line) {
195
219
  return new Promise((resolve, reject) => {
196
220
  const ok = stream.write(line + "\n", (err) => err ? reject(err) : undefined);
197
221
  if (ok) {
@@ -210,7 +234,7 @@ export async function saveOutputIfNeeded(content, outputPath) {
210
234
  }
211
235
  const { dirname } = await import("node:path");
212
236
  await fs.mkdir(dirname(outputPath), { recursive: true });
213
- const partPath = `${outputPath}.part`;
237
+ const partPath = stagingPath(outputPath);
214
238
  try {
215
239
  if (typeof content === "string") {
216
240
  await fs.writeFile(partPath, content, "utf8");
@@ -218,7 +242,9 @@ export async function saveOutputIfNeeded(content, outputPath) {
218
242
  else {
219
243
  await fs.writeFile(partPath, content);
220
244
  }
245
+ const digest = digestBuffer(content);
221
246
  await fs.rename(partPath, outputPath);
247
+ return digest;
222
248
  }
223
249
  catch (error) {
224
250
  await fs.unlink(partPath).catch(() => { });