gangtise-openapi-cli 0.37.1 → 0.39.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +50 -39
- package/dist/src/cli.js +96 -28
- package/dist/src/core/args.js +4 -7
- package/dist/src/core/client.js +80 -51
- package/dist/src/core/download.js +42 -8
- package/dist/src/core/endpoints.js +7 -0
- package/dist/src/core/errors.js +8 -0
- package/dist/src/core/normalize.js +35 -0
- package/dist/src/core/output.js +43 -17
- package/dist/src/core/perSecurity.js +87 -0
- package/dist/src/core/printer.js +126 -11
- package/dist/src/core/quoteSharding.js +109 -19
- package/dist/src/core/rowSink.js +196 -0
- package/dist/src/core/transport.js +51 -0
- package/dist/src/version.js +1 -1
- package/gangtise-openapi/SKILL.md +57 -158
- package/gangtise-openapi/references/commands/ai.md +6 -5
- package/gangtise-openapi/references/commands/fundamental.md +7 -6
- package/gangtise-openapi/references/commands/indicator.md +53 -85
- package/gangtise-openapi/references/commands/insight.md +11 -11
- package/gangtise-openapi/references/commands/quote.md +45 -19
- package/gangtise-openapi/references/commands/reference-and-lookup.md +5 -5
- package/gangtise-openapi/references/commands/tool.md +2 -2
- package/gangtise-openapi/references/commands/vault.md +3 -3
- package/gangtise-openapi/references/errors.md +135 -0
- package/gangtise-openapi/references/examples.md +14 -11
- package/gangtise-openapi/references/fields.md +21 -24
- package/gangtise-openapi/references/lookup-ids.md +3 -3
- package/gangtise-openapi/references/response-schema.md +9 -9
- package/package.json +3 -2
package/dist/src/core/args.js
CHANGED
|
@@ -131,6 +131,7 @@ function epochMillis(value) {
|
|
|
131
131
|
return Number(value) * 1000;
|
|
132
132
|
return undefined;
|
|
133
133
|
}
|
|
134
|
+
const BEIJING_OFFSET = "+08:00";
|
|
134
135
|
export function toTimestamp13(value) {
|
|
135
136
|
if (value === undefined)
|
|
136
137
|
return undefined;
|
|
@@ -140,13 +141,9 @@ export function toTimestamp13(value) {
|
|
|
140
141
|
if (!datetimeFieldsValid(value))
|
|
141
142
|
return undefined;
|
|
142
143
|
const parts = LOCAL_DATETIME.exec(value);
|
|
143
|
-
const [, y, , mo, d, hh = "
|
|
144
|
-
const
|
|
145
|
-
|
|
146
|
-
const faithful = dt.getFullYear() === year && dt.getMonth() === Number(mo) - 1
|
|
147
|
-
&& dt.getDate() === Number(d) && dt.getHours() === Number(hh)
|
|
148
|
-
&& dt.getMinutes() === Number(mi) && dt.getSeconds() === Number(ss);
|
|
149
|
-
return faithful ? dt.getTime() : undefined;
|
|
144
|
+
const [, y, , mo, d, hh = "00", mi = "00", ss = "00"] = parts;
|
|
145
|
+
const instant = new Date(`${y}-${mo}-${d}T${hh}:${mi}:${ss}${BEIJING_OFFSET}`).getTime();
|
|
146
|
+
return Number.isNaN(instant) ? undefined : instant;
|
|
150
147
|
}
|
|
151
148
|
export function parseTimestamp13(value, optionName) {
|
|
152
149
|
if (value === undefined)
|
package/dist/src/core/client.js
CHANGED
|
@@ -4,18 +4,29 @@ import path from "node:path";
|
|
|
4
4
|
import { pipeline } from "node:stream/promises";
|
|
5
5
|
import { FormData, request } from "undici";
|
|
6
6
|
import { isTokenCacheValid, normalizeToken, readTokenCache, requireAccessCredentials, writeTokenCache } from "./auth.js";
|
|
7
|
-
import { ApiError, attachEnvelopeTraceId, ValidationError } from "./errors.js";
|
|
7
|
+
import { ApiError, attachEnvelopeTraceId, markStructural, ValidationError } from "./errors.js";
|
|
8
8
|
import { ENDPOINTS, resolveTimeoutMs } from "./endpoints.js";
|
|
9
9
|
import { getLookupData } from "./lookupData/index.js";
|
|
10
|
-
import {
|
|
10
|
+
import { stagingPath } from "./output.js";
|
|
11
|
+
import { decodeResponseBody, getDispatcher, isVerbose, logTiming, markRetryable, PAGE_CONCURRENCY, parseRetryAfterMs, quoteBigIntFields, runInOrder, withRetry } from "./transport.js";
|
|
12
|
+
import { attachRowSink } from "./rowSink.js";
|
|
11
13
|
const AUTH_RETRY_CODES = new Set(["8000014", "8000015", "0000001008", "999002"]);
|
|
12
14
|
export class GangtiseClient {
|
|
13
15
|
config;
|
|
16
|
+
rowSink;
|
|
14
17
|
refreshPromise = null;
|
|
15
18
|
memoCache = null;
|
|
16
19
|
envTokenInvalidated = false;
|
|
17
|
-
|
|
20
|
+
rowSinkClaimed = false;
|
|
21
|
+
constructor(config, rowSink) {
|
|
18
22
|
this.config = config;
|
|
23
|
+
this.rowSink = rowSink;
|
|
24
|
+
}
|
|
25
|
+
claimRowSink() {
|
|
26
|
+
if (!this.rowSink || this.rowSinkClaimed)
|
|
27
|
+
return undefined;
|
|
28
|
+
this.rowSinkClaimed = true;
|
|
29
|
+
return this.rowSink;
|
|
19
30
|
}
|
|
20
31
|
async getAuthorizationHeader(forceRefresh = false) {
|
|
21
32
|
if (this.config.token && !this.envTokenInvalidated && !forceRefresh) {
|
|
@@ -151,27 +162,42 @@ export class GangtiseClient {
|
|
|
151
162
|
return firstPage;
|
|
152
163
|
}
|
|
153
164
|
const total = firstPage.total;
|
|
154
|
-
const collected = [...firstPage.list];
|
|
155
165
|
const available = Math.max(total - startFrom, 0);
|
|
156
166
|
const target = requestedSize === undefined ? available : Math.min(requestedSize, available);
|
|
167
|
+
const sink = this.claimRowSink();
|
|
168
|
+
if (sink && Array.isArray(firstPage.fieldList))
|
|
169
|
+
sink.setFieldList(firstPage.fieldList);
|
|
170
|
+
const collected = [];
|
|
171
|
+
let count = 0;
|
|
172
|
+
const keep = async (rows) => {
|
|
173
|
+
const kept = requestedSize === undefined ? rows : rows.slice(0, Math.max(0, requestedSize - count));
|
|
174
|
+
if (kept.length === 0)
|
|
175
|
+
return;
|
|
176
|
+
count += kept.length;
|
|
177
|
+
if (sink)
|
|
178
|
+
await sink.push(kept);
|
|
179
|
+
else
|
|
180
|
+
for (const row of kept)
|
|
181
|
+
collected.push(row);
|
|
182
|
+
};
|
|
183
|
+
const result = () => {
|
|
184
|
+
const out = { ...firstPage, total, list: collected };
|
|
185
|
+
return sink ? attachRowSink(out, sink) : out;
|
|
186
|
+
};
|
|
187
|
+
await keep(firstPage.list);
|
|
157
188
|
if (firstPage.list.length < firstPageSize) {
|
|
158
|
-
const out =
|
|
159
|
-
|
|
160
|
-
total
|
|
161
|
-
list: requestedSize === undefined ? collected : collected.slice(0, requestedSize),
|
|
162
|
-
};
|
|
163
|
-
if (collected.length < target) {
|
|
164
|
-
process.stderr.write(`[gangtise] warning: server returned a short page (${collected.length} rows) but reported total=${total}; treating it as the end of data — results may be incomplete\n`);
|
|
189
|
+
const out = result();
|
|
190
|
+
if (count < target) {
|
|
191
|
+
process.stderr.write(`[gangtise] warning: server returned a short page (${count} rows) but reported total=${total}; treating it as the end of data — results may be incomplete\n`);
|
|
165
192
|
out.partial = true;
|
|
166
193
|
}
|
|
167
194
|
return out;
|
|
168
195
|
}
|
|
169
|
-
if (
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
total,
|
|
173
|
-
|
|
174
|
-
};
|
|
196
|
+
if (count >= target) {
|
|
197
|
+
const out = result();
|
|
198
|
+
if (requestedSize === undefined && total > firstPageSize)
|
|
199
|
+
await this.flagIfTotalCapped(endpoint, initialBody, total, out, count);
|
|
200
|
+
return out;
|
|
175
201
|
}
|
|
176
202
|
const MAX_PAGES = 1000;
|
|
177
203
|
let truncatedByPageCap = false;
|
|
@@ -190,10 +216,11 @@ export class GangtiseClient {
|
|
|
190
216
|
}
|
|
191
217
|
let unexpectedShape = false;
|
|
192
218
|
let totalDrift = false;
|
|
219
|
+
let laterPartial = false;
|
|
193
220
|
const failedPages = [];
|
|
194
221
|
let firstError = null;
|
|
195
222
|
let aborted = false;
|
|
196
|
-
|
|
223
|
+
await runInOrder(pageRequests, PAGE_CONCURRENCY, async (req) => {
|
|
197
224
|
if (aborted) {
|
|
198
225
|
failedPages.push(req);
|
|
199
226
|
return [];
|
|
@@ -211,6 +238,8 @@ export class GangtiseClient {
|
|
|
211
238
|
}
|
|
212
239
|
if (page.total !== total)
|
|
213
240
|
totalDrift = true;
|
|
241
|
+
if (page.partial === true)
|
|
242
|
+
laterPartial = true;
|
|
214
243
|
return page.list;
|
|
215
244
|
}
|
|
216
245
|
catch (error) {
|
|
@@ -220,12 +249,7 @@ export class GangtiseClient {
|
|
|
220
249
|
failedPages.push(req);
|
|
221
250
|
return [];
|
|
222
251
|
}
|
|
223
|
-
});
|
|
224
|
-
for (const list of pages) {
|
|
225
|
-
if (list.length === 0)
|
|
226
|
-
continue;
|
|
227
|
-
collected.push(...list);
|
|
228
|
-
}
|
|
252
|
+
}, (list) => keep(list));
|
|
229
253
|
if (unexpectedShape) {
|
|
230
254
|
process.stderr.write(`[gangtise] warning: a page response had unexpected shape; its rows are missing (counted in failedPages)\n`);
|
|
231
255
|
}
|
|
@@ -233,44 +257,45 @@ export class GangtiseClient {
|
|
|
233
257
|
process.stderr.write(`[gangtise] warning: 'total' changed across pages (data shifted during fetch); rows may be duplicated or missing\n`);
|
|
234
258
|
}
|
|
235
259
|
if (truncatedByPageCap) {
|
|
236
|
-
process.stderr.write(`[gangtise] warning: hit the ${MAX_PAGES}-page safety cap; fetched ${
|
|
260
|
+
process.stderr.write(`[gangtise] warning: hit the ${MAX_PAGES}-page safety cap; fetched ${count} of ${total} rows. Narrow the query (e.g. a shorter date range) or pass --size to fetch a bounded subset.\n`);
|
|
237
261
|
}
|
|
238
|
-
const short =
|
|
239
|
-
|
|
262
|
+
const short = count < target;
|
|
263
|
+
const out = result();
|
|
240
264
|
if (requestedSize === undefined && total > 0 && !short && !totalDrift && !truncatedByPageCap && failedPages.length === 0) {
|
|
241
|
-
|
|
242
|
-
const probe = await this.requestJson(endpoint, { ...initialBody, from: total, size: 1 });
|
|
243
|
-
if (this.isPaginatedListResponse(probe) && probe.list.length > 0)
|
|
244
|
-
totalCapped = true;
|
|
245
|
-
}
|
|
246
|
-
catch {
|
|
247
|
-
}
|
|
248
|
-
}
|
|
249
|
-
if (totalCapped) {
|
|
250
|
-
process.stderr.write(`[gangtise] warning: ${endpoint.key} reported total=${total} but rows exist past that offset — 'total' is a server-side cap, not the real count. This export is TRUNCATED at ${collected.length} rows. Narrow the query (date range / filters) and fetch in slices.\n`);
|
|
251
|
-
}
|
|
252
|
-
const out = {
|
|
253
|
-
...firstPage,
|
|
254
|
-
total,
|
|
255
|
-
list: requestedSize === undefined ? collected : collected.slice(0, requestedSize),
|
|
256
|
-
};
|
|
257
|
-
if (totalCapped) {
|
|
258
|
-
out.partial = true;
|
|
259
|
-
out.totalCapped = true;
|
|
265
|
+
await this.flagIfTotalCapped(endpoint, initialBody, total, out, count);
|
|
260
266
|
}
|
|
261
267
|
if (short || totalDrift || failedPages.length > 0)
|
|
262
268
|
out.partial = true;
|
|
269
|
+
if (laterPartial) {
|
|
270
|
+
out.partial = true;
|
|
271
|
+
process.stderr.write(`[gangtise] warning: a later page reported itself partial; the merged result is marked partial\n`);
|
|
272
|
+
}
|
|
263
273
|
if (failedPages.length > 0) {
|
|
264
274
|
out.failedPages = failedPages.map((p) => ({ from: p.from, size: p.size }));
|
|
265
275
|
const detail = firstError instanceof Error ? `: ${firstError.message}` : "";
|
|
266
276
|
const skippedHint = aborted ? " A page hit a non-retryable error (e.g. rate limit); remaining pages were skipped." : "";
|
|
267
|
-
process.stderr.write(`[gangtise] warning: ${failedPages.length}/${pageRequests.length} pages not fetched${detail}; results are partial — got ${
|
|
277
|
+
process.stderr.write(`[gangtise] warning: ${failedPages.length}/${pageRequests.length} pages not fetched${detail}; results are partial — got ${count}/${total} rows (see failedPages).${skippedHint}\n`);
|
|
268
278
|
}
|
|
269
279
|
else if (short && !truncatedByPageCap && !totalDrift) {
|
|
270
|
-
process.stderr.write(`[gangtise] warning: server returned ${
|
|
280
|
+
process.stderr.write(`[gangtise] warning: server returned ${count} of ${total} rows (a later page came back short); results may be incomplete\n`);
|
|
271
281
|
}
|
|
272
282
|
return out;
|
|
273
283
|
}
|
|
284
|
+
async flagIfTotalCapped(endpoint, initialBody, total, out, collectedLength) {
|
|
285
|
+
let totalCapped = false;
|
|
286
|
+
try {
|
|
287
|
+
const probe = await this.requestJson(endpoint, { ...initialBody, from: total, size: 1 });
|
|
288
|
+
if (this.isPaginatedListResponse(probe) && probe.list.length > 0)
|
|
289
|
+
totalCapped = true;
|
|
290
|
+
}
|
|
291
|
+
catch {
|
|
292
|
+
}
|
|
293
|
+
if (!totalCapped)
|
|
294
|
+
return;
|
|
295
|
+
process.stderr.write(`[gangtise] warning: ${endpoint.key} reported total=${total} but rows exist past that offset — 'total' is a server-side cap, not the real count. This export is TRUNCATED at ${collectedLength} rows. Narrow the query (date range / filters) and fetch in slices.\n`);
|
|
296
|
+
out.partial = true;
|
|
297
|
+
out.totalCapped = true;
|
|
298
|
+
}
|
|
274
299
|
async login() {
|
|
275
300
|
const authorization = await this.getAuthorizationHeader();
|
|
276
301
|
const cache = await readTokenCache(this.config.tokenCachePath);
|
|
@@ -340,7 +365,11 @@ export class GangtiseClient {
|
|
|
340
365
|
if (response.statusCode >= 400) {
|
|
341
366
|
this.throwHttpError(parsed, response.statusCode, retryAfterMs);
|
|
342
367
|
}
|
|
343
|
-
|
|
368
|
+
const payload = this.unwrapEnvelope(parsed, response.statusCode, retryAfterMs);
|
|
369
|
+
if (endpoint.expects === "list" && !(payload && typeof payload === "object" && Array.isArray(payload.list))) {
|
|
370
|
+
throw markStructural(new ApiError(`${endpoint.key} returned no list payload (got ${payload === null ? "null" : Array.isArray(payload) ? "an array" : typeof payload}) — the response layout may have changed`, undefined, response.statusCode, parsed));
|
|
371
|
+
}
|
|
372
|
+
return payload;
|
|
344
373
|
}
|
|
345
374
|
catch (error) {
|
|
346
375
|
await this.refreshAuthIfRecoverable(error, useAuth, authState, usedAuthorization);
|
|
@@ -476,7 +505,7 @@ export class GangtiseClient {
|
|
|
476
505
|
}
|
|
477
506
|
if (options?.streamTo) {
|
|
478
507
|
await fs.mkdir(path.dirname(options.streamTo), { recursive: true });
|
|
479
|
-
const partPath =
|
|
508
|
+
const partPath = stagingPath(options.streamTo);
|
|
480
509
|
try {
|
|
481
510
|
await pipeline(response.body, createWriteStream(partPath));
|
|
482
511
|
await fs.rename(partPath, options.streamTo);
|
|
@@ -519,6 +548,6 @@ export class GangtiseClient {
|
|
|
519
548
|
if (endpoint.kind === 'json' && endpoint.pagination?.enabled) {
|
|
520
549
|
return this.requestPaginated(endpoint, body);
|
|
521
550
|
}
|
|
522
|
-
return this.requestJson(endpoint, body);
|
|
551
|
+
return this.requestJson(endpoint, body, endpoint.key !== 'auth.login');
|
|
523
552
|
}
|
|
524
553
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import fs from "node:fs/promises";
|
|
2
|
-
import { extname } from "node:path";
|
|
2
|
+
import { dirname, extname } from "node:path";
|
|
3
3
|
import { DownloadError } from "./errors.js";
|
|
4
|
-
import { saveOutputIfNeeded } from "./output.js";
|
|
4
|
+
import { saveOutputIfNeeded, stagingPath } from "./output.js";
|
|
5
5
|
import { extractTitles, lookupTitleCache, readTitleCache, TITLE_LOOKUP_SIZE, writeTitleCache } from "./titleCache.js";
|
|
6
6
|
function sanitizeFilename(name) {
|
|
7
7
|
return name.replace(/[/\\:*?"<>|\u0000-\u001f]/g, "_");
|
|
@@ -16,18 +16,38 @@ function truncateFilename(name, maxBytes = 200) {
|
|
|
16
16
|
return stem.join("") + ext;
|
|
17
17
|
}
|
|
18
18
|
export async function uniquePath(p) {
|
|
19
|
-
const
|
|
20
|
-
|
|
19
|
+
const claim = async (f) => {
|
|
20
|
+
try {
|
|
21
|
+
const handle = await fs.open(f, "wx");
|
|
22
|
+
await handle.close();
|
|
23
|
+
return true;
|
|
24
|
+
}
|
|
25
|
+
catch (error) {
|
|
26
|
+
if (error.code === "EEXIST")
|
|
27
|
+
return false;
|
|
28
|
+
throw error;
|
|
29
|
+
}
|
|
30
|
+
};
|
|
31
|
+
await fs.mkdir(dirname(p) || ".", { recursive: true });
|
|
32
|
+
if (await claim(p))
|
|
21
33
|
return p;
|
|
22
34
|
const ext = extname(p);
|
|
23
35
|
const stem = p.slice(0, p.length - ext.length);
|
|
24
36
|
for (let i = 1; i <= 99; i++) {
|
|
25
37
|
const candidate = `${stem}-${i}${ext}`;
|
|
26
|
-
if (
|
|
38
|
+
if (await claim(candidate))
|
|
27
39
|
return candidate;
|
|
28
40
|
}
|
|
29
41
|
throw new DownloadError(`Refusing to overwrite: 100 files already share the name "${p}" — pass --output or clean up the directory`);
|
|
30
42
|
}
|
|
43
|
+
export async function releaseClaim(p) {
|
|
44
|
+
try {
|
|
45
|
+
if ((await fs.stat(p)).size === 0)
|
|
46
|
+
await fs.unlink(p);
|
|
47
|
+
}
|
|
48
|
+
catch {
|
|
49
|
+
}
|
|
50
|
+
}
|
|
31
51
|
const MIME_EXT = {
|
|
32
52
|
"application/pdf": ".pdf",
|
|
33
53
|
"application/msword": ".doc",
|
|
@@ -119,7 +139,7 @@ async function downloadUrlTo(url, outputPath) {
|
|
|
119
139
|
return "signed-url";
|
|
120
140
|
}
|
|
121
141
|
};
|
|
122
|
-
const partPath =
|
|
142
|
+
const partPath = stagingPath(outputPath);
|
|
123
143
|
await withRetry(async () => {
|
|
124
144
|
const startedAt = Date.now();
|
|
125
145
|
const requestOptions = {
|
|
@@ -178,13 +198,27 @@ export async function saveDownloadResult(result, fallbackName, output) {
|
|
|
178
198
|
if (file.data instanceof Uint8Array) {
|
|
179
199
|
const autoName = (file.filename ? sanitizeFilename(file.filename) : undefined) ?? (safeFallback + extFromContentType(file.contentType));
|
|
180
200
|
const outputPath = output ?? await uniquePath(truncateFilename(autoName));
|
|
181
|
-
|
|
201
|
+
try {
|
|
202
|
+
await saveOutputIfNeeded(file.data, outputPath);
|
|
203
|
+
}
|
|
204
|
+
catch (error) {
|
|
205
|
+
if (!output)
|
|
206
|
+
await releaseClaim(outputPath);
|
|
207
|
+
throw error;
|
|
208
|
+
}
|
|
182
209
|
process.stdout.write(`${outputPath}\n`);
|
|
183
210
|
return;
|
|
184
211
|
}
|
|
185
212
|
if (typeof file.text === "string") {
|
|
186
213
|
const outputPath = output ?? await uniquePath(truncateFilename(`${safeFallback}.txt`));
|
|
187
|
-
|
|
214
|
+
try {
|
|
215
|
+
await saveOutputIfNeeded(file.text, outputPath);
|
|
216
|
+
}
|
|
217
|
+
catch (error) {
|
|
218
|
+
if (!output)
|
|
219
|
+
await releaseClaim(outputPath);
|
|
220
|
+
throw error;
|
|
221
|
+
}
|
|
188
222
|
process.stdout.write(`${outputPath}\n`);
|
|
189
223
|
return;
|
|
190
224
|
}
|
|
@@ -273,42 +273,49 @@ const ENDPOINT_DEFS = {
|
|
|
273
273
|
path: "/application/open-quote/kline/daily",
|
|
274
274
|
kind: "json",
|
|
275
275
|
description: "Query A-share historical daily kline (SH/SZ/BJ)",
|
|
276
|
+
expects: "list",
|
|
276
277
|
},
|
|
277
278
|
"quote.day-kline-hk": {
|
|
278
279
|
method: "POST",
|
|
279
280
|
path: "/application/open-quote/kline-hk/daily",
|
|
280
281
|
kind: "json",
|
|
281
282
|
description: "Query HK stock historical daily kline (HK)",
|
|
283
|
+
expects: "list",
|
|
282
284
|
},
|
|
283
285
|
"quote.day-kline-us": {
|
|
284
286
|
method: "POST",
|
|
285
287
|
path: "/application/open-quote/kline-us/daily",
|
|
286
288
|
kind: "json",
|
|
287
289
|
description: "Query US stock historical daily kline (NYSE/NASDAQ/AMEX)",
|
|
290
|
+
expects: "list",
|
|
288
291
|
},
|
|
289
292
|
"quote.index-day-kline": {
|
|
290
293
|
method: "POST",
|
|
291
294
|
path: "/application/open-quote/index/kline/daily",
|
|
292
295
|
kind: "json",
|
|
293
296
|
description: "Query SH/SZ/BJ index daily kline",
|
|
297
|
+
expects: "list",
|
|
294
298
|
},
|
|
295
299
|
"quote.minute-kline": {
|
|
296
300
|
method: "POST",
|
|
297
301
|
path: "/application/open-quote/kline/minute",
|
|
298
302
|
kind: "json",
|
|
299
303
|
description: "Query A-share minute kline (SH/SZ/BJ)",
|
|
304
|
+
expects: "list",
|
|
300
305
|
},
|
|
301
306
|
"quote.realtime": {
|
|
302
307
|
method: "POST",
|
|
303
308
|
path: "/application/open-quote/quote/realtime",
|
|
304
309
|
kind: "json",
|
|
305
310
|
description: "Query realtime quote snapshot (A-share / HK / US)",
|
|
311
|
+
expects: "list",
|
|
306
312
|
},
|
|
307
313
|
"quote.fund-flow": {
|
|
308
314
|
method: "POST",
|
|
309
315
|
path: "/application/open-quote/fund-flow/daily",
|
|
310
316
|
kind: "json",
|
|
311
317
|
description: "Query A-share daily fund flow (SH/SZ/BJ; small/medium/large/xlarge orders + main net inflow)",
|
|
318
|
+
expects: "list",
|
|
312
319
|
},
|
|
313
320
|
"fundamental.income-statement": {
|
|
314
321
|
method: "POST",
|
package/dist/src/core/errors.js
CHANGED
|
@@ -17,6 +17,14 @@ export function attachEnvelopeTraceId(payload, traceId) {
|
|
|
17
17
|
}
|
|
18
18
|
return payload;
|
|
19
19
|
}
|
|
20
|
+
const STRUCTURAL = Symbol("gangtise.structural");
|
|
21
|
+
export function markStructural(error) {
|
|
22
|
+
Object.defineProperty(error, STRUCTURAL, { value: true, enumerable: false, configurable: true });
|
|
23
|
+
return error;
|
|
24
|
+
}
|
|
25
|
+
export function isStructuralError(error) {
|
|
26
|
+
return Boolean(error && typeof error === "object" && error[STRUCTURAL] === true);
|
|
27
|
+
}
|
|
20
28
|
const ERROR_HINTS = {
|
|
21
29
|
"999001": "检查 GANGTISE_TOKEN 或 AK/SK 是否已 export。",
|
|
22
30
|
"999002": "有 AK/SK 时 CLI 会自动重新登录重试一次,否则请重新登录。",
|
|
@@ -14,6 +14,37 @@ export function zipFieldRow(fields, row, source) {
|
|
|
14
14
|
return acc;
|
|
15
15
|
}, {});
|
|
16
16
|
}
|
|
17
|
+
export function columnarSchemaValid(fields, list) {
|
|
18
|
+
if (!fields)
|
|
19
|
+
return false;
|
|
20
|
+
if (new Set(fields.map(String)).size !== fields.length)
|
|
21
|
+
return false;
|
|
22
|
+
return list.every((row) => !Array.isArray(row) || row.length === fields.length);
|
|
23
|
+
}
|
|
24
|
+
export function flagMissingFields(data, requested, label) {
|
|
25
|
+
if (!requested?.length || !data || typeof data !== "object" || Array.isArray(data))
|
|
26
|
+
return;
|
|
27
|
+
const rec = data;
|
|
28
|
+
if (!Array.isArray(rec.fieldList))
|
|
29
|
+
return;
|
|
30
|
+
const returned = new Set(rec.fieldList.map(String));
|
|
31
|
+
const missing = requested.filter((field) => !returned.has(field));
|
|
32
|
+
if (missing.length === 0)
|
|
33
|
+
return;
|
|
34
|
+
rec.partial = true;
|
|
35
|
+
rec.missingFields = missing;
|
|
36
|
+
process.stderr.write(`[gangtise] warning: ${label} returned no column for ${missing.join(", ")} — the server drops a field name it does not recognise (or no longer serves) without an error. Check the name against references/fields.md; result marked partial (exit 3).\n`);
|
|
37
|
+
}
|
|
38
|
+
export function assertColumnarHeader(fields, source) {
|
|
39
|
+
if (!fields) {
|
|
40
|
+
throw new ValidationError(`响应包含数组形式的行但没有 fieldList,无法确定各列的含义,已拒绝输出。这是上游响应结构异常,请报障${traceSuffix(source)}。`);
|
|
41
|
+
}
|
|
42
|
+
const names = fields.map(String);
|
|
43
|
+
const dupes = [...new Set(names.filter((name, i) => names.indexOf(name) !== i))];
|
|
44
|
+
if (dupes.length > 0) {
|
|
45
|
+
throw new ValidationError(`响应 fieldList 有重复列名(${dupes.join(", ")})——按位置拍平时后一列会覆盖前一列,已拒绝输出。这是上游响应结构异常,请报障${traceSuffix(source)}。`);
|
|
46
|
+
}
|
|
47
|
+
}
|
|
17
48
|
export function normalizeRows(value) {
|
|
18
49
|
if (!value || typeof value !== "object") {
|
|
19
50
|
return value;
|
|
@@ -24,12 +55,16 @@ export function normalizeRows(value) {
|
|
|
24
55
|
const record = value;
|
|
25
56
|
if (Array.isArray(record.fieldList) && Array.isArray(record.list)) {
|
|
26
57
|
const fields = record.fieldList;
|
|
58
|
+
if (record.list.some(Array.isArray))
|
|
59
|
+
assertColumnarHeader(fields, record);
|
|
27
60
|
const normalizedList = record.list.map((row) => (Array.isArray(row) ? zipFieldRow(fields, row, record) : row));
|
|
28
61
|
const { fieldList, list, ...meta } = record;
|
|
29
62
|
const hasMeta = Object.keys(meta).length > 0;
|
|
30
63
|
return hasMeta ? { ...meta, list: normalizedList } : normalizedList;
|
|
31
64
|
}
|
|
32
65
|
if (Array.isArray(record.list)) {
|
|
66
|
+
if (record.list.some(Array.isArray))
|
|
67
|
+
assertColumnarHeader(undefined, record);
|
|
33
68
|
const { list, ...meta } = record;
|
|
34
69
|
const hasMeta = Object.keys(meta).length > 0;
|
|
35
70
|
return hasMeta ? { ...meta, list } : list;
|
package/dist/src/core/output.js
CHANGED
|
@@ -1,6 +1,26 @@
|
|
|
1
|
+
import { createHash, randomBytes } from "node:crypto";
|
|
1
2
|
import fs from "node:fs/promises";
|
|
2
3
|
import { ConfigError } from "./errors.js";
|
|
3
4
|
const OUTPUT_FORMATS = ["table", "json", "jsonl", "csv", "markdown"];
|
|
5
|
+
export function stagingPath(target, suffix = "part") {
|
|
6
|
+
return `${target}.${randomBytes(4).toString("hex")}.${suffix}`;
|
|
7
|
+
}
|
|
8
|
+
export function digestBuffer(content) {
|
|
9
|
+
const buf = typeof content === "string" ? Buffer.from(content, "utf8") : content;
|
|
10
|
+
return { bytes: buf.byteLength, sha256: createHash("sha256").update(buf).digest("hex") };
|
|
11
|
+
}
|
|
12
|
+
export async function digestFile(filePath) {
|
|
13
|
+
const { createReadStream } = await import("node:fs");
|
|
14
|
+
const hash = createHash("sha256");
|
|
15
|
+
let bytes = 0;
|
|
16
|
+
const stream = createReadStream(filePath);
|
|
17
|
+
for await (const chunk of stream) {
|
|
18
|
+
const buf = chunk;
|
|
19
|
+
bytes += buf.byteLength;
|
|
20
|
+
hash.update(buf);
|
|
21
|
+
}
|
|
22
|
+
return { bytes, sha256: hash.digest("hex") };
|
|
23
|
+
}
|
|
4
24
|
export function parseOutputFormat(value) {
|
|
5
25
|
const format = value ?? "table";
|
|
6
26
|
if (OUTPUT_FORMATS.includes(format)) {
|
|
@@ -24,7 +44,7 @@ function displayWidth(value) {
|
|
|
24
44
|
}
|
|
25
45
|
return width;
|
|
26
46
|
}
|
|
27
|
-
function formatScalar(value) {
|
|
47
|
+
export function formatScalar(value) {
|
|
28
48
|
if (value === null || value === undefined) {
|
|
29
49
|
return "";
|
|
30
50
|
}
|
|
@@ -104,16 +124,19 @@ function renderCsv(rows) {
|
|
|
104
124
|
const body = rows.map((row) => columns.map((column) => csvEscape(formatScalar(row[column]))).join(","));
|
|
105
125
|
return [header, ...body].join("\n");
|
|
106
126
|
}
|
|
127
|
+
export function jsonlItems(value) {
|
|
128
|
+
const list = value && typeof value === "object" && !Array.isArray(value) ? value.list : undefined;
|
|
129
|
+
return Array.isArray(list) ? list : toRows(value);
|
|
130
|
+
}
|
|
131
|
+
export function countOutputRows(value, format) {
|
|
132
|
+
return format === "jsonl" ? jsonlItems(value).length : toRows(value).length;
|
|
133
|
+
}
|
|
107
134
|
export function renderOutput(value, format) {
|
|
108
135
|
switch (format) {
|
|
109
136
|
case "json":
|
|
110
137
|
return JSON.stringify(value, null, 2);
|
|
111
|
-
case "jsonl":
|
|
112
|
-
|
|
113
|
-
? value.list
|
|
114
|
-
: null;
|
|
115
|
-
return (items ?? toRows(value)).map((item) => JSON.stringify(item)).join("\n");
|
|
116
|
-
}
|
|
138
|
+
case "jsonl":
|
|
139
|
+
return jsonlItems(value).map((item) => JSON.stringify(item)).join("\n");
|
|
117
140
|
case "csv":
|
|
118
141
|
return renderCsv(toRows(value));
|
|
119
142
|
case "markdown":
|
|
@@ -125,29 +148,29 @@ export function renderOutput(value, format) {
|
|
|
125
148
|
}
|
|
126
149
|
export async function streamOutputToFile(value, format, outputPath) {
|
|
127
150
|
if (format !== "jsonl" && format !== "csv")
|
|
128
|
-
return
|
|
151
|
+
return null;
|
|
129
152
|
const list = pickList(value);
|
|
130
153
|
if (!list)
|
|
131
|
-
return
|
|
154
|
+
return null;
|
|
132
155
|
if (list.length < 1000)
|
|
133
|
-
return
|
|
156
|
+
return null;
|
|
134
157
|
let csvRows = [];
|
|
135
158
|
let csvColumns = [];
|
|
136
159
|
if (format === "csv") {
|
|
137
160
|
csvRows = list.filter((row) => Boolean(row && typeof row === "object" && !Array.isArray(row)));
|
|
138
161
|
if (csvRows.length === 0)
|
|
139
|
-
return
|
|
162
|
+
return null;
|
|
140
163
|
csvColumns = Array.from(new Set(csvRows.flatMap((row) => Object.keys(row))));
|
|
141
164
|
}
|
|
142
165
|
const { dirname } = await import("node:path");
|
|
143
166
|
const { createWriteStream } = await import("node:fs");
|
|
144
167
|
await fs.mkdir(dirname(outputPath), { recursive: true });
|
|
145
|
-
const partPath =
|
|
168
|
+
const partPath = stagingPath(outputPath);
|
|
146
169
|
const stream = createWriteStream(partPath, { encoding: "utf8" });
|
|
147
170
|
stream.on("error", () => { });
|
|
148
171
|
try {
|
|
149
172
|
if (format === "jsonl") {
|
|
150
|
-
for (const item of
|
|
173
|
+
for (const item of jsonlItems(value)) {
|
|
151
174
|
await writeLine(stream, JSON.stringify(item));
|
|
152
175
|
}
|
|
153
176
|
}
|
|
@@ -161,7 +184,9 @@ export async function streamOutputToFile(value, format, outputPath) {
|
|
|
161
184
|
await new Promise((resolve, reject) => {
|
|
162
185
|
stream.end((err) => err ? reject(err) : resolve());
|
|
163
186
|
});
|
|
187
|
+
const digest = await digestFile(partPath);
|
|
164
188
|
await fs.rename(partPath, outputPath);
|
|
189
|
+
return digest;
|
|
165
190
|
}
|
|
166
191
|
catch (error) {
|
|
167
192
|
stream.destroy();
|
|
@@ -171,7 +196,6 @@ export async function streamOutputToFile(value, format, outputPath) {
|
|
|
171
196
|
await fs.unlink(partPath).catch(() => { });
|
|
172
197
|
throw error;
|
|
173
198
|
}
|
|
174
|
-
return true;
|
|
175
199
|
}
|
|
176
200
|
export function pickList(value) {
|
|
177
201
|
if (Array.isArray(value))
|
|
@@ -183,7 +207,7 @@ export function pickList(value) {
|
|
|
183
207
|
}
|
|
184
208
|
return null;
|
|
185
209
|
}
|
|
186
|
-
function csvEscape(value) {
|
|
210
|
+
export function csvEscape(value) {
|
|
187
211
|
let out = value;
|
|
188
212
|
if (/^[=@\t\r]/.test(out) || (/^[+\-]/.test(out) && !Number.isFinite(Number(out))))
|
|
189
213
|
out = "'" + out;
|
|
@@ -191,7 +215,7 @@ function csvEscape(value) {
|
|
|
191
215
|
return `"${out.replaceAll("\"", "\"\"")}"`;
|
|
192
216
|
return out;
|
|
193
217
|
}
|
|
194
|
-
function writeLine(stream, line) {
|
|
218
|
+
export function writeLine(stream, line) {
|
|
195
219
|
return new Promise((resolve, reject) => {
|
|
196
220
|
const ok = stream.write(line + "\n", (err) => err ? reject(err) : undefined);
|
|
197
221
|
if (ok) {
|
|
@@ -210,7 +234,7 @@ export async function saveOutputIfNeeded(content, outputPath) {
|
|
|
210
234
|
}
|
|
211
235
|
const { dirname } = await import("node:path");
|
|
212
236
|
await fs.mkdir(dirname(outputPath), { recursive: true });
|
|
213
|
-
const partPath =
|
|
237
|
+
const partPath = stagingPath(outputPath);
|
|
214
238
|
try {
|
|
215
239
|
if (typeof content === "string") {
|
|
216
240
|
await fs.writeFile(partPath, content, "utf8");
|
|
@@ -218,7 +242,9 @@ export async function saveOutputIfNeeded(content, outputPath) {
|
|
|
218
242
|
else {
|
|
219
243
|
await fs.writeFile(partPath, content);
|
|
220
244
|
}
|
|
245
|
+
const digest = digestBuffer(content);
|
|
221
246
|
await fs.rename(partPath, outputPath);
|
|
247
|
+
return digest;
|
|
222
248
|
}
|
|
223
249
|
catch (error) {
|
|
224
250
|
await fs.unlink(partPath).catch(() => { });
|