gangtise-openapi-cli 0.34.1 → 0.36.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +41 -12
- package/dist/src/cli.js +4 -268
- package/dist/src/core/args.js +45 -149
- package/dist/src/core/asyncContent.js +0 -22
- package/dist/src/core/auth.js +0 -14
- package/dist/src/core/client.js +0 -151
- package/dist/src/core/commandBodies.js +10 -38
- package/dist/src/core/download.js +0 -39
- package/dist/src/core/endpoints.js +0 -31
- package/dist/src/core/errors.js +53 -86
- package/dist/src/core/fileParse.js +0 -16
- package/dist/src/core/indicatorMatrix.js +0 -207
- package/dist/src/core/normalize.js +0 -14
- package/dist/src/core/output.js +0 -56
- package/dist/src/core/printer.js +0 -19
- package/dist/src/core/quoteSharding.js +0 -58
- package/dist/src/core/titleCache.js +0 -71
- package/dist/src/core/transport.js +0 -45
- package/dist/src/version.js +1 -2
- package/gangtise-openapi/SKILL.md +10 -7
- package/gangtise-openapi/references/commands/ai.md +3 -1
- package/gangtise-openapi/references/commands/indicator.md +23 -3
- package/gangtise-openapi/references/commands/insight.md +26 -5
- package/gangtise-openapi/references/examples.md +2 -1
- package/package.json +1 -1
package/dist/src/core/client.js
CHANGED
|
@@ -8,22 +8,11 @@ import { ApiError, attachEnvelopeTraceId, ValidationError } from "./errors.js";
|
|
|
8
8
|
import { ENDPOINTS, resolveTimeoutMs } from "./endpoints.js";
|
|
9
9
|
import { getLookupData } from "./lookupData/index.js";
|
|
10
10
|
import { decodeResponseBody, getDispatcher, isVerbose, logTiming, markRetryable, PAGE_CONCURRENCY, parseRetryAfterMs, quoteBigIntFields, runWithConcurrency, withRetry } from "./transport.js";
|
|
11
|
-
// Auth errors that warrant a forced re-login + one replay: the token was rejected
|
|
12
|
-
// server-side while still looking valid by local expiry, so only a forced refresh
|
|
13
|
-
// recovers it. 0000001008 is the legacy code (probed 2026-07-20: still what the
|
|
14
|
-
// token filter emits); 999002 TOKEN_INVALID is its 2026-07-17 replacement, listed
|
|
15
|
-
// ahead of the rollout so self-heal does not silently die when the filter switches.
|
|
16
|
-
// 8000014/8000015 are the retired AK/SK codes, kept for older server builds.
|
|
17
|
-
// 999011 CREDENTIAL_INVALID is not here and could not act if it were — it comes from
|
|
18
|
-
// auth.login, which runs useAuth=false and so never reaches this check. Its "never
|
|
19
|
-
// replay" guarantee lives in transport's TERMINAL_API_CODES instead.
|
|
20
11
|
const AUTH_RETRY_CODES = new Set(["8000014", "8000015", "0000001008", "999002"]);
|
|
21
12
|
export class GangtiseClient {
|
|
22
13
|
config;
|
|
23
14
|
refreshPromise = null;
|
|
24
15
|
memoCache = null;
|
|
25
|
-
// After an injected env token (GANGTISE_TOKEN) is rejected and we self-heal via
|
|
26
|
-
// login, stop preferring that now-stale token so the retry uses the fresh one.
|
|
27
16
|
envTokenInvalidated = false;
|
|
28
17
|
constructor(config) {
|
|
29
18
|
this.config = config;
|
|
@@ -53,15 +42,10 @@ export class GangtiseClient {
|
|
|
53
42
|
accessKey: credentials.accessKey,
|
|
54
43
|
secretKey: credentials.secretKey,
|
|
55
44
|
}, false);
|
|
56
|
-
// Validate the shape before touching it: a missing accessToken used to surface
|
|
57
|
-
// as a bare TypeError from normalizeToken, hiding the real cause.
|
|
58
45
|
if (typeof envelope?.accessToken !== "string" || !envelope.accessToken) {
|
|
59
46
|
throw new ApiError("Login succeeded but the response carried no accessToken", undefined, undefined, envelope);
|
|
60
47
|
}
|
|
61
48
|
const accessToken = normalizeToken(envelope.accessToken);
|
|
62
|
-
// A non-numeric expiresIn would make expiresAt NaN — the cache would never
|
|
63
|
-
// validate and every command would silently re-login. Degrade to 0 instead
|
|
64
|
-
// (token works now, next process logs in again).
|
|
65
49
|
const expiresIn = Number.isFinite(envelope.expiresIn) ? envelope.expiresIn : 0;
|
|
66
50
|
const expiresAt = Math.floor(Date.now() / 1000) + expiresIn;
|
|
67
51
|
const cache = { ...envelope, accessToken, expiresIn, expiresAt };
|
|
@@ -70,20 +54,11 @@ export class GangtiseClient {
|
|
|
70
54
|
await writeTokenCache(this.config.tokenCachePath, cache);
|
|
71
55
|
}
|
|
72
56
|
catch (error) {
|
|
73
|
-
// A read-only HOME or full disk must not fail the request — the in-memory
|
|
74
|
-
// token is valid; we just can't persist it for the next process.
|
|
75
57
|
const msg = error instanceof Error ? error.message : String(error);
|
|
76
58
|
process.stderr.write(`[gangtise] warning: could not persist token cache: ${msg}\n`);
|
|
77
59
|
}
|
|
78
60
|
return accessToken;
|
|
79
61
|
}
|
|
80
|
-
/**
|
|
81
|
-
* On a recoverable auth error (expired/invalid token codes), force a one-time
|
|
82
|
-
* token refresh and re-throw as retryable so withRetry replays the request.
|
|
83
|
-
* Otherwise — or once we've already retried this request — it's a no-op and
|
|
84
|
-
* the caller re-throws the original error. `authState` persists across the
|
|
85
|
-
* withRetry attempts so we only refresh once per logical request.
|
|
86
|
-
*/
|
|
87
62
|
async refreshAuthIfRecoverable(error, useAuth, authState, usedAuthorization) {
|
|
88
63
|
if (useAuth
|
|
89
64
|
&& !authState.retried
|
|
@@ -94,13 +69,6 @@ export class GangtiseClient {
|
|
|
94
69
|
&& this.config.secretKey) {
|
|
95
70
|
authState.retried = true;
|
|
96
71
|
this.envTokenInvalidated = true;
|
|
97
|
-
// If the failed request was still carrying an OLDER token than the one now in
|
|
98
|
-
// memoCache, another request already refreshed — replay with the fresh token
|
|
99
|
-
// instead of logging in again (back-to-back logins can kick each other's
|
|
100
|
-
// sessions server-side, the 0000001008 semantics). If the failed request used
|
|
101
|
-
// the CURRENT token, that token is genuinely dead: force a new login. A time
|
|
102
|
-
// window is NOT a valid proxy here — right after the initial login the window
|
|
103
|
-
// is always "recent", which would skip the refresh exactly when it's needed.
|
|
104
72
|
const memoToken = this.memoCache && isTokenCacheValid(this.memoCache) ? normalizeToken(this.memoCache.accessToken) : null;
|
|
105
73
|
const alreadyRefreshed = memoToken !== null && usedAuthorization !== undefined && usedAuthorization !== memoToken;
|
|
106
74
|
if (!alreadyRefreshed) {
|
|
@@ -110,9 +78,6 @@ export class GangtiseClient {
|
|
|
110
78
|
throw markRetryable(new ApiError(error.message, error.code, error.statusCode, error.details));
|
|
111
79
|
}
|
|
112
80
|
}
|
|
113
|
-
/** `new URL("/a/b", "https://proxy/prefix")` drops "/prefix" — an absolute path
|
|
114
|
-
* replaces the base's path per the URL spec. Join manually so a reverse-proxy
|
|
115
|
-
* GANGTISE_BASE_URL with a path prefix keeps working. */
|
|
116
81
|
buildUrl(path) {
|
|
117
82
|
const base = this.config.baseUrl.endsWith("/") ? this.config.baseUrl : `${this.config.baseUrl}/`;
|
|
118
83
|
return new URL(path.replace(/^\//, ""), base);
|
|
@@ -142,9 +107,6 @@ export class GangtiseClient {
|
|
|
142
107
|
throw new ApiError(parsed.msg || "API request failed", code, statusCode, parsed, retryAfterMs);
|
|
143
108
|
}
|
|
144
109
|
if ('data' in parsed) {
|
|
145
|
-
// Carry the envelope's traceId onto the payload: the EDE endpoints wrap a
|
|
146
|
-
// second envelope inside `data` and raise their own failures from it, by
|
|
147
|
-
// which point this is the only traceId in reach.
|
|
148
110
|
return attachEnvelopeTraceId(parsed.data, parsed.traceId);
|
|
149
111
|
}
|
|
150
112
|
return parsed;
|
|
@@ -177,7 +139,6 @@ export class GangtiseClient {
|
|
|
177
139
|
const startFrom = typeof initialBody.from === 'number' && Number.isFinite(initialBody.from) ? initialBody.from : 0;
|
|
178
140
|
const requestedSize = typeof initialBody.size === 'number' && Number.isFinite(initialBody.size) ? initialBody.size : undefined;
|
|
179
141
|
const maxPageSize = endpoint.pagination?.maxPageSize ?? requestedSize ?? 20;
|
|
180
|
-
// First page: serial — we need total before deciding how many more requests to fan out.
|
|
181
142
|
const firstPageSize = requestedSize === undefined ? maxPageSize : Math.min(maxPageSize, requestedSize);
|
|
182
143
|
const firstPage = await this.requestJson(endpoint, {
|
|
183
144
|
...initialBody,
|
|
@@ -185,26 +146,7 @@ export class GangtiseClient {
|
|
|
185
146
|
size: firstPageSize,
|
|
186
147
|
});
|
|
187
148
|
if (!this.isPaginatedListResponse(firstPage)) {
|
|
188
|
-
// Shape drift (e.g. total arriving as a string) silently degrades fetch-all
|
|
189
|
-
// to a single page with no partial marker. This is NOT hypothetical: passing
|
|
190
|
-
// industryList to insight foreign-opinion / independent-opinion makes the
|
|
191
|
-
// server answer 200 + data:null, which then renders as one phantom row
|
|
192
|
-
// ({"value":null} in jsonl, a blank row in table/markdown). Verbose-only was
|
|
193
|
-
// the wrong bar — a caller who never sets GANGTISE_VERBOSE reads that row as
|
|
194
|
-
// real data, so warn on stderr unconditionally.
|
|
195
149
|
process.stderr.write(`[gangtise] warning: ${endpoint.key} is marked paginated but the first page has an unexpected shape (no numeric total + list); returning it as-is\n`);
|
|
196
|
-
// A warning only helps a human. Scripts read the exit code, and without one they
|
|
197
|
-
// cannot tell "this filter legitimately matched nothing" from "this filter is
|
|
198
|
-
// broken". Reuse exit 3 (result is not what was asked for) rather than inventing a
|
|
199
|
-
// fourth code — fetch-all silently degraded to one page either way.
|
|
200
|
-
//
|
|
201
|
-
// Applies to EVERY malformed shape, not just `null`: a string `total` truncates the
|
|
202
|
-
// result to page 1, which looks complete and is therefore worse than an obviously
|
|
203
|
-
// empty payload. All 24 paginated endpoints are genuine {total, list} lists (the
|
|
204
|
-
// odd-shaped ones like reference.constant-list are not marked paginated), so there
|
|
205
|
-
// is no legitimate response that lands here. Endpoints where `null` IS a valid
|
|
206
|
-
// answer (ai.one-pager for a security with no generated content) are unpaginated
|
|
207
|
-
// and never reach this branch.
|
|
208
150
|
process.exitCode = 3;
|
|
209
151
|
return firstPage;
|
|
210
152
|
}
|
|
@@ -212,9 +154,6 @@ export class GangtiseClient {
|
|
|
212
154
|
const collected = [...firstPage.list];
|
|
213
155
|
const available = Math.max(total - startFrom, 0);
|
|
214
156
|
const target = requestedSize === undefined ? available : Math.min(requestedSize, available);
|
|
215
|
-
// Last page reached on first request. If `total` promises more rows than the
|
|
216
|
-
// short page delivered, the server's page cap may be lower than our configured
|
|
217
|
-
// maxPageSize — say so instead of silently returning a subset as "everything".
|
|
218
157
|
if (firstPage.list.length < firstPageSize) {
|
|
219
158
|
const out = {
|
|
220
159
|
...firstPage,
|
|
@@ -223,8 +162,6 @@ export class GangtiseClient {
|
|
|
223
162
|
};
|
|
224
163
|
if (collected.length < target) {
|
|
225
164
|
process.stderr.write(`[gangtise] warning: server returned a short page (${collected.length} rows) but reported total=${total}; treating it as the end of data — results may be incomplete\n`);
|
|
226
|
-
// Machine-readable counterpart of the warning: scripts key off partial /
|
|
227
|
-
// exit code 3, and must not mistake a truncated result for a complete one.
|
|
228
165
|
out.partial = true;
|
|
229
166
|
}
|
|
230
167
|
return out;
|
|
@@ -236,9 +173,6 @@ export class GangtiseClient {
|
|
|
236
173
|
list: requestedSize === undefined ? collected : collected.slice(0, requestedSize),
|
|
237
174
|
};
|
|
238
175
|
}
|
|
239
|
-
// Build remaining page requests. The cap lives inside the loop: a corrupt
|
|
240
|
-
// server `total` (e.g. 9e15) must not materialize millions of page objects —
|
|
241
|
-
// or spin for minutes — before a post-hoc truncation applies.
|
|
242
176
|
const MAX_PAGES = 1000;
|
|
243
177
|
let truncatedByPageCap = false;
|
|
244
178
|
const pageRequests = [];
|
|
@@ -256,11 +190,6 @@ export class GangtiseClient {
|
|
|
256
190
|
}
|
|
257
191
|
let unexpectedShape = false;
|
|
258
192
|
let totalDrift = false;
|
|
259
|
-
// Fail-soft fan-out: a hard page failure (rate-limit 903301, no-perm, retries
|
|
260
|
-
// exhausted) must NOT discard the pages already fetched. Catch per page, record
|
|
261
|
-
// it, and stop starting new requests so we don't keep burning quota into a rate
|
|
262
|
-
// limit. Mirrors quoteSharding's partial-result tolerance — but firstPage already
|
|
263
|
-
// succeeded to get here, so unlike sharding there's no total-failure case.
|
|
264
193
|
const failedPages = [];
|
|
265
194
|
let firstError = null;
|
|
266
195
|
let aborted = false;
|
|
@@ -276,8 +205,6 @@ export class GangtiseClient {
|
|
|
276
205
|
size: req.size,
|
|
277
206
|
});
|
|
278
207
|
if (!this.isPaginatedListResponse(page)) {
|
|
279
|
-
// Treat a shape-broken page like a failed page: its rows are missing, so
|
|
280
|
-
// the result must carry the partial marker instead of looking complete.
|
|
281
208
|
unexpectedShape = true;
|
|
282
209
|
failedPages.push(req);
|
|
283
210
|
return [];
|
|
@@ -305,24 +232,10 @@ export class GangtiseClient {
|
|
|
305
232
|
if (totalDrift) {
|
|
306
233
|
process.stderr.write(`[gangtise] warning: 'total' changed across pages (data shifted during fetch); rows may be duplicated or missing\n`);
|
|
307
234
|
}
|
|
308
|
-
// Always surface a cap-induced truncation (not gated on verbose): the user
|
|
309
|
-
// asked for everything and is silently getting a subset, mirroring the
|
|
310
|
-
// partial-result warning in quoteSharding.
|
|
311
235
|
if (truncatedByPageCap) {
|
|
312
236
|
process.stderr.write(`[gangtise] warning: hit the ${MAX_PAGES}-page safety cap; fetched ${collected.length} of ${total} rows. Narrow the query (e.g. a shorter date range) or pass --size to fetch a bounded subset.\n`);
|
|
313
237
|
}
|
|
314
238
|
const short = collected.length < target;
|
|
315
|
-
// `total` is not always the real row count. Three opinion endpoints report a fixed
|
|
316
|
-
// 10000 while `from=30000` still returns real rows with monotonically older publish
|
|
317
|
-
// times (the shape of Elasticsearch's default track_total_hits). A fetch-all then
|
|
318
|
-
// stops exactly at the cap with collected === total, so every completeness check
|
|
319
|
-
// below passes and the truncated export looks complete — the worst failure mode we
|
|
320
|
-
// have, because `opinion` bills 30 credits per row.
|
|
321
|
-
//
|
|
322
|
-
// Probe one row past the claimed end rather than hardcoding any number: the server
|
|
323
|
-
// can change the cap, and evidence survives that where a constant would not. Only on
|
|
324
|
-
// a genuine fetch-all that otherwise looked complete — when `total` is honest the
|
|
325
|
-
// probe comes back empty (and bills nothing, since these endpoints charge per row).
|
|
326
239
|
let totalCapped = false;
|
|
327
240
|
if (requestedSize === undefined && total > 0 && !short && !totalDrift && !truncatedByPageCap && failedPages.length === 0) {
|
|
328
241
|
try {
|
|
@@ -331,8 +244,6 @@ export class GangtiseClient {
|
|
|
331
244
|
totalCapped = true;
|
|
332
245
|
}
|
|
333
246
|
catch {
|
|
334
|
-
// A failed probe must never fail an otherwise-complete export: we simply do not
|
|
335
|
-
// learn whether the total was capped, which is the pre-existing behaviour.
|
|
336
247
|
}
|
|
337
248
|
}
|
|
338
249
|
if (totalCapped) {
|
|
@@ -347,15 +258,6 @@ export class GangtiseClient {
|
|
|
347
258
|
out.partial = true;
|
|
348
259
|
out.totalCapped = true;
|
|
349
260
|
}
|
|
350
|
-
// Unified completeness backstop. Whatever the cause — a failed/shape-broken page,
|
|
351
|
-
// a short later page (server page cap < maxPageSize), the MAX_PAGES cap, or `total`
|
|
352
|
-
// drifting mid-fetch — the result is partial. A short row count or a total drift each
|
|
353
|
-
// force it; so does any failedPages entry on its own — an over-returning sibling page
|
|
354
|
-
// can lift the row count back to target and mask the hole (short would read false), yet
|
|
355
|
-
// the failedPages branch below still writes "results are partial" to stderr, so the flag
|
|
356
|
-
// must agree. printData maps partial → exit 3 so a script can't read a truncated export
|
|
357
|
-
// as complete. The cap and drift branches above already warned on stderr; failedPages
|
|
358
|
-
// warns below.
|
|
359
261
|
if (short || totalDrift || failedPages.length > 0)
|
|
360
262
|
out.partial = true;
|
|
361
263
|
if (failedPages.length > 0) {
|
|
@@ -365,9 +267,6 @@ export class GangtiseClient {
|
|
|
365
267
|
process.stderr.write(`[gangtise] warning: ${failedPages.length}/${pageRequests.length} pages not fetched${detail}; results are partial — got ${collected.length}/${total} rows (see failedPages).${skippedHint}\n`);
|
|
366
268
|
}
|
|
367
269
|
else if (short && !truncatedByPageCap && !totalDrift) {
|
|
368
|
-
// A short later page with no failure, cap, or drift to explain it: the server
|
|
369
|
-
// simply delivered fewer rows than `total` promised. Warn so an interactive run
|
|
370
|
-
// sees why the result is partial (the other causes each warn on their own path).
|
|
371
270
|
process.stderr.write(`[gangtise] warning: server returned ${collected.length} of ${total} rows (a later page came back short); results may be incomplete\n`);
|
|
372
271
|
}
|
|
373
272
|
return out;
|
|
@@ -389,19 +288,12 @@ export class GangtiseClient {
|
|
|
389
288
|
const authState = { retried: false };
|
|
390
289
|
const timeoutMs = resolveTimeoutMs(this.config.timeoutMs, endpoint);
|
|
391
290
|
return withRetry(async () => {
|
|
392
|
-
// An upload endpoint hands undici the FormData itself: undici derives the
|
|
393
|
-
// multipart content-type (boundary included) from the body, so setting one
|
|
394
|
-
// here would corrupt the request.
|
|
395
291
|
const isUpload = endpoint.kind === 'upload';
|
|
396
292
|
const headers = {
|
|
397
|
-
// undici does not auto-decompress; decodeResponseBody gunzips below. Server
|
|
398
|
-
// gzip cuts JSON payloads ~3-10x (measured 3.6x on constant-list).
|
|
399
293
|
'accept-encoding': 'gzip',
|
|
400
294
|
};
|
|
401
295
|
if (!isUpload)
|
|
402
296
|
headers['content-type'] = 'application/json';
|
|
403
|
-
// Keep the header we actually sent: the self-heal check compares it against
|
|
404
|
-
// the current memoCache token to tell "stale token" from "fresh token died".
|
|
405
297
|
let usedAuthorization;
|
|
406
298
|
if (useAuth) {
|
|
407
299
|
usedAuthorization = await this.getAuthorizationHeader();
|
|
@@ -416,14 +308,10 @@ export class GangtiseClient {
|
|
|
416
308
|
bodyTimeout: timeoutMs,
|
|
417
309
|
dispatcher,
|
|
418
310
|
});
|
|
419
|
-
// Only buffer + gunzip when the server actually compressed; an unencoded
|
|
420
|
-
// response reads as text directly (and keeps existing behavior on that path).
|
|
421
311
|
const encoding = response.headers['content-encoding'];
|
|
422
312
|
const gzipped = (Array.isArray(encoding) ? encoding[0] : encoding)?.toLowerCase().trim() === 'gzip';
|
|
423
313
|
let text;
|
|
424
314
|
if (gzipped) {
|
|
425
|
-
// A proxy/middlebox can declare gzip and deliver garbage — surface that as
|
|
426
|
-
// an ApiError with request context instead of a bare zlib Z_DATA_ERROR.
|
|
427
315
|
const bytes = new Uint8Array(await response.body.arrayBuffer());
|
|
428
316
|
try {
|
|
429
317
|
text = decodeResponseBody(bytes, encoding);
|
|
@@ -437,9 +325,6 @@ export class GangtiseClient {
|
|
|
437
325
|
text = await response.body.text();
|
|
438
326
|
}
|
|
439
327
|
logTiming(`${endpoint.method} ${endpoint.path}`, Date.now() - startedAt, `${response.statusCode}, ${text.length}B`);
|
|
440
|
-
// Parse Retry-After once so every error path below (JSON parse failure AND the
|
|
441
|
-
// envelope/HTTP-error throw) carries it — a non-JSON 429/503 must still honor
|
|
442
|
-
// the server's rate window instead of falling back to default backoff.
|
|
443
328
|
const retryAfterMs = parseRetryAfterMs(response.headers['retry-after'], Date.now());
|
|
444
329
|
let parsed;
|
|
445
330
|
try {
|
|
@@ -452,8 +337,6 @@ export class GangtiseClient {
|
|
|
452
337
|
throw new ApiError(message, undefined, response.statusCode, text.slice(0, 500), retryAfterMs);
|
|
453
338
|
}
|
|
454
339
|
try {
|
|
455
|
-
// Auth errors can arrive as HTTP 4xx or as a 200-wrapped error envelope;
|
|
456
|
-
// both routes must reach the self-heal check below.
|
|
457
340
|
if (response.statusCode >= 400) {
|
|
458
341
|
this.throwHttpError(parsed, response.statusCode, retryAfterMs);
|
|
459
342
|
}
|
|
@@ -472,15 +355,6 @@ export class GangtiseClient {
|
|
|
472
355
|
process.stderr.write(`[gangtise] retry ${attempt} after ${delay.toFixed(0)}ms: ${msg.slice(0, 120)}\n`);
|
|
473
356
|
},
|
|
474
357
|
}).catch((error) => {
|
|
475
|
-
// Until 2026-08-01 EDE answered a no-data query with 999999, so this hint
|
|
476
|
-
// led with "多为查询无数据". No-data now comes back as a null CELL with its
|
|
477
|
-
// row and column intact (re-probed 2026-08-08), which leaves 999999 meaning
|
|
478
|
-
// an actual server fault — but the parameter checklist
|
|
479
|
-
// is still the right first move, because a wrong param name or date axis
|
|
480
|
-
// silently yields an EMPTY TABLE rather than this error. Only the data
|
|
481
|
-
// endpoints take a date/security/params; indicator.search shares the
|
|
482
|
-
// no-999999 policy but has just a keyword, so it keeps the generic hint
|
|
483
|
-
// instead of nonsensical date/scope/param guidance.
|
|
484
358
|
const isIndicatorFetch = endpoint.key === 'indicator.cross-section' || endpoint.key === 'indicator.time-series' || endpoint.key === 'indicator.screener';
|
|
485
359
|
if (isIndicatorFetch && error instanceof ApiError && error.code === '999999') {
|
|
486
360
|
throw new ApiError(error.message, error.code, error.statusCode, error.details, error.retryAfterMs, 'EDE 取数故障。先核对参数再重试:参数名以 indicator search 的 parameterList 为准(传错名是静默失效)、日期匹配指标周期(财务报表类用报告期末如 2025-12-31,PE/PB 等日频估值用交易日)、标的在 scopeList 覆盖内、required 参数已补。注意此码不表示无数据——有效 code 无数据现在返回保留行列的 null 单元格,空表则多半是整个轴的 code 都没被识别。');
|
|
@@ -488,21 +362,15 @@ export class GangtiseClient {
|
|
|
488
362
|
throw error;
|
|
489
363
|
});
|
|
490
364
|
}
|
|
491
|
-
/** POST a file as multipart/form-data under the field name `file`. Reuses
|
|
492
|
-
* requestJson for auth / retry / envelope handling — only the body differs. */
|
|
493
365
|
async uploadFile(endpointKey, file) {
|
|
494
366
|
const endpoint = ENDPOINTS[endpointKey];
|
|
495
367
|
if (!endpoint || endpoint.kind !== 'upload') {
|
|
496
368
|
throw new ApiError(`Not an upload endpoint: ${endpointKey}`);
|
|
497
369
|
}
|
|
498
370
|
const form = new FormData();
|
|
499
|
-
// Cast: TS 5.7+ types Uint8Array as Uint8Array<ArrayBufferLike>, which BlobPart
|
|
500
|
-
// (ArrayBufferView<ArrayBuffer>) rejects; a Node Buffer is always ArrayBuffer-backed.
|
|
501
371
|
form.append('file', new Blob([file.data], { type: file.contentType ?? 'application/octet-stream' }), file.filename);
|
|
502
372
|
return this.requestJson(endpoint, form);
|
|
503
373
|
}
|
|
504
|
-
/** `body` is only sent for POST download endpoints (the file-parse result
|
|
505
|
-
* endpoint takes `{taskId}` as JSON and answers with the ZIP bytes). */
|
|
506
374
|
async download(endpoint, query, options, body) {
|
|
507
375
|
const dispatcher = getDispatcher();
|
|
508
376
|
const url = this.buildUrl(endpoint.path);
|
|
@@ -526,10 +394,6 @@ export class GangtiseClient {
|
|
|
526
394
|
bodyTimeout: this.config.timeoutMs,
|
|
527
395
|
dispatcher,
|
|
528
396
|
});
|
|
529
|
-
// undici does not follow redirects, and a download endpoint may 302 to a
|
|
530
|
-
// pre-signed object-store URL — without this the redirect body would be
|
|
531
|
-
// saved as the "file". Follow up to 3 hops, dropping Authorization once the
|
|
532
|
-
// redirect leaves the API origin so the bearer never reaches storage hosts.
|
|
533
397
|
for (let hops = 0; hops < 3 && response.statusCode >= 300 && response.statusCode < 400; hops++) {
|
|
534
398
|
const locationHeader = response.headers.location;
|
|
535
399
|
const location = Array.isArray(locationHeader) ? locationHeader[0] : locationHeader;
|
|
@@ -548,16 +412,11 @@ export class GangtiseClient {
|
|
|
548
412
|
dispatcher,
|
|
549
413
|
});
|
|
550
414
|
}
|
|
551
|
-
// The loop above can exit with a 3xx still in hand (hop limit exceeded, or a
|
|
552
|
-
// redirect without Location) — that response must never be treated as file
|
|
553
|
-
// content: its HTML placeholder body would be saved as the "downloaded file".
|
|
554
415
|
if (response.statusCode >= 300 && response.statusCode < 400) {
|
|
555
416
|
await response.body.text().catch(() => { });
|
|
556
417
|
throw new ApiError(`Download failed: unresolved redirect (HTTP ${response.statusCode})`, undefined, response.statusCode);
|
|
557
418
|
}
|
|
558
419
|
const contentType = Array.isArray(response.headers['content-type']) ? response.headers['content-type'][0] : response.headers['content-type'];
|
|
559
|
-
// From the final (post-redirect) response, so a rate-limited download honors
|
|
560
|
-
// Retry-After too — every error branch below passes it into the ApiError.
|
|
561
420
|
const retryAfterMs = parseRetryAfterMs(response.headers['retry-after'], Date.now());
|
|
562
421
|
if (contentType?.includes('application/json')) {
|
|
563
422
|
const text = await response.body.text();
|
|
@@ -604,9 +463,6 @@ export class GangtiseClient {
|
|
|
604
463
|
const filenameMatch = Array.isArray(contentDisposition)
|
|
605
464
|
? contentDisposition[0]?.match(/filename\*=UTF-8''([^;]+)|filename="?([^";]+)"?/i)
|
|
606
465
|
: contentDisposition?.match(/filename\*=UTF-8''([^;]+)|filename="?([^";]+)"?/i);
|
|
607
|
-
// A plain filename= value with a bare % ("增长100%.pdf") is not valid URI
|
|
608
|
-
// encoding — decodeURIComponent would throw and fail the whole download over
|
|
609
|
-
// a cosmetic hint. Fall back to the raw value instead.
|
|
610
466
|
let filename;
|
|
611
467
|
if (filenameMatch) {
|
|
612
468
|
const raw = filenameMatch[1] || filenameMatch[2];
|
|
@@ -617,13 +473,8 @@ export class GangtiseClient {
|
|
|
617
473
|
filename = raw;
|
|
618
474
|
}
|
|
619
475
|
}
|
|
620
|
-
// Stream directly to disk when caller already knows the destination
|
|
621
476
|
if (options?.streamTo) {
|
|
622
477
|
await fs.mkdir(path.dirname(options.streamTo), { recursive: true });
|
|
623
|
-
// Stream into a .part sibling and rename over the target only on success:
|
|
624
|
-
// writing to the target directly would truncate an existing file on the
|
|
625
|
-
// FIRST byte and delete it on failure — a failed re-download (or each
|
|
626
|
-
// withRetry attempt) must never destroy the user's previous good file.
|
|
627
478
|
const partPath = `${options.streamTo}.part`;
|
|
628
479
|
try {
|
|
629
480
|
await pipeline(response.body, createWriteStream(partPath));
|
|
@@ -644,8 +495,6 @@ export class GangtiseClient {
|
|
|
644
495
|
filename,
|
|
645
496
|
};
|
|
646
497
|
}, {
|
|
647
|
-
// Download endpoints carry per-篇 billing too (summary/foreign-report/
|
|
648
|
-
// my-conference at 50/篇) — honor the endpoint's retry policy here as well.
|
|
649
498
|
policy: endpoint.retry ?? "default",
|
|
650
499
|
onRetry: (attempt, error, delay) => {
|
|
651
500
|
if (!isVerbose())
|
|
@@ -34,27 +34,6 @@ export function buildStockPoolStocksBody(options) {
|
|
|
34
34
|
poolIdList: options.poolId?.length ? options.poolId : ["all"],
|
|
35
35
|
};
|
|
36
36
|
}
|
|
37
|
-
/** Parameter keys that REPLACE an indicator's `tradeDate`. The 2026-08-01 EDE
|
|
38
|
-
* revision dropped the root-level `date`, so `--date` now fans out into each
|
|
39
|
-
* indicator's `tradeDate`. A caller-supplied `reportDate` suppresses that
|
|
40
|
-
* injection for its indicator, so the two date fields are never sent together.
|
|
41
|
-
*
|
|
42
|
-
* This is load-bearing, and the server has moved twice on it. A report-period
|
|
43
|
-
* indicator answered a `tradeDate` with an EMPTY result (probed 2026-08-01 on
|
|
44
|
-
* `is_op_rev_mom`), then resolved it to the enclosing report period (2026-08-08,
|
|
45
|
-
* same 33.4903 either way), and since 2026-08-14 REJECTS it outright:
|
|
46
|
-
* `100003 指标 is_op_rev_mom 不支持参数 tradeDate; 缺少必填参数 reportDate`
|
|
47
|
-
* (re-probed 2026-08-15; `reportDate` still returns 33.4903). So a caller-supplied
|
|
48
|
-
* `reportDate` is now the only way to reach those indicators, and suppressing the
|
|
49
|
-
* `tradeDate` injection is what makes it work rather than merely being tidier.
|
|
50
|
-
* Injecting into an indicator that takes no date at all is harmless —
|
|
51
|
-
* `pty_op_scope` still answers normally with a stray `tradeDate` attached.
|
|
52
|
-
*
|
|
53
|
-
* `sDate` is deliberately NOT here. It is an interval START, not a substitute:
|
|
54
|
-
* `qte_vol_intvl` declares `tradeDate` required (the interval END) and `sDate`
|
|
55
|
-
* optional. Treating it as a replacement dropped `--date` and silently moved the
|
|
56
|
-
* interval end — probed 2026-08-02, 茅台 sDate=2024-01-02 returned 2,265,873,849
|
|
57
|
-
* without a tradeDate vs 65,687,435 with tradeDate=2024-01-31, both exit 0. */
|
|
58
37
|
const DATE_PARAM_KEYS = new Set(["tradeDate", "reportDate"]);
|
|
59
38
|
function withQueryDate(groups, codes, date) {
|
|
60
39
|
const merged = new Map();
|
|
@@ -65,11 +44,15 @@ function withQueryDate(groups, codes, date) {
|
|
|
65
44
|
if (!group) {
|
|
66
45
|
merged.set(code, { indicatorCode: code, parameters: [{ paramKey: "tradeDate", paramValue: date }] });
|
|
67
46
|
}
|
|
68
|
-
else if (!group.parameters.some((param) => DATE_PARAM_KEYS.has(param.paramKey))) {
|
|
47
|
+
else if (!group.noQueryDate && !group.parameters.some((param) => DATE_PARAM_KEYS.has(param.paramKey))) {
|
|
69
48
|
group.parameters.push({ paramKey: "tradeDate", paramValue: date });
|
|
70
49
|
}
|
|
71
50
|
}
|
|
72
|
-
return [...merged.values()];
|
|
51
|
+
return [...merged.values()].map(stripMarker);
|
|
52
|
+
}
|
|
53
|
+
function stripMarker(item) {
|
|
54
|
+
const { noQueryDate, ...rest } = item;
|
|
55
|
+
return rest;
|
|
73
56
|
}
|
|
74
57
|
export function buildIndicatorCrossSectionBody(options) {
|
|
75
58
|
return {
|
|
@@ -89,8 +72,7 @@ export function buildIndicatorTimeSeriesBody(options) {
|
|
|
89
72
|
calendarType: options.calendarType,
|
|
90
73
|
currency: options.currency,
|
|
91
74
|
scale: options.scale,
|
|
92
|
-
|
|
93
|
-
indicatorParamList: parseIndicatorParams(options.indicatorParam) ?? [],
|
|
75
|
+
indicatorParamList: (parseIndicatorParams(options.indicatorParam) ?? []).map(stripMarker),
|
|
94
76
|
};
|
|
95
77
|
}
|
|
96
78
|
export function buildIndicatorScreenerBody(options) {
|
|
@@ -98,18 +80,8 @@ export function buildIndicatorScreenerBody(options) {
|
|
|
98
80
|
return {
|
|
99
81
|
universe: maybeArray(options.security),
|
|
100
82
|
expression: options.expression,
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
// per-indicator exception.
|
|
105
|
-
//
|
|
106
|
-
// It also used to be load-bearing: through 2026-08-02 the screener DROPPED
|
|
107
|
-
// any indicator sent with `parameters: []`, so the official doc's own
|
|
108
|
-
// `F3 contains '酒'` example could not work as written. Re-probed 2026-08-03:
|
|
109
|
-
// fixed server-side. Kept anyway — unconditional is simpler than conditional,
|
|
110
|
-
// and it survives a rollback.
|
|
111
|
-
indicatorList: indicators.map((indicator) => (indicator.parameters.some((param) => DATE_PARAM_KEYS.has(param.paramKey))
|
|
112
|
-
? indicator
|
|
113
|
-
: { ...indicator, parameters: [...indicator.parameters, { paramKey: "tradeDate", paramValue: options.date }] })),
|
|
83
|
+
indicatorList: indicators.map((indicator) => (indicator.noQueryDate || indicator.parameters.some((param) => DATE_PARAM_KEYS.has(param.paramKey))
|
|
84
|
+
? stripMarker(indicator)
|
|
85
|
+
: { ...stripMarker(indicator), parameters: [...indicator.parameters, { paramKey: "tradeDate", paramValue: options.date }] })),
|
|
114
86
|
};
|
|
115
87
|
}
|
|
@@ -3,34 +3,18 @@ import { extname } from "node:path";
|
|
|
3
3
|
import { DownloadError } from "./errors.js";
|
|
4
4
|
import { saveOutputIfNeeded } from "./output.js";
|
|
5
5
|
import { lookupTitleCache, readTitleCache, TITLE_LOOKUP_SIZE } from "./titleCache.js";
|
|
6
|
-
/** Replace filesystem-unsafe characters (path separators, wildcards, and control
|
|
7
|
-
* characters / NUL) with `_` so a title or a server-supplied filename can't create
|
|
8
|
-
* stray subdirectories, escape the intended output path, or break fs.writeFile.
|
|
9
|
-
* Shared by title-based naming and the download fallback. */
|
|
10
6
|
function sanitizeFilename(name) {
|
|
11
7
|
return name.replace(/[/\\:*?"<>|\u0000-\u001f]/g, "_");
|
|
12
8
|
}
|
|
13
|
-
/** Keep auto-derived filenames under 200 UTF-8 bytes, preserving the extension.
|
|
14
|
-
* ext4 caps directory entries at 255 bytes — long Chinese titles hit that at
|
|
15
|
-
* ~85 chars (3 bytes each) and fs.writeFile throws ENAMETOOLONG after the body
|
|
16
|
-
* has already been downloaded. */
|
|
17
9
|
function truncateFilename(name, maxBytes = 200) {
|
|
18
10
|
if (Buffer.byteLength(name, "utf8") <= maxBytes)
|
|
19
11
|
return name;
|
|
20
12
|
const ext = extname(name);
|
|
21
|
-
// Trim by code point, not UTF-16 unit: a cut inside a surrogate pair would put
|
|
22
|
-
// a lone surrogate in the name, which reaches the filesystem as U+FFFD (�).
|
|
23
13
|
const stem = [...name.slice(0, name.length - ext.length)];
|
|
24
14
|
while (stem.length > 1 && Buffer.byteLength(stem.join("") + ext, "utf8") > maxBytes)
|
|
25
15
|
stem.pop();
|
|
26
16
|
return stem.join("") + ext;
|
|
27
17
|
}
|
|
28
|
-
/** Pick a non-existing path by suffixing -1, -2, … before the extension, so batch
|
|
29
|
-
* downloads whose titles collide ("2025年第一季度报告" from several companies) don't
|
|
30
|
-
* silently overwrite each other. Only auto-derived names go through this — an
|
|
31
|
-
* explicit --output path keeps plain overwrite semantics. Throws instead of
|
|
32
|
-
* falling back to the original path once the suffixes run out: returning `p`
|
|
33
|
-
* there would silently overwrite the very first file. */
|
|
34
18
|
export async function uniquePath(p) {
|
|
35
19
|
const exists = (f) => fs.access(f).then(() => true, () => false);
|
|
36
20
|
if (!(await exists(p)))
|
|
@@ -93,7 +77,6 @@ export async function resolveTitle(client, result, listEndpoint, idField, idValu
|
|
|
93
77
|
return buildFilename(cached);
|
|
94
78
|
}
|
|
95
79
|
catch {
|
|
96
|
-
// Ignore corrupt cache data and fall back to the list endpoint.
|
|
97
80
|
}
|
|
98
81
|
try {
|
|
99
82
|
const resp = await client.call(listEndpoint, { from: 0, size: TITLE_LOOKUP_SIZE });
|
|
@@ -115,13 +98,8 @@ async function downloadUrlTo(url, outputPath) {
|
|
|
115
98
|
const { request } = await import("undici");
|
|
116
99
|
const { getDispatcher, logTiming, withRetry } = await import("./transport.js");
|
|
117
100
|
const { loadConfig } = await import("./config.js");
|
|
118
|
-
// Through the transport layer instead of a bare global fetch: the configured
|
|
119
|
-
// timeout applies (a slow-drip CDN can no longer hang the CLI indefinitely),
|
|
120
|
-
// network-level failures retry, and --verbose logs the request.
|
|
121
101
|
const timeoutMs = loadConfig().timeoutMs;
|
|
122
102
|
await fs.mkdir(dirname(outputPath), { recursive: true });
|
|
123
|
-
// Signed URLs carry credentials in the query string — verbose logs must strip
|
|
124
|
-
// search/hash so signatures never land in terminal/CI logs.
|
|
125
103
|
const redactUrl = (u) => {
|
|
126
104
|
try {
|
|
127
105
|
const parsed = new URL(u);
|
|
@@ -131,7 +109,6 @@ async function downloadUrlTo(url, outputPath) {
|
|
|
131
109
|
return "signed-url";
|
|
132
110
|
}
|
|
133
111
|
};
|
|
134
|
-
// .part + rename so a failed follow-download never destroys an existing file.
|
|
135
112
|
const partPath = `${outputPath}.part`;
|
|
136
113
|
await withRetry(async () => {
|
|
137
114
|
const startedAt = Date.now();
|
|
@@ -139,18 +116,11 @@ async function downloadUrlTo(url, outputPath) {
|
|
|
139
116
|
method: "GET",
|
|
140
117
|
headersTimeout: timeoutMs,
|
|
141
118
|
bodyTimeout: timeoutMs,
|
|
142
|
-
// headers/body timeouts are IDLE timeouts — a stream trickling one byte
|
|
143
|
-
// per interval resets them forever. A generous total deadline (10× the
|
|
144
|
-
// per-request timeout) bounds the whole transfer without killing large
|
|
145
|
-
// legitimate downloads.
|
|
146
119
|
signal: AbortSignal.timeout(timeoutMs * 10),
|
|
147
120
|
dispatcher: getDispatcher(),
|
|
148
121
|
};
|
|
149
122
|
let currentUrl = url;
|
|
150
123
|
let response = await request(currentUrl, requestOptions);
|
|
151
|
-
// undici does not follow redirects (the old global fetch did): a signed URL
|
|
152
|
-
// may 302 to the real CDN object. Follow a bounded number of hops; no
|
|
153
|
-
// Authorization header is involved on this path, so nothing leaks cross-origin.
|
|
154
124
|
for (let hops = 0; hops < 3 && response.statusCode >= 300 && response.statusCode < 400; hops++) {
|
|
155
125
|
const locationHeader = response.headers.location;
|
|
156
126
|
const location = Array.isArray(locationHeader) ? locationHeader[0] : locationHeader;
|
|
@@ -166,7 +136,6 @@ async function downloadUrlTo(url, outputPath) {
|
|
|
166
136
|
}
|
|
167
137
|
if (response.statusCode >= 400) {
|
|
168
138
|
await response.body.text().catch(() => { });
|
|
169
|
-
// DownloadError (not retried): signed URLs expire — replaying a 403 is useless.
|
|
170
139
|
throw new DownloadError(`Failed to fetch download URL (HTTP ${response.statusCode})`);
|
|
171
140
|
}
|
|
172
141
|
try {
|
|
@@ -182,7 +151,6 @@ async function downloadUrlTo(url, outputPath) {
|
|
|
182
151
|
await fs.rename(partPath, outputPath);
|
|
183
152
|
}
|
|
184
153
|
catch (error) {
|
|
185
|
-
// e.g. outputPath turned out to be a directory — don't leave the .part behind.
|
|
186
154
|
await fs.unlink(partPath).catch(() => { });
|
|
187
155
|
throw error;
|
|
188
156
|
}
|
|
@@ -192,16 +160,12 @@ export async function saveDownloadResult(result, fallbackName, output) {
|
|
|
192
160
|
throw new DownloadError("Unexpected download response");
|
|
193
161
|
}
|
|
194
162
|
const file = result;
|
|
195
|
-
// The fallback prefix embeds a user-supplied id (e.g. --report-id): sanitize it
|
|
196
|
-
// like server-provided names so an auto filename never interprets input as a path.
|
|
197
163
|
const safeFallback = sanitizeFilename(fallbackName);
|
|
198
164
|
if (typeof file.savedPath === "string") {
|
|
199
165
|
process.stdout.write(`${file.savedPath}\n`);
|
|
200
166
|
return;
|
|
201
167
|
}
|
|
202
168
|
if (file.data instanceof Uint8Array) {
|
|
203
|
-
// Sanitize the server-provided filename so a Content-Disposition value with
|
|
204
|
-
// / or : can't write outside the intended path (same rule as buildFilename).
|
|
205
169
|
const autoName = (file.filename ? sanitizeFilename(file.filename) : undefined) ?? (safeFallback + extFromContentType(file.contentType));
|
|
206
170
|
const outputPath = output ?? await uniquePath(truncateFilename(autoName));
|
|
207
171
|
await saveOutputIfNeeded(file.data, outputPath);
|
|
@@ -216,9 +180,6 @@ export async function saveDownloadResult(result, fallbackName, output) {
|
|
|
216
180
|
}
|
|
217
181
|
if (typeof file.url === "string") {
|
|
218
182
|
if (output) {
|
|
219
|
-
// The server handed us a (typically signed, short-lived) URL instead of the
|
|
220
|
-
// bytes. The user asked for a file — follow the URL and stream the content
|
|
221
|
-
// to disk instead of writing the URL string into a fake .pdf.
|
|
222
183
|
await downloadUrlTo(file.url, output);
|
|
223
184
|
process.stdout.write(`${output}\n`);
|
|
224
185
|
return;
|