gangtise-openapi-cli 0.34.0 → 0.35.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -5
- package/dist/src/cli.js +4 -268
- package/dist/src/core/args.js +19 -136
- package/dist/src/core/asyncContent.js +0 -22
- package/dist/src/core/auth.js +0 -14
- package/dist/src/core/client.js +5 -152
- package/dist/src/core/commandBodies.js +7 -35
- package/dist/src/core/download.js +0 -39
- package/dist/src/core/endpoints.js +0 -31
- package/dist/src/core/errors.js +61 -63
- package/dist/src/core/fileParse.js +0 -16
- package/dist/src/core/indicatorMatrix.js +0 -207
- package/dist/src/core/normalize.js +0 -14
- package/dist/src/core/output.js +0 -56
- package/dist/src/core/printer.js +0 -19
- package/dist/src/core/quoteSharding.js +0 -58
- package/dist/src/core/titleCache.js +45 -50
- package/dist/src/core/transport.js +0 -45
- package/dist/src/version.js +1 -2
- package/gangtise-openapi/SKILL.md +2 -2
- package/gangtise-openapi/references/commands/indicator.md +24 -5
- package/gangtise-openapi/references/commands/insight.md +26 -5
- package/gangtise-openapi/references/examples.md +2 -1
- package/package.json +1 -1
package/dist/src/core/client.js
CHANGED
|
@@ -8,22 +8,11 @@ import { ApiError, attachEnvelopeTraceId, ValidationError } from "./errors.js";
|
|
|
8
8
|
import { ENDPOINTS, resolveTimeoutMs } from "./endpoints.js";
|
|
9
9
|
import { getLookupData } from "./lookupData/index.js";
|
|
10
10
|
import { decodeResponseBody, getDispatcher, isVerbose, logTiming, markRetryable, PAGE_CONCURRENCY, parseRetryAfterMs, quoteBigIntFields, runWithConcurrency, withRetry } from "./transport.js";
|
|
11
|
-
// Auth errors that warrant a forced re-login + one replay: the token was rejected
|
|
12
|
-
// server-side while still looking valid by local expiry, so only a forced refresh
|
|
13
|
-
// recovers it. 0000001008 is the legacy code (probed 2026-07-20: still what the
|
|
14
|
-
// token filter emits); 999002 TOKEN_INVALID is its 2026-07-17 replacement, listed
|
|
15
|
-
// ahead of the rollout so self-heal does not silently die when the filter switches.
|
|
16
|
-
// 8000014/8000015 are the retired AK/SK codes, kept for older server builds.
|
|
17
|
-
// 999011 CREDENTIAL_INVALID is not here and could not act if it were — it comes from
|
|
18
|
-
// auth.login, which runs useAuth=false and so never reaches this check. Its "never
|
|
19
|
-
// replay" guarantee lives in transport's TERMINAL_API_CODES instead.
|
|
20
11
|
const AUTH_RETRY_CODES = new Set(["8000014", "8000015", "0000001008", "999002"]);
|
|
21
12
|
export class GangtiseClient {
|
|
22
13
|
config;
|
|
23
14
|
refreshPromise = null;
|
|
24
15
|
memoCache = null;
|
|
25
|
-
// After an injected env token (GANGTISE_TOKEN) is rejected and we self-heal via
|
|
26
|
-
// login, stop preferring that now-stale token so the retry uses the fresh one.
|
|
27
16
|
envTokenInvalidated = false;
|
|
28
17
|
constructor(config) {
|
|
29
18
|
this.config = config;
|
|
@@ -53,15 +42,10 @@ export class GangtiseClient {
|
|
|
53
42
|
accessKey: credentials.accessKey,
|
|
54
43
|
secretKey: credentials.secretKey,
|
|
55
44
|
}, false);
|
|
56
|
-
// Validate the shape before touching it: a missing accessToken used to surface
|
|
57
|
-
// as a bare TypeError from normalizeToken, hiding the real cause.
|
|
58
45
|
if (typeof envelope?.accessToken !== "string" || !envelope.accessToken) {
|
|
59
46
|
throw new ApiError("Login succeeded but the response carried no accessToken", undefined, undefined, envelope);
|
|
60
47
|
}
|
|
61
48
|
const accessToken = normalizeToken(envelope.accessToken);
|
|
62
|
-
// A non-numeric expiresIn would make expiresAt NaN — the cache would never
|
|
63
|
-
// validate and every command would silently re-login. Degrade to 0 instead
|
|
64
|
-
// (token works now, next process logs in again).
|
|
65
49
|
const expiresIn = Number.isFinite(envelope.expiresIn) ? envelope.expiresIn : 0;
|
|
66
50
|
const expiresAt = Math.floor(Date.now() / 1000) + expiresIn;
|
|
67
51
|
const cache = { ...envelope, accessToken, expiresIn, expiresAt };
|
|
@@ -70,20 +54,11 @@ export class GangtiseClient {
|
|
|
70
54
|
await writeTokenCache(this.config.tokenCachePath, cache);
|
|
71
55
|
}
|
|
72
56
|
catch (error) {
|
|
73
|
-
// A read-only HOME or full disk must not fail the request — the in-memory
|
|
74
|
-
// token is valid; we just can't persist it for the next process.
|
|
75
57
|
const msg = error instanceof Error ? error.message : String(error);
|
|
76
58
|
process.stderr.write(`[gangtise] warning: could not persist token cache: ${msg}\n`);
|
|
77
59
|
}
|
|
78
60
|
return accessToken;
|
|
79
61
|
}
|
|
80
|
-
/**
|
|
81
|
-
* On a recoverable auth error (expired/invalid token codes), force a one-time
|
|
82
|
-
* token refresh and re-throw as retryable so withRetry replays the request.
|
|
83
|
-
* Otherwise — or once we've already retried this request — it's a no-op and
|
|
84
|
-
* the caller re-throws the original error. `authState` persists across the
|
|
85
|
-
* withRetry attempts so we only refresh once per logical request.
|
|
86
|
-
*/
|
|
87
62
|
async refreshAuthIfRecoverable(error, useAuth, authState, usedAuthorization) {
|
|
88
63
|
if (useAuth
|
|
89
64
|
&& !authState.retried
|
|
@@ -94,13 +69,6 @@ export class GangtiseClient {
|
|
|
94
69
|
&& this.config.secretKey) {
|
|
95
70
|
authState.retried = true;
|
|
96
71
|
this.envTokenInvalidated = true;
|
|
97
|
-
// If the failed request was still carrying an OLDER token than the one now in
|
|
98
|
-
// memoCache, another request already refreshed — replay with the fresh token
|
|
99
|
-
// instead of logging in again (back-to-back logins can kick each other's
|
|
100
|
-
// sessions server-side, the 0000001008 semantics). If the failed request used
|
|
101
|
-
// the CURRENT token, that token is genuinely dead: force a new login. A time
|
|
102
|
-
// window is NOT a valid proxy here — right after the initial login the window
|
|
103
|
-
// is always "recent", which would skip the refresh exactly when it's needed.
|
|
104
72
|
const memoToken = this.memoCache && isTokenCacheValid(this.memoCache) ? normalizeToken(this.memoCache.accessToken) : null;
|
|
105
73
|
const alreadyRefreshed = memoToken !== null && usedAuthorization !== undefined && usedAuthorization !== memoToken;
|
|
106
74
|
if (!alreadyRefreshed) {
|
|
@@ -110,9 +78,6 @@ export class GangtiseClient {
|
|
|
110
78
|
throw markRetryable(new ApiError(error.message, error.code, error.statusCode, error.details));
|
|
111
79
|
}
|
|
112
80
|
}
|
|
113
|
-
/** `new URL("/a/b", "https://proxy/prefix")` drops "/prefix" — an absolute path
|
|
114
|
-
* replaces the base's path per the URL spec. Join manually so a reverse-proxy
|
|
115
|
-
* GANGTISE_BASE_URL with a path prefix keeps working. */
|
|
116
81
|
buildUrl(path) {
|
|
117
82
|
const base = this.config.baseUrl.endsWith("/") ? this.config.baseUrl : `${this.config.baseUrl}/`;
|
|
118
83
|
return new URL(path.replace(/^\//, ""), base);
|
|
@@ -142,9 +107,6 @@ export class GangtiseClient {
|
|
|
142
107
|
throw new ApiError(parsed.msg || "API request failed", code, statusCode, parsed, retryAfterMs);
|
|
143
108
|
}
|
|
144
109
|
if ('data' in parsed) {
|
|
145
|
-
// Carry the envelope's traceId onto the payload: the EDE endpoints wrap a
|
|
146
|
-
// second envelope inside `data` and raise their own failures from it, by
|
|
147
|
-
// which point this is the only traceId in reach.
|
|
148
110
|
return attachEnvelopeTraceId(parsed.data, parsed.traceId);
|
|
149
111
|
}
|
|
150
112
|
return parsed;
|
|
@@ -177,7 +139,6 @@ export class GangtiseClient {
|
|
|
177
139
|
const startFrom = typeof initialBody.from === 'number' && Number.isFinite(initialBody.from) ? initialBody.from : 0;
|
|
178
140
|
const requestedSize = typeof initialBody.size === 'number' && Number.isFinite(initialBody.size) ? initialBody.size : undefined;
|
|
179
141
|
const maxPageSize = endpoint.pagination?.maxPageSize ?? requestedSize ?? 20;
|
|
180
|
-
// First page: serial — we need total before deciding how many more requests to fan out.
|
|
181
142
|
const firstPageSize = requestedSize === undefined ? maxPageSize : Math.min(maxPageSize, requestedSize);
|
|
182
143
|
const firstPage = await this.requestJson(endpoint, {
|
|
183
144
|
...initialBody,
|
|
@@ -185,26 +146,7 @@ export class GangtiseClient {
|
|
|
185
146
|
size: firstPageSize,
|
|
186
147
|
});
|
|
187
148
|
if (!this.isPaginatedListResponse(firstPage)) {
|
|
188
|
-
// Shape drift (e.g. total arriving as a string) silently degrades fetch-all
|
|
189
|
-
// to a single page with no partial marker. This is NOT hypothetical: passing
|
|
190
|
-
// industryList to insight foreign-opinion / independent-opinion makes the
|
|
191
|
-
// server answer 200 + data:null, which then renders as one phantom row
|
|
192
|
-
// ({"value":null} in jsonl, a blank row in table/markdown). Verbose-only was
|
|
193
|
-
// the wrong bar — a caller who never sets GANGTISE_VERBOSE reads that row as
|
|
194
|
-
// real data, so warn on stderr unconditionally.
|
|
195
149
|
process.stderr.write(`[gangtise] warning: ${endpoint.key} is marked paginated but the first page has an unexpected shape (no numeric total + list); returning it as-is\n`);
|
|
196
|
-
// A warning only helps a human. Scripts read the exit code, and without one they
|
|
197
|
-
// cannot tell "this filter legitimately matched nothing" from "this filter is
|
|
198
|
-
// broken". Reuse exit 3 (result is not what was asked for) rather than inventing a
|
|
199
|
-
// fourth code — fetch-all silently degraded to one page either way.
|
|
200
|
-
//
|
|
201
|
-
// Applies to EVERY malformed shape, not just `null`: a string `total` truncates the
|
|
202
|
-
// result to page 1, which looks complete and is therefore worse than an obviously
|
|
203
|
-
// empty payload. All 24 paginated endpoints are genuine {total, list} lists (the
|
|
204
|
-
// odd-shaped ones like reference.constant-list are not marked paginated), so there
|
|
205
|
-
// is no legitimate response that lands here. Endpoints where `null` IS a valid
|
|
206
|
-
// answer (ai.one-pager for a security with no generated content) are unpaginated
|
|
207
|
-
// and never reach this branch.
|
|
208
150
|
process.exitCode = 3;
|
|
209
151
|
return firstPage;
|
|
210
152
|
}
|
|
@@ -212,9 +154,6 @@ export class GangtiseClient {
|
|
|
212
154
|
const collected = [...firstPage.list];
|
|
213
155
|
const available = Math.max(total - startFrom, 0);
|
|
214
156
|
const target = requestedSize === undefined ? available : Math.min(requestedSize, available);
|
|
215
|
-
// Last page reached on first request. If `total` promises more rows than the
|
|
216
|
-
// short page delivered, the server's page cap may be lower than our configured
|
|
217
|
-
// maxPageSize — say so instead of silently returning a subset as "everything".
|
|
218
157
|
if (firstPage.list.length < firstPageSize) {
|
|
219
158
|
const out = {
|
|
220
159
|
...firstPage,
|
|
@@ -223,8 +162,6 @@ export class GangtiseClient {
|
|
|
223
162
|
};
|
|
224
163
|
if (collected.length < target) {
|
|
225
164
|
process.stderr.write(`[gangtise] warning: server returned a short page (${collected.length} rows) but reported total=${total}; treating it as the end of data — results may be incomplete\n`);
|
|
226
|
-
// Machine-readable counterpart of the warning: scripts key off partial /
|
|
227
|
-
// exit code 3, and must not mistake a truncated result for a complete one.
|
|
228
165
|
out.partial = true;
|
|
229
166
|
}
|
|
230
167
|
return out;
|
|
@@ -236,9 +173,6 @@ export class GangtiseClient {
|
|
|
236
173
|
list: requestedSize === undefined ? collected : collected.slice(0, requestedSize),
|
|
237
174
|
};
|
|
238
175
|
}
|
|
239
|
-
// Build remaining page requests. The cap lives inside the loop: a corrupt
|
|
240
|
-
// server `total` (e.g. 9e15) must not materialize millions of page objects —
|
|
241
|
-
// or spin for minutes — before a post-hoc truncation applies.
|
|
242
176
|
const MAX_PAGES = 1000;
|
|
243
177
|
let truncatedByPageCap = false;
|
|
244
178
|
const pageRequests = [];
|
|
@@ -256,11 +190,6 @@ export class GangtiseClient {
|
|
|
256
190
|
}
|
|
257
191
|
let unexpectedShape = false;
|
|
258
192
|
let totalDrift = false;
|
|
259
|
-
// Fail-soft fan-out: a hard page failure (rate-limit 903301, no-perm, retries
|
|
260
|
-
// exhausted) must NOT discard the pages already fetched. Catch per page, record
|
|
261
|
-
// it, and stop starting new requests so we don't keep burning quota into a rate
|
|
262
|
-
// limit. Mirrors quoteSharding's partial-result tolerance — but firstPage already
|
|
263
|
-
// succeeded to get here, so unlike sharding there's no total-failure case.
|
|
264
193
|
const failedPages = [];
|
|
265
194
|
let firstError = null;
|
|
266
195
|
let aborted = false;
|
|
@@ -276,8 +205,6 @@ export class GangtiseClient {
|
|
|
276
205
|
size: req.size,
|
|
277
206
|
});
|
|
278
207
|
if (!this.isPaginatedListResponse(page)) {
|
|
279
|
-
// Treat a shape-broken page like a failed page: its rows are missing, so
|
|
280
|
-
// the result must carry the partial marker instead of looking complete.
|
|
281
208
|
unexpectedShape = true;
|
|
282
209
|
failedPages.push(req);
|
|
283
210
|
return [];
|
|
@@ -305,39 +232,27 @@ export class GangtiseClient {
|
|
|
305
232
|
if (totalDrift) {
|
|
306
233
|
process.stderr.write(`[gangtise] warning: 'total' changed across pages (data shifted during fetch); rows may be duplicated or missing\n`);
|
|
307
234
|
}
|
|
308
|
-
// Always surface a cap-induced truncation (not gated on verbose): the user
|
|
309
|
-
// asked for everything and is silently getting a subset, mirroring the
|
|
310
|
-
// partial-result warning in quoteSharding.
|
|
311
235
|
if (truncatedByPageCap) {
|
|
312
236
|
process.stderr.write(`[gangtise] warning: hit the ${MAX_PAGES}-page safety cap; fetched ${collected.length} of ${total} rows. Narrow the query (e.g. a shorter date range) or pass --size to fetch a bounded subset.\n`);
|
|
313
237
|
}
|
|
314
238
|
const short = collected.length < target;
|
|
315
|
-
// `total` is not always the real row count. Three opinion endpoints report a fixed
|
|
316
|
-
// 10000 while `from=30000` still returns real rows with monotonically older publish
|
|
317
|
-
// times (the shape of Elasticsearch's default track_total_hits). A fetch-all then
|
|
318
|
-
// stops exactly at the cap with collected === total, so every completeness check
|
|
319
|
-
// below passes and the truncated export looks complete — the worst failure mode we
|
|
320
|
-
// have, because `opinion` bills 30 credits per row.
|
|
321
|
-
//
|
|
322
|
-
// Probe one row past the claimed end rather than hardcoding any number: the server
|
|
323
|
-
// can change the cap, and evidence survives that where a constant would not. Only on
|
|
324
|
-
// a genuine fetch-all that otherwise looked complete — when `total` is honest the
|
|
325
|
-
// probe comes back empty (and bills nothing, since these endpoints charge per row).
|
|
326
239
|
let totalCapped = false;
|
|
327
|
-
if (requestedSize === undefined && total > 0 && !short && !totalDrift && !truncatedByPageCap && failedPages.length === 0) {
|
|
240
|
+
if (endpoint.retry !== "no-replay" && requestedSize === undefined && total > 0 && !short && !totalDrift && !truncatedByPageCap && failedPages.length === 0) {
|
|
328
241
|
try {
|
|
329
242
|
const probe = await this.requestJson(endpoint, { ...initialBody, from: total, size: 1 });
|
|
330
243
|
if (this.isPaginatedListResponse(probe) && probe.list.length > 0)
|
|
331
244
|
totalCapped = true;
|
|
332
245
|
}
|
|
333
246
|
catch {
|
|
334
|
-
// A failed probe must never fail an otherwise-complete export: we simply do not
|
|
335
|
-
// learn whether the total was capped, which is the pre-existing behaviour.
|
|
336
247
|
}
|
|
337
248
|
}
|
|
338
249
|
if (totalCapped) {
|
|
339
250
|
process.stderr.write(`[gangtise] warning: ${endpoint.key} reported total=${total} but rows exist past that offset — 'total' is a server-side cap, not the real count. This export is TRUNCATED at ${collected.length} rows. Narrow the query (date range / filters) and fetch in slices.\n`);
|
|
340
251
|
}
|
|
252
|
+
if (endpoint.retry === "no-replay" && requestedSize === undefined && total > 0
|
|
253
|
+
&& !short && !totalDrift && !truncatedByPageCap && failedPages.length === 0 && isVerbose()) {
|
|
254
|
+
process.stderr.write(`[gangtise] total-cap probe skipped on ${endpoint.key} (per-call billing) — a capped 'total' would go undetected here\n`);
|
|
255
|
+
}
|
|
341
256
|
const out = {
|
|
342
257
|
...firstPage,
|
|
343
258
|
total,
|
|
@@ -347,15 +262,6 @@ export class GangtiseClient {
|
|
|
347
262
|
out.partial = true;
|
|
348
263
|
out.totalCapped = true;
|
|
349
264
|
}
|
|
350
|
-
// Unified completeness backstop. Whatever the cause — a failed/shape-broken page,
|
|
351
|
-
// a short later page (server page cap < maxPageSize), the MAX_PAGES cap, or `total`
|
|
352
|
-
// drifting mid-fetch — the result is partial. A short row count or a total drift each
|
|
353
|
-
// force it; so does any failedPages entry on its own — an over-returning sibling page
|
|
354
|
-
// can lift the row count back to target and mask the hole (short would read false), yet
|
|
355
|
-
// the failedPages branch below still writes "results are partial" to stderr, so the flag
|
|
356
|
-
// must agree. printData maps partial → exit 3 so a script can't read a truncated export
|
|
357
|
-
// as complete. The cap and drift branches above already warned on stderr; failedPages
|
|
358
|
-
// warns below.
|
|
359
265
|
if (short || totalDrift || failedPages.length > 0)
|
|
360
266
|
out.partial = true;
|
|
361
267
|
if (failedPages.length > 0) {
|
|
@@ -365,9 +271,6 @@ export class GangtiseClient {
|
|
|
365
271
|
process.stderr.write(`[gangtise] warning: ${failedPages.length}/${pageRequests.length} pages not fetched${detail}; results are partial — got ${collected.length}/${total} rows (see failedPages).${skippedHint}\n`);
|
|
366
272
|
}
|
|
367
273
|
else if (short && !truncatedByPageCap && !totalDrift) {
|
|
368
|
-
// A short later page with no failure, cap, or drift to explain it: the server
|
|
369
|
-
// simply delivered fewer rows than `total` promised. Warn so an interactive run
|
|
370
|
-
// sees why the result is partial (the other causes each warn on their own path).
|
|
371
274
|
process.stderr.write(`[gangtise] warning: server returned ${collected.length} of ${total} rows (a later page came back short); results may be incomplete\n`);
|
|
372
275
|
}
|
|
373
276
|
return out;
|
|
@@ -389,19 +292,12 @@ export class GangtiseClient {
|
|
|
389
292
|
const authState = { retried: false };
|
|
390
293
|
const timeoutMs = resolveTimeoutMs(this.config.timeoutMs, endpoint);
|
|
391
294
|
return withRetry(async () => {
|
|
392
|
-
// An upload endpoint hands undici the FormData itself: undici derives the
|
|
393
|
-
// multipart content-type (boundary included) from the body, so setting one
|
|
394
|
-
// here would corrupt the request.
|
|
395
295
|
const isUpload = endpoint.kind === 'upload';
|
|
396
296
|
const headers = {
|
|
397
|
-
// undici does not auto-decompress; decodeResponseBody gunzips below. Server
|
|
398
|
-
// gzip cuts JSON payloads ~3-10x (measured 3.6x on constant-list).
|
|
399
297
|
'accept-encoding': 'gzip',
|
|
400
298
|
};
|
|
401
299
|
if (!isUpload)
|
|
402
300
|
headers['content-type'] = 'application/json';
|
|
403
|
-
// Keep the header we actually sent: the self-heal check compares it against
|
|
404
|
-
// the current memoCache token to tell "stale token" from "fresh token died".
|
|
405
301
|
let usedAuthorization;
|
|
406
302
|
if (useAuth) {
|
|
407
303
|
usedAuthorization = await this.getAuthorizationHeader();
|
|
@@ -416,14 +312,10 @@ export class GangtiseClient {
|
|
|
416
312
|
bodyTimeout: timeoutMs,
|
|
417
313
|
dispatcher,
|
|
418
314
|
});
|
|
419
|
-
// Only buffer + gunzip when the server actually compressed; an unencoded
|
|
420
|
-
// response reads as text directly (and keeps existing behavior on that path).
|
|
421
315
|
const encoding = response.headers['content-encoding'];
|
|
422
316
|
const gzipped = (Array.isArray(encoding) ? encoding[0] : encoding)?.toLowerCase().trim() === 'gzip';
|
|
423
317
|
let text;
|
|
424
318
|
if (gzipped) {
|
|
425
|
-
// A proxy/middlebox can declare gzip and deliver garbage — surface that as
|
|
426
|
-
// an ApiError with request context instead of a bare zlib Z_DATA_ERROR.
|
|
427
319
|
const bytes = new Uint8Array(await response.body.arrayBuffer());
|
|
428
320
|
try {
|
|
429
321
|
text = decodeResponseBody(bytes, encoding);
|
|
@@ -437,9 +329,6 @@ export class GangtiseClient {
|
|
|
437
329
|
text = await response.body.text();
|
|
438
330
|
}
|
|
439
331
|
logTiming(`${endpoint.method} ${endpoint.path}`, Date.now() - startedAt, `${response.statusCode}, ${text.length}B`);
|
|
440
|
-
// Parse Retry-After once so every error path below (JSON parse failure AND the
|
|
441
|
-
// envelope/HTTP-error throw) carries it — a non-JSON 429/503 must still honor
|
|
442
|
-
// the server's rate window instead of falling back to default backoff.
|
|
443
332
|
const retryAfterMs = parseRetryAfterMs(response.headers['retry-after'], Date.now());
|
|
444
333
|
let parsed;
|
|
445
334
|
try {
|
|
@@ -452,8 +341,6 @@ export class GangtiseClient {
|
|
|
452
341
|
throw new ApiError(message, undefined, response.statusCode, text.slice(0, 500), retryAfterMs);
|
|
453
342
|
}
|
|
454
343
|
try {
|
|
455
|
-
// Auth errors can arrive as HTTP 4xx or as a 200-wrapped error envelope;
|
|
456
|
-
// both routes must reach the self-heal check below.
|
|
457
344
|
if (response.statusCode >= 400) {
|
|
458
345
|
this.throwHttpError(parsed, response.statusCode, retryAfterMs);
|
|
459
346
|
}
|
|
@@ -472,15 +359,6 @@ export class GangtiseClient {
|
|
|
472
359
|
process.stderr.write(`[gangtise] retry ${attempt} after ${delay.toFixed(0)}ms: ${msg.slice(0, 120)}\n`);
|
|
473
360
|
},
|
|
474
361
|
}).catch((error) => {
|
|
475
|
-
// Until 2026-08-01 EDE answered a no-data query with 999999, so this hint
|
|
476
|
-
// led with "多为查询无数据". No-data now comes back as a null CELL with its
|
|
477
|
-
// row and column intact (re-probed 2026-08-08), which leaves 999999 meaning
|
|
478
|
-
// an actual server fault — but the parameter checklist
|
|
479
|
-
// is still the right first move, because a wrong param name or date axis
|
|
480
|
-
// silently yields an EMPTY TABLE rather than this error. Only the data
|
|
481
|
-
// endpoints take a date/security/params; indicator.search shares the
|
|
482
|
-
// no-999999 policy but has just a keyword, so it keeps the generic hint
|
|
483
|
-
// instead of nonsensical date/scope/param guidance.
|
|
484
362
|
const isIndicatorFetch = endpoint.key === 'indicator.cross-section' || endpoint.key === 'indicator.time-series' || endpoint.key === 'indicator.screener';
|
|
485
363
|
if (isIndicatorFetch && error instanceof ApiError && error.code === '999999') {
|
|
486
364
|
throw new ApiError(error.message, error.code, error.statusCode, error.details, error.retryAfterMs, 'EDE 取数故障。先核对参数再重试:参数名以 indicator search 的 parameterList 为准(传错名是静默失效)、日期匹配指标周期(财务报表类用报告期末如 2025-12-31,PE/PB 等日频估值用交易日)、标的在 scopeList 覆盖内、required 参数已补。注意此码不表示无数据——有效 code 无数据现在返回保留行列的 null 单元格,空表则多半是整个轴的 code 都没被识别。');
|
|
@@ -488,21 +366,15 @@ export class GangtiseClient {
|
|
|
488
366
|
throw error;
|
|
489
367
|
});
|
|
490
368
|
}
|
|
491
|
-
/** POST a file as multipart/form-data under the field name `file`. Reuses
|
|
492
|
-
* requestJson for auth / retry / envelope handling — only the body differs. */
|
|
493
369
|
async uploadFile(endpointKey, file) {
|
|
494
370
|
const endpoint = ENDPOINTS[endpointKey];
|
|
495
371
|
if (!endpoint || endpoint.kind !== 'upload') {
|
|
496
372
|
throw new ApiError(`Not an upload endpoint: ${endpointKey}`);
|
|
497
373
|
}
|
|
498
374
|
const form = new FormData();
|
|
499
|
-
// Cast: TS 5.7+ types Uint8Array as Uint8Array<ArrayBufferLike>, which BlobPart
|
|
500
|
-
// (ArrayBufferView<ArrayBuffer>) rejects; a Node Buffer is always ArrayBuffer-backed.
|
|
501
375
|
form.append('file', new Blob([file.data], { type: file.contentType ?? 'application/octet-stream' }), file.filename);
|
|
502
376
|
return this.requestJson(endpoint, form);
|
|
503
377
|
}
|
|
504
|
-
/** `body` is only sent for POST download endpoints (the file-parse result
|
|
505
|
-
* endpoint takes `{taskId}` as JSON and answers with the ZIP bytes). */
|
|
506
378
|
async download(endpoint, query, options, body) {
|
|
507
379
|
const dispatcher = getDispatcher();
|
|
508
380
|
const url = this.buildUrl(endpoint.path);
|
|
@@ -526,10 +398,6 @@ export class GangtiseClient {
|
|
|
526
398
|
bodyTimeout: this.config.timeoutMs,
|
|
527
399
|
dispatcher,
|
|
528
400
|
});
|
|
529
|
-
// undici does not follow redirects, and a download endpoint may 302 to a
|
|
530
|
-
// pre-signed object-store URL — without this the redirect body would be
|
|
531
|
-
// saved as the "file". Follow up to 3 hops, dropping Authorization once the
|
|
532
|
-
// redirect leaves the API origin so the bearer never reaches storage hosts.
|
|
533
401
|
for (let hops = 0; hops < 3 && response.statusCode >= 300 && response.statusCode < 400; hops++) {
|
|
534
402
|
const locationHeader = response.headers.location;
|
|
535
403
|
const location = Array.isArray(locationHeader) ? locationHeader[0] : locationHeader;
|
|
@@ -548,16 +416,11 @@ export class GangtiseClient {
|
|
|
548
416
|
dispatcher,
|
|
549
417
|
});
|
|
550
418
|
}
|
|
551
|
-
// The loop above can exit with a 3xx still in hand (hop limit exceeded, or a
|
|
552
|
-
// redirect without Location) — that response must never be treated as file
|
|
553
|
-
// content: its HTML placeholder body would be saved as the "downloaded file".
|
|
554
419
|
if (response.statusCode >= 300 && response.statusCode < 400) {
|
|
555
420
|
await response.body.text().catch(() => { });
|
|
556
421
|
throw new ApiError(`Download failed: unresolved redirect (HTTP ${response.statusCode})`, undefined, response.statusCode);
|
|
557
422
|
}
|
|
558
423
|
const contentType = Array.isArray(response.headers['content-type']) ? response.headers['content-type'][0] : response.headers['content-type'];
|
|
559
|
-
// From the final (post-redirect) response, so a rate-limited download honors
|
|
560
|
-
// Retry-After too — every error branch below passes it into the ApiError.
|
|
561
424
|
const retryAfterMs = parseRetryAfterMs(response.headers['retry-after'], Date.now());
|
|
562
425
|
if (contentType?.includes('application/json')) {
|
|
563
426
|
const text = await response.body.text();
|
|
@@ -604,9 +467,6 @@ export class GangtiseClient {
|
|
|
604
467
|
const filenameMatch = Array.isArray(contentDisposition)
|
|
605
468
|
? contentDisposition[0]?.match(/filename\*=UTF-8''([^;]+)|filename="?([^";]+)"?/i)
|
|
606
469
|
: contentDisposition?.match(/filename\*=UTF-8''([^;]+)|filename="?([^";]+)"?/i);
|
|
607
|
-
// A plain filename= value with a bare % ("增长100%.pdf") is not valid URI
|
|
608
|
-
// encoding — decodeURIComponent would throw and fail the whole download over
|
|
609
|
-
// a cosmetic hint. Fall back to the raw value instead.
|
|
610
470
|
let filename;
|
|
611
471
|
if (filenameMatch) {
|
|
612
472
|
const raw = filenameMatch[1] || filenameMatch[2];
|
|
@@ -617,13 +477,8 @@ export class GangtiseClient {
|
|
|
617
477
|
filename = raw;
|
|
618
478
|
}
|
|
619
479
|
}
|
|
620
|
-
// Stream directly to disk when caller already knows the destination
|
|
621
480
|
if (options?.streamTo) {
|
|
622
481
|
await fs.mkdir(path.dirname(options.streamTo), { recursive: true });
|
|
623
|
-
// Stream into a .part sibling and rename over the target only on success:
|
|
624
|
-
// writing to the target directly would truncate an existing file on the
|
|
625
|
-
// FIRST byte and delete it on failure — a failed re-download (or each
|
|
626
|
-
// withRetry attempt) must never destroy the user's previous good file.
|
|
627
482
|
const partPath = `${options.streamTo}.part`;
|
|
628
483
|
try {
|
|
629
484
|
await pipeline(response.body, createWriteStream(partPath));
|
|
@@ -644,8 +499,6 @@ export class GangtiseClient {
|
|
|
644
499
|
filename,
|
|
645
500
|
};
|
|
646
501
|
}, {
|
|
647
|
-
// Download endpoints carry per-篇 billing too (summary/foreign-report/
|
|
648
|
-
// my-conference at 50/篇) — honor the endpoint's retry policy here as well.
|
|
649
502
|
policy: endpoint.retry ?? "default",
|
|
650
503
|
onRetry: (attempt, error, delay) => {
|
|
651
504
|
if (!isVerbose())
|
|
@@ -34,27 +34,6 @@ export function buildStockPoolStocksBody(options) {
|
|
|
34
34
|
poolIdList: options.poolId?.length ? options.poolId : ["all"],
|
|
35
35
|
};
|
|
36
36
|
}
|
|
37
|
-
/** Parameter keys that REPLACE an indicator's `tradeDate`. The 2026-08-01 EDE
|
|
38
|
-
* revision dropped the root-level `date`, so `--date` now fans out into each
|
|
39
|
-
* indicator's `tradeDate`. A caller-supplied `reportDate` suppresses that
|
|
40
|
-
* injection for its indicator, so the two date fields are never sent together.
|
|
41
|
-
*
|
|
42
|
-
* This is load-bearing, and the server has moved twice on it. A report-period
|
|
43
|
-
* indicator answered a `tradeDate` with an EMPTY result (probed 2026-08-01 on
|
|
44
|
-
* `is_op_rev_mom`), then resolved it to the enclosing report period (2026-08-08,
|
|
45
|
-
* same 33.4903 either way), and since 2026-08-14 REJECTS it outright:
|
|
46
|
-
* `100003 指标 is_op_rev_mom 不支持参数 tradeDate; 缺少必填参数 reportDate`
|
|
47
|
-
* (re-probed 2026-08-15; `reportDate` still returns 33.4903). So a caller-supplied
|
|
48
|
-
* `reportDate` is now the only way to reach those indicators, and suppressing the
|
|
49
|
-
* `tradeDate` injection is what makes it work rather than merely being tidier.
|
|
50
|
-
* Injecting into an indicator that takes no date at all is harmless —
|
|
51
|
-
* `pty_op_scope` still answers normally with a stray `tradeDate` attached.
|
|
52
|
-
*
|
|
53
|
-
* `sDate` is deliberately NOT here. It is an interval START, not a substitute:
|
|
54
|
-
* `qte_vol_intvl` declares `tradeDate` required (the interval END) and `sDate`
|
|
55
|
-
* optional. Treating it as a replacement dropped `--date` and silently moved the
|
|
56
|
-
* interval end — probed 2026-08-02, 茅台 sDate=2024-01-02 returned 2,265,873,849
|
|
57
|
-
* without a tradeDate vs 65,687,435 with tradeDate=2024-01-31, both exit 0. */
|
|
58
37
|
const DATE_PARAM_KEYS = new Set(["tradeDate", "reportDate"]);
|
|
59
38
|
function withQueryDate(groups, codes, date) {
|
|
60
39
|
const merged = new Map();
|
|
@@ -65,11 +44,15 @@ function withQueryDate(groups, codes, date) {
|
|
|
65
44
|
if (!group) {
|
|
66
45
|
merged.set(code, { indicatorCode: code, parameters: [{ paramKey: "tradeDate", paramValue: date }] });
|
|
67
46
|
}
|
|
68
|
-
else if (!group.parameters.some((param) => DATE_PARAM_KEYS.has(param.paramKey))) {
|
|
47
|
+
else if (!group.noQueryDate && !group.parameters.some((param) => DATE_PARAM_KEYS.has(param.paramKey))) {
|
|
69
48
|
group.parameters.push({ paramKey: "tradeDate", paramValue: date });
|
|
70
49
|
}
|
|
71
50
|
}
|
|
72
|
-
return [...merged.values()];
|
|
51
|
+
return [...merged.values()].map(stripMarker);
|
|
52
|
+
}
|
|
53
|
+
function stripMarker(item) {
|
|
54
|
+
const { noQueryDate, ...rest } = item;
|
|
55
|
+
return rest;
|
|
73
56
|
}
|
|
74
57
|
export function buildIndicatorCrossSectionBody(options) {
|
|
75
58
|
return {
|
|
@@ -89,8 +72,7 @@ export function buildIndicatorTimeSeriesBody(options) {
|
|
|
89
72
|
calendarType: options.calendarType,
|
|
90
73
|
currency: options.currency,
|
|
91
74
|
scale: options.scale,
|
|
92
|
-
|
|
93
|
-
indicatorParamList: parseIndicatorParams(options.indicatorParam) ?? [],
|
|
75
|
+
indicatorParamList: (parseIndicatorParams(options.indicatorParam) ?? []).map(stripMarker),
|
|
94
76
|
};
|
|
95
77
|
}
|
|
96
78
|
export function buildIndicatorScreenerBody(options) {
|
|
@@ -98,16 +80,6 @@ export function buildIndicatorScreenerBody(options) {
|
|
|
98
80
|
return {
|
|
99
81
|
universe: maybeArray(options.security),
|
|
100
82
|
expression: options.expression,
|
|
101
|
-
// Every indicator gets a date, including ones whose parameterList is empty.
|
|
102
|
-
// Harmless there (a parameterless indicator answers normally with a stray
|
|
103
|
-
// tradeDate), and it keeps one rule for the whole list rather than a
|
|
104
|
-
// per-indicator exception.
|
|
105
|
-
//
|
|
106
|
-
// It also used to be load-bearing: through 2026-08-02 the screener DROPPED
|
|
107
|
-
// any indicator sent with `parameters: []`, so the official doc's own
|
|
108
|
-
// `F3 contains '酒'` example could not work as written. Re-probed 2026-08-03:
|
|
109
|
-
// fixed server-side. Kept anyway — unconditional is simpler than conditional,
|
|
110
|
-
// and it survives a rollback.
|
|
111
83
|
indicatorList: indicators.map((indicator) => (indicator.parameters.some((param) => DATE_PARAM_KEYS.has(param.paramKey))
|
|
112
84
|
? indicator
|
|
113
85
|
: { ...indicator, parameters: [...indicator.parameters, { paramKey: "tradeDate", paramValue: options.date }] })),
|
|
@@ -3,34 +3,18 @@ import { extname } from "node:path";
|
|
|
3
3
|
import { DownloadError } from "./errors.js";
|
|
4
4
|
import { saveOutputIfNeeded } from "./output.js";
|
|
5
5
|
import { lookupTitleCache, readTitleCache, TITLE_LOOKUP_SIZE } from "./titleCache.js";
|
|
6
|
-
/** Replace filesystem-unsafe characters (path separators, wildcards, and control
|
|
7
|
-
* characters / NUL) with `_` so a title or a server-supplied filename can't create
|
|
8
|
-
* stray subdirectories, escape the intended output path, or break fs.writeFile.
|
|
9
|
-
* Shared by title-based naming and the download fallback. */
|
|
10
6
|
function sanitizeFilename(name) {
|
|
11
7
|
return name.replace(/[/\\:*?"<>|\u0000-\u001f]/g, "_");
|
|
12
8
|
}
|
|
13
|
-
/** Keep auto-derived filenames under 200 UTF-8 bytes, preserving the extension.
|
|
14
|
-
* ext4 caps directory entries at 255 bytes — long Chinese titles hit that at
|
|
15
|
-
* ~85 chars (3 bytes each) and fs.writeFile throws ENAMETOOLONG after the body
|
|
16
|
-
* has already been downloaded. */
|
|
17
9
|
function truncateFilename(name, maxBytes = 200) {
|
|
18
10
|
if (Buffer.byteLength(name, "utf8") <= maxBytes)
|
|
19
11
|
return name;
|
|
20
12
|
const ext = extname(name);
|
|
21
|
-
// Trim by code point, not UTF-16 unit: a cut inside a surrogate pair would put
|
|
22
|
-
// a lone surrogate in the name, which reaches the filesystem as U+FFFD (�).
|
|
23
13
|
const stem = [...name.slice(0, name.length - ext.length)];
|
|
24
14
|
while (stem.length > 1 && Buffer.byteLength(stem.join("") + ext, "utf8") > maxBytes)
|
|
25
15
|
stem.pop();
|
|
26
16
|
return stem.join("") + ext;
|
|
27
17
|
}
|
|
28
|
-
/** Pick a non-existing path by suffixing -1, -2, … before the extension, so batch
|
|
29
|
-
* downloads whose titles collide ("2025年第一季度报告" from several companies) don't
|
|
30
|
-
* silently overwrite each other. Only auto-derived names go through this — an
|
|
31
|
-
* explicit --output path keeps plain overwrite semantics. Throws instead of
|
|
32
|
-
* falling back to the original path once the suffixes run out: returning `p`
|
|
33
|
-
* there would silently overwrite the very first file. */
|
|
34
18
|
export async function uniquePath(p) {
|
|
35
19
|
const exists = (f) => fs.access(f).then(() => true, () => false);
|
|
36
20
|
if (!(await exists(p)))
|
|
@@ -93,7 +77,6 @@ export async function resolveTitle(client, result, listEndpoint, idField, idValu
|
|
|
93
77
|
return buildFilename(cached);
|
|
94
78
|
}
|
|
95
79
|
catch {
|
|
96
|
-
// Ignore corrupt cache data and fall back to the list endpoint.
|
|
97
80
|
}
|
|
98
81
|
try {
|
|
99
82
|
const resp = await client.call(listEndpoint, { from: 0, size: TITLE_LOOKUP_SIZE });
|
|
@@ -115,13 +98,8 @@ async function downloadUrlTo(url, outputPath) {
|
|
|
115
98
|
const { request } = await import("undici");
|
|
116
99
|
const { getDispatcher, logTiming, withRetry } = await import("./transport.js");
|
|
117
100
|
const { loadConfig } = await import("./config.js");
|
|
118
|
-
// Through the transport layer instead of a bare global fetch: the configured
|
|
119
|
-
// timeout applies (a slow-drip CDN can no longer hang the CLI indefinitely),
|
|
120
|
-
// network-level failures retry, and --verbose logs the request.
|
|
121
101
|
const timeoutMs = loadConfig().timeoutMs;
|
|
122
102
|
await fs.mkdir(dirname(outputPath), { recursive: true });
|
|
123
|
-
// Signed URLs carry credentials in the query string — verbose logs must strip
|
|
124
|
-
// search/hash so signatures never land in terminal/CI logs.
|
|
125
103
|
const redactUrl = (u) => {
|
|
126
104
|
try {
|
|
127
105
|
const parsed = new URL(u);
|
|
@@ -131,7 +109,6 @@ async function downloadUrlTo(url, outputPath) {
|
|
|
131
109
|
return "signed-url";
|
|
132
110
|
}
|
|
133
111
|
};
|
|
134
|
-
// .part + rename so a failed follow-download never destroys an existing file.
|
|
135
112
|
const partPath = `${outputPath}.part`;
|
|
136
113
|
await withRetry(async () => {
|
|
137
114
|
const startedAt = Date.now();
|
|
@@ -139,18 +116,11 @@ async function downloadUrlTo(url, outputPath) {
|
|
|
139
116
|
method: "GET",
|
|
140
117
|
headersTimeout: timeoutMs,
|
|
141
118
|
bodyTimeout: timeoutMs,
|
|
142
|
-
// headers/body timeouts are IDLE timeouts — a stream trickling one byte
|
|
143
|
-
// per interval resets them forever. A generous total deadline (10× the
|
|
144
|
-
// per-request timeout) bounds the whole transfer without killing large
|
|
145
|
-
// legitimate downloads.
|
|
146
119
|
signal: AbortSignal.timeout(timeoutMs * 10),
|
|
147
120
|
dispatcher: getDispatcher(),
|
|
148
121
|
};
|
|
149
122
|
let currentUrl = url;
|
|
150
123
|
let response = await request(currentUrl, requestOptions);
|
|
151
|
-
// undici does not follow redirects (the old global fetch did): a signed URL
|
|
152
|
-
// may 302 to the real CDN object. Follow a bounded number of hops; no
|
|
153
|
-
// Authorization header is involved on this path, so nothing leaks cross-origin.
|
|
154
124
|
for (let hops = 0; hops < 3 && response.statusCode >= 300 && response.statusCode < 400; hops++) {
|
|
155
125
|
const locationHeader = response.headers.location;
|
|
156
126
|
const location = Array.isArray(locationHeader) ? locationHeader[0] : locationHeader;
|
|
@@ -166,7 +136,6 @@ async function downloadUrlTo(url, outputPath) {
|
|
|
166
136
|
}
|
|
167
137
|
if (response.statusCode >= 400) {
|
|
168
138
|
await response.body.text().catch(() => { });
|
|
169
|
-
// DownloadError (not retried): signed URLs expire — replaying a 403 is useless.
|
|
170
139
|
throw new DownloadError(`Failed to fetch download URL (HTTP ${response.statusCode})`);
|
|
171
140
|
}
|
|
172
141
|
try {
|
|
@@ -182,7 +151,6 @@ async function downloadUrlTo(url, outputPath) {
|
|
|
182
151
|
await fs.rename(partPath, outputPath);
|
|
183
152
|
}
|
|
184
153
|
catch (error) {
|
|
185
|
-
// e.g. outputPath turned out to be a directory — don't leave the .part behind.
|
|
186
154
|
await fs.unlink(partPath).catch(() => { });
|
|
187
155
|
throw error;
|
|
188
156
|
}
|
|
@@ -192,16 +160,12 @@ export async function saveDownloadResult(result, fallbackName, output) {
|
|
|
192
160
|
throw new DownloadError("Unexpected download response");
|
|
193
161
|
}
|
|
194
162
|
const file = result;
|
|
195
|
-
// The fallback prefix embeds a user-supplied id (e.g. --report-id): sanitize it
|
|
196
|
-
// like server-provided names so an auto filename never interprets input as a path.
|
|
197
163
|
const safeFallback = sanitizeFilename(fallbackName);
|
|
198
164
|
if (typeof file.savedPath === "string") {
|
|
199
165
|
process.stdout.write(`${file.savedPath}\n`);
|
|
200
166
|
return;
|
|
201
167
|
}
|
|
202
168
|
if (file.data instanceof Uint8Array) {
|
|
203
|
-
// Sanitize the server-provided filename so a Content-Disposition value with
|
|
204
|
-
// / or : can't write outside the intended path (same rule as buildFilename).
|
|
205
169
|
const autoName = (file.filename ? sanitizeFilename(file.filename) : undefined) ?? (safeFallback + extFromContentType(file.contentType));
|
|
206
170
|
const outputPath = output ?? await uniquePath(truncateFilename(autoName));
|
|
207
171
|
await saveOutputIfNeeded(file.data, outputPath);
|
|
@@ -216,9 +180,6 @@ export async function saveDownloadResult(result, fallbackName, output) {
|
|
|
216
180
|
}
|
|
217
181
|
if (typeof file.url === "string") {
|
|
218
182
|
if (output) {
|
|
219
|
-
// The server handed us a (typically signed, short-lived) URL instead of the
|
|
220
|
-
// bytes. The user asked for a file — follow the URL and stream the content
|
|
221
|
-
// to disk instead of writing the URL string into a fake .pdf.
|
|
222
183
|
await downloadUrlTo(file.url, output);
|
|
223
184
|
process.stdout.write(`${output}\n`);
|
|
224
185
|
return;
|