gangtise-openapi-cli 0.11.1 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +123 -7
- package/dist/src/cli.js +181 -31
- package/dist/src/core/asyncContent.js +13 -4
- package/dist/src/core/client.js +208 -125
- package/dist/src/core/commandBodies.js +1 -0
- package/dist/src/core/download.js +4 -0
- package/dist/src/core/endpoints.js +104 -8
- package/dist/src/core/output.js +64 -0
- package/dist/src/core/printer.js +7 -3
- package/dist/src/core/quoteSharding.js +81 -0
- package/dist/src/core/titleCache.js +58 -10
- package/dist/src/core/transport.js +91 -0
- package/dist/src/version.js +1 -1
- package/package.json +1 -1
package/dist/src/core/client.js
CHANGED
|
@@ -1,21 +1,35 @@
|
|
|
1
|
+
import { createWriteStream } from "node:fs";
|
|
2
|
+
import fs from "node:fs/promises";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { pipeline } from "node:stream/promises";
|
|
1
5
|
import { request } from "undici";
|
|
2
6
|
import { isTokenCacheValid, normalizeToken, readTokenCache, requireAccessCredentials, writeTokenCache } from "./auth.js";
|
|
3
7
|
import { ApiError, ValidationError } from "./errors.js";
|
|
4
8
|
import { ENDPOINTS } from "./endpoints.js";
|
|
5
9
|
import { getLookupData } from "./lookupData/index.js";
|
|
10
|
+
import { getDispatcher, isVerbose, logTiming, markRetryable, runWithConcurrency, withRetry } from "./transport.js";
|
|
11
|
+
const PAGINATION_CONCURRENCY = Number(process.env.GANGTISE_PAGE_CONCURRENCY ?? 5) || 5;
|
|
12
|
+
const AUTH_RETRY_CODES = new Set(["8000014", "8000015"]);
|
|
6
13
|
export class GangtiseClient {
|
|
7
14
|
config;
|
|
8
15
|
refreshPromise = null;
|
|
16
|
+
memoCache = null;
|
|
9
17
|
constructor(config) {
|
|
10
18
|
this.config = config;
|
|
11
19
|
}
|
|
12
|
-
async getAuthorizationHeader() {
|
|
13
|
-
if (this.config.token) {
|
|
20
|
+
async getAuthorizationHeader(forceRefresh = false) {
|
|
21
|
+
if (this.config.token && !forceRefresh) {
|
|
14
22
|
return normalizeToken(this.config.token);
|
|
15
23
|
}
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
24
|
+
if (!forceRefresh) {
|
|
25
|
+
if (isTokenCacheValid(this.memoCache)) {
|
|
26
|
+
return normalizeToken(this.memoCache.accessToken);
|
|
27
|
+
}
|
|
28
|
+
const cache = await readTokenCache(this.config.tokenCachePath);
|
|
29
|
+
if (isTokenCacheValid(cache)) {
|
|
30
|
+
this.memoCache = cache;
|
|
31
|
+
return normalizeToken(cache.accessToken);
|
|
32
|
+
}
|
|
19
33
|
}
|
|
20
34
|
if (!this.refreshPromise) {
|
|
21
35
|
this.refreshPromise = this.doTokenRefresh().finally(() => { this.refreshPromise = null; });
|
|
@@ -30,11 +44,9 @@ export class GangtiseClient {
|
|
|
30
44
|
}, false);
|
|
31
45
|
const accessToken = normalizeToken(envelope.accessToken);
|
|
32
46
|
const expiresAt = Math.floor(Date.now() / 1000) + envelope.expiresIn;
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
expiresAt,
|
|
37
|
-
});
|
|
47
|
+
const cache = { ...envelope, accessToken, expiresAt };
|
|
48
|
+
this.memoCache = cache;
|
|
49
|
+
await writeTokenCache(this.config.tokenCachePath, cache);
|
|
38
50
|
return accessToken;
|
|
39
51
|
}
|
|
40
52
|
isEnvelope(parsed) {
|
|
@@ -100,55 +112,73 @@ export class GangtiseClient {
|
|
|
100
112
|
const startFrom = typeof initialBody.from === 'number' && Number.isFinite(initialBody.from) ? initialBody.from : 0;
|
|
101
113
|
const requestedSize = typeof initialBody.size === 'number' && Number.isFinite(initialBody.size) ? initialBody.size : undefined;
|
|
102
114
|
const maxPageSize = endpoint.pagination?.maxPageSize ?? requestedSize ?? 20;
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
115
|
+
// First page: serial — we need total before deciding how many more requests to fan out.
|
|
116
|
+
const firstPageSize = requestedSize === undefined ? maxPageSize : Math.min(maxPageSize, requestedSize);
|
|
117
|
+
const firstPage = await this.requestJson(endpoint, {
|
|
118
|
+
...initialBody,
|
|
119
|
+
from: startFrom,
|
|
120
|
+
size: firstPageSize,
|
|
121
|
+
});
|
|
122
|
+
if (!this.isPaginatedListResponse(firstPage))
|
|
123
|
+
return firstPage;
|
|
124
|
+
const total = firstPage.total;
|
|
125
|
+
const collected = [...firstPage.list];
|
|
126
|
+
// Last page reached on first request
|
|
127
|
+
if (firstPage.list.length < firstPageSize) {
|
|
128
|
+
return {
|
|
129
|
+
...firstPage,
|
|
130
|
+
total,
|
|
131
|
+
list: requestedSize === undefined ? collected : collected.slice(0, requestedSize),
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
const available = Math.max(total - startFrom, 0);
|
|
135
|
+
const target = requestedSize === undefined ? available : Math.min(requestedSize, available);
|
|
136
|
+
if (collected.length >= target) {
|
|
137
|
+
return {
|
|
138
|
+
...firstPage,
|
|
139
|
+
total,
|
|
140
|
+
list: requestedSize === undefined ? collected : collected.slice(0, requestedSize),
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
const pageRequests = [];
|
|
144
|
+
let nextFrom = startFrom + firstPage.list.length;
|
|
145
|
+
const endFrom = startFrom + target;
|
|
146
|
+
while (nextFrom < endFrom) {
|
|
147
|
+
const remaining = endFrom - nextFrom;
|
|
148
|
+
const size = Math.min(maxPageSize, remaining);
|
|
149
|
+
pageRequests.push({ from: nextFrom, size });
|
|
150
|
+
nextFrom += size;
|
|
151
|
+
}
|
|
107
152
|
const MAX_PAGES = 1000;
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
}
|
|
153
|
+
if (pageRequests.length + 1 > MAX_PAGES) {
|
|
154
|
+
pageRequests.length = MAX_PAGES - 1;
|
|
155
|
+
}
|
|
156
|
+
let unexpectedShape = false;
|
|
157
|
+
let totalDrift = false;
|
|
158
|
+
const pages = await runWithConcurrency(pageRequests, PAGINATION_CONCURRENCY, async (req) => {
|
|
115
159
|
const page = await this.requestJson(endpoint, {
|
|
116
160
|
...initialBody,
|
|
117
|
-
from:
|
|
118
|
-
size:
|
|
161
|
+
from: req.from,
|
|
162
|
+
size: req.size,
|
|
119
163
|
});
|
|
120
164
|
if (!this.isPaginatedListResponse(page)) {
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
}
|
|
124
|
-
return {
|
|
125
|
-
...firstPage,
|
|
126
|
-
total,
|
|
127
|
-
list: requestedSize === undefined ? collected : collected.slice(0, requestedSize),
|
|
128
|
-
};
|
|
129
|
-
}
|
|
130
|
-
if (!firstPage) {
|
|
131
|
-
firstPage = page;
|
|
132
|
-
total = page.total;
|
|
133
|
-
}
|
|
134
|
-
if (page.list.length === 0) {
|
|
135
|
-
break;
|
|
136
|
-
}
|
|
137
|
-
collected.push(...page.list);
|
|
138
|
-
nextFrom += page.list.length;
|
|
139
|
-
const available = total === undefined ? undefined : Math.max(total - startFrom, 0);
|
|
140
|
-
if (requestedSize !== undefined && collected.length >= requestedSize) {
|
|
141
|
-
break;
|
|
142
|
-
}
|
|
143
|
-
if (available !== undefined && collected.length >= available) {
|
|
144
|
-
break;
|
|
145
|
-
}
|
|
146
|
-
if (page.list.length < remaining) {
|
|
147
|
-
break;
|
|
165
|
+
unexpectedShape = true;
|
|
166
|
+
return [];
|
|
148
167
|
}
|
|
168
|
+
if (page.total !== total)
|
|
169
|
+
totalDrift = true;
|
|
170
|
+
return page.list;
|
|
171
|
+
});
|
|
172
|
+
for (const list of pages) {
|
|
173
|
+
if (list.length === 0)
|
|
174
|
+
continue;
|
|
175
|
+
collected.push(...list);
|
|
176
|
+
}
|
|
177
|
+
if (unexpectedShape && isVerbose()) {
|
|
178
|
+
process.stderr.write(`[gangtise] warning: a page response had unexpected shape; results may be incomplete\n`);
|
|
149
179
|
}
|
|
150
|
-
if (
|
|
151
|
-
|
|
180
|
+
if (totalDrift && isVerbose()) {
|
|
181
|
+
process.stderr.write(`[gangtise] warning: 'total' changed across pages (data shifted during fetch)\n`);
|
|
152
182
|
}
|
|
153
183
|
return {
|
|
154
184
|
...firstPage,
|
|
@@ -168,100 +198,153 @@ export class GangtiseClient {
|
|
|
168
198
|
if (endpoint.path.startsWith('/guide/')) {
|
|
169
199
|
return this.readLocalLookup(endpoint);
|
|
170
200
|
}
|
|
171
|
-
const
|
|
172
|
-
'content-type': 'application/json',
|
|
173
|
-
};
|
|
174
|
-
if (useAuth) {
|
|
175
|
-
headers.Authorization = await this.getAuthorizationHeader();
|
|
176
|
-
}
|
|
177
|
-
const response = await request(new URL(endpoint.path, this.config.baseUrl), {
|
|
178
|
-
method: endpoint.method,
|
|
179
|
-
headers,
|
|
180
|
-
body: endpoint.method === 'GET' ? undefined : JSON.stringify(body ?? {}),
|
|
181
|
-
headersTimeout: this.config.timeoutMs,
|
|
182
|
-
bodyTimeout: this.config.timeoutMs,
|
|
183
|
-
});
|
|
184
|
-
const text = await response.body.text();
|
|
185
|
-
let parsed;
|
|
186
|
-
try {
|
|
187
|
-
parsed = JSON.parse(text);
|
|
188
|
-
}
|
|
189
|
-
catch {
|
|
190
|
-
const message = response.statusCode >= 400
|
|
191
|
-
? `API request failed (HTTP ${response.statusCode})`
|
|
192
|
-
: 'Failed to parse API response';
|
|
193
|
-
throw new ApiError(message, undefined, response.statusCode, text.slice(0, 500));
|
|
194
|
-
}
|
|
195
|
-
if (response.statusCode >= 400) {
|
|
196
|
-
this.throwHttpError(parsed, response.statusCode);
|
|
197
|
-
}
|
|
198
|
-
return this.unwrapEnvelope(parsed, response.statusCode);
|
|
199
|
-
}
|
|
200
|
-
async download(endpoint, query) {
|
|
201
|
-
const authorization = await this.getAuthorizationHeader();
|
|
201
|
+
const dispatcher = getDispatcher();
|
|
202
202
|
const url = new URL(endpoint.path, this.config.baseUrl);
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
Authorization
|
|
210
|
-
}
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
203
|
+
let authRetried = false;
|
|
204
|
+
return withRetry(async () => {
|
|
205
|
+
const headers = {
|
|
206
|
+
'content-type': 'application/json',
|
|
207
|
+
};
|
|
208
|
+
if (useAuth) {
|
|
209
|
+
headers.Authorization = await this.getAuthorizationHeader();
|
|
210
|
+
}
|
|
211
|
+
const startedAt = Date.now();
|
|
212
|
+
const response = await request(url, {
|
|
213
|
+
method: endpoint.method,
|
|
214
|
+
headers,
|
|
215
|
+
body: endpoint.method === 'GET' ? undefined : JSON.stringify(body ?? {}),
|
|
216
|
+
headersTimeout: this.config.timeoutMs,
|
|
217
|
+
bodyTimeout: this.config.timeoutMs,
|
|
218
|
+
dispatcher,
|
|
219
|
+
});
|
|
216
220
|
const text = await response.body.text();
|
|
221
|
+
logTiming(`${endpoint.method} ${endpoint.path}`, Date.now() - startedAt, `${response.statusCode}, ${text.length}B`);
|
|
217
222
|
let parsed;
|
|
218
223
|
try {
|
|
219
224
|
parsed = JSON.parse(text);
|
|
220
225
|
}
|
|
221
226
|
catch {
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
227
|
+
const message = response.statusCode >= 400
|
|
228
|
+
? `API request failed (HTTP ${response.statusCode})`
|
|
229
|
+
: 'Failed to parse API response';
|
|
230
|
+
throw new ApiError(message, undefined, response.statusCode, text.slice(0, 500));
|
|
226
231
|
}
|
|
227
232
|
if (response.statusCode >= 400) {
|
|
228
233
|
this.throwHttpError(parsed, response.statusCode);
|
|
229
234
|
}
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
235
|
+
try {
|
|
236
|
+
return this.unwrapEnvelope(parsed, response.statusCode);
|
|
237
|
+
}
|
|
238
|
+
catch (error) {
|
|
239
|
+
// Auto-recover from auth errors by forcing a token refresh once.
|
|
240
|
+
if (useAuth
|
|
241
|
+
&& !authRetried
|
|
242
|
+
&& error instanceof ApiError
|
|
243
|
+
&& error.code
|
|
244
|
+
&& AUTH_RETRY_CODES.has(error.code)
|
|
245
|
+
&& (this.config.accessKey && this.config.secretKey)) {
|
|
246
|
+
authRetried = true;
|
|
247
|
+
this.memoCache = null;
|
|
248
|
+
await this.getAuthorizationHeader(true);
|
|
249
|
+
throw markRetryable(new ApiError(error.message, error.code, error.statusCode, error.details));
|
|
250
|
+
}
|
|
251
|
+
throw error;
|
|
252
|
+
}
|
|
253
|
+
}, {
|
|
254
|
+
onRetry: (attempt, error, delay) => {
|
|
255
|
+
if (!isVerbose())
|
|
256
|
+
return;
|
|
257
|
+
const msg = error instanceof Error ? error.message : String(error);
|
|
258
|
+
process.stderr.write(`[gangtise] retry ${attempt} after ${delay.toFixed(0)}ms: ${msg.slice(0, 120)}\n`);
|
|
259
|
+
},
|
|
260
|
+
});
|
|
261
|
+
}
|
|
262
|
+
async download(endpoint, query, options) {
|
|
263
|
+
const dispatcher = getDispatcher();
|
|
264
|
+
const url = new URL(endpoint.path, this.config.baseUrl);
|
|
265
|
+
Object.entries(query).forEach(([key, value]) => {
|
|
266
|
+
url.searchParams.set(key, String(value));
|
|
267
|
+
});
|
|
268
|
+
return withRetry(async () => {
|
|
269
|
+
const authorization = await this.getAuthorizationHeader();
|
|
270
|
+
const startedAt = Date.now();
|
|
271
|
+
const response = await request(url, {
|
|
272
|
+
method: endpoint.method,
|
|
273
|
+
headers: { Authorization: authorization },
|
|
274
|
+
headersTimeout: this.config.timeoutMs,
|
|
275
|
+
bodyTimeout: this.config.timeoutMs,
|
|
276
|
+
dispatcher,
|
|
277
|
+
});
|
|
278
|
+
const contentType = Array.isArray(response.headers['content-type']) ? response.headers['content-type'][0] : response.headers['content-type'];
|
|
279
|
+
if (contentType?.includes('application/json')) {
|
|
280
|
+
const text = await response.body.text();
|
|
281
|
+
logTiming(`GET ${endpoint.path} (json)`, Date.now() - startedAt, `${response.statusCode}, ${text.length}B`);
|
|
282
|
+
let parsed;
|
|
283
|
+
try {
|
|
284
|
+
parsed = JSON.parse(text);
|
|
285
|
+
}
|
|
286
|
+
catch {
|
|
287
|
+
if (response.statusCode >= 400) {
|
|
288
|
+
throw new ApiError('Download failed', undefined, response.statusCode, text);
|
|
289
|
+
}
|
|
290
|
+
return { text, contentType };
|
|
291
|
+
}
|
|
292
|
+
if (response.statusCode >= 400) {
|
|
293
|
+
this.throwHttpError(parsed, response.statusCode);
|
|
294
|
+
}
|
|
295
|
+
const data = this.unwrapEnvelope(parsed, response.statusCode);
|
|
296
|
+
if (data && typeof data === 'object' && 'url' in data && typeof data.url === 'string') {
|
|
297
|
+
return { url: String(data.url), contentType };
|
|
298
|
+
}
|
|
299
|
+
return { text: JSON.stringify(data, null, 2), contentType };
|
|
300
|
+
}
|
|
301
|
+
if (contentType?.includes('text/plain') || contentType?.includes('text/html')) {
|
|
302
|
+
const text = await response.body.text();
|
|
303
|
+
logTiming(`GET ${endpoint.path} (text)`, Date.now() - startedAt, `${response.statusCode}, ${text.length}B`);
|
|
304
|
+
if (response.statusCode >= 400) {
|
|
305
|
+
throw new ApiError('Download failed', undefined, response.statusCode, text);
|
|
306
|
+
}
|
|
307
|
+
return { text, contentType };
|
|
233
308
|
}
|
|
234
|
-
return { text: JSON.stringify(data, null, 2), contentType };
|
|
235
|
-
}
|
|
236
|
-
if (contentType?.includes('text/plain') || contentType?.includes('text/html')) {
|
|
237
|
-
const text = await response.body.text();
|
|
238
309
|
if (response.statusCode >= 400) {
|
|
310
|
+
const text = await response.body.text();
|
|
239
311
|
throw new ApiError('Download failed', undefined, response.statusCode, text);
|
|
240
312
|
}
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
313
|
+
const contentDisposition = response.headers['content-disposition'];
|
|
314
|
+
const filenameMatch = Array.isArray(contentDisposition)
|
|
315
|
+
? contentDisposition[0]?.match(/filename\*=UTF-8''([^;]+)|filename="?([^";]+)"?/i)
|
|
316
|
+
: contentDisposition?.match(/filename\*=UTF-8''([^;]+)|filename="?([^";]+)"?/i);
|
|
317
|
+
const filename = filenameMatch ? decodeURIComponent(filenameMatch[1] || filenameMatch[2]) : undefined;
|
|
318
|
+
// Stream directly to disk when caller already knows the destination
|
|
319
|
+
if (options?.streamTo) {
|
|
320
|
+
await fs.mkdir(path.dirname(options.streamTo), { recursive: true });
|
|
321
|
+
await pipeline(response.body, createWriteStream(options.streamTo));
|
|
322
|
+
logTiming(`GET ${endpoint.path} (stream)`, Date.now() - startedAt, `${response.statusCode}`);
|
|
323
|
+
return { contentType, filename, savedPath: options.streamTo };
|
|
324
|
+
}
|
|
325
|
+
const buffer = await response.body.arrayBuffer();
|
|
326
|
+
logTiming(`GET ${endpoint.path} (binary)`, Date.now() - startedAt, `${response.statusCode}, ${buffer.byteLength}B`);
|
|
327
|
+
return {
|
|
328
|
+
data: new Uint8Array(buffer),
|
|
329
|
+
contentType,
|
|
330
|
+
filename,
|
|
331
|
+
};
|
|
332
|
+
}, {
|
|
333
|
+
onRetry: (attempt, error, delay) => {
|
|
334
|
+
if (!isVerbose())
|
|
335
|
+
return;
|
|
336
|
+
const msg = error instanceof Error ? error.message : String(error);
|
|
337
|
+
process.stderr.write(`[gangtise] download retry ${attempt} after ${delay.toFixed(0)}ms: ${msg.slice(0, 120)}\n`);
|
|
338
|
+
},
|
|
339
|
+
});
|
|
257
340
|
}
|
|
258
|
-
async call(endpointKey, body, query) {
|
|
341
|
+
async call(endpointKey, body, query, options) {
|
|
259
342
|
const endpoint = ENDPOINTS[endpointKey];
|
|
260
343
|
if (!endpoint) {
|
|
261
344
|
throw new ApiError(`Unknown endpoint key: ${endpointKey}`);
|
|
262
345
|
}
|
|
263
346
|
if (endpoint.kind === 'download') {
|
|
264
|
-
return this.download(endpoint, query ?? {});
|
|
347
|
+
return this.download(endpoint, query ?? {}, options);
|
|
265
348
|
}
|
|
266
349
|
if (endpoint.kind === 'json' && endpoint.pagination?.enabled) {
|
|
267
350
|
return this.requestPaginated(endpoint, body);
|
|
@@ -15,6 +15,7 @@ export function buildWechatMessageListBody(options) {
|
|
|
15
15
|
startTime: options.startTime,
|
|
16
16
|
endTime: options.endTime,
|
|
17
17
|
keyword: options.keyword,
|
|
18
|
+
securityList: maybeArray(options.security),
|
|
18
19
|
wechatGroupIdList: maybeArray(options.wechatGroupId),
|
|
19
20
|
industryIdList: maybeArray(options.industry),
|
|
20
21
|
categoryList: maybeArray(options.category),
|
|
@@ -71,6 +71,10 @@ export async function saveDownloadResult(result, fallbackName, output) {
|
|
|
71
71
|
throw new DownloadError("Unexpected download response");
|
|
72
72
|
}
|
|
73
73
|
const file = result;
|
|
74
|
+
if (typeof file.savedPath === "string") {
|
|
75
|
+
process.stdout.write(`${file.savedPath}\n`);
|
|
76
|
+
return;
|
|
77
|
+
}
|
|
74
78
|
if (file.data instanceof Uint8Array) {
|
|
75
79
|
const outputPath = output ?? file.filename ?? (fallbackName + extFromContentType(file.contentType));
|
|
76
80
|
await saveOutputIfNeeded(file.data, outputPath);
|
|
@@ -70,7 +70,7 @@ export const ENDPOINTS = {
|
|
|
70
70
|
method: "POST",
|
|
71
71
|
path: "/application/open-insight/chief-opinion/getList",
|
|
72
72
|
kind: "json",
|
|
73
|
-
description: "List chief opinions",
|
|
73
|
+
description: "List domestic institution chief opinions",
|
|
74
74
|
pagination: { enabled: true, maxPageSize: 50 },
|
|
75
75
|
},
|
|
76
76
|
"insight.summary.list": {
|
|
@@ -155,7 +155,7 @@ export const ENDPOINTS = {
|
|
|
155
155
|
method: "POST",
|
|
156
156
|
path: "/application/open-insight/announcement/getList",
|
|
157
157
|
kind: "json",
|
|
158
|
-
description: "List announcements",
|
|
158
|
+
description: "List A-share announcements",
|
|
159
159
|
pagination: { enabled: true, maxPageSize: 50 },
|
|
160
160
|
},
|
|
161
161
|
"insight.announcement.download": {
|
|
@@ -163,7 +163,53 @@ export const ENDPOINTS = {
|
|
|
163
163
|
method: "GET",
|
|
164
164
|
path: "/application/open-insight/announcement/download/file",
|
|
165
165
|
kind: "download",
|
|
166
|
-
description: "Download announcement file",
|
|
166
|
+
description: "Download A-share announcement file",
|
|
167
|
+
},
|
|
168
|
+
"insight.announcement-hk.list": {
|
|
169
|
+
key: "insight.announcement-hk.list",
|
|
170
|
+
method: "POST",
|
|
171
|
+
path: "/application/open-insight/announcement-hk/getList",
|
|
172
|
+
kind: "json",
|
|
173
|
+
description: "List HK announcements",
|
|
174
|
+
pagination: { enabled: true, maxPageSize: 50 },
|
|
175
|
+
},
|
|
176
|
+
"insight.announcement-hk.download": {
|
|
177
|
+
key: "insight.announcement-hk.download",
|
|
178
|
+
method: "GET",
|
|
179
|
+
path: "/application/open-insight/announcement-hk/download/file",
|
|
180
|
+
kind: "download",
|
|
181
|
+
description: "Download HK announcement file",
|
|
182
|
+
},
|
|
183
|
+
"insight.foreign-opinion.list": {
|
|
184
|
+
key: "insight.foreign-opinion.list",
|
|
185
|
+
method: "POST",
|
|
186
|
+
path: "/application/open-insight/foreign-opinion/getList",
|
|
187
|
+
kind: "json",
|
|
188
|
+
description: "List foreign institution opinions",
|
|
189
|
+
pagination: { enabled: true, maxPageSize: 50 },
|
|
190
|
+
},
|
|
191
|
+
"insight.independent-opinion.list": {
|
|
192
|
+
key: "insight.independent-opinion.list",
|
|
193
|
+
method: "POST",
|
|
194
|
+
path: "/application/open-insight/independent-opinion/getList",
|
|
195
|
+
kind: "json",
|
|
196
|
+
description: "List foreign independent analyst opinions",
|
|
197
|
+
pagination: { enabled: true, maxPageSize: 50 },
|
|
198
|
+
},
|
|
199
|
+
"insight.independent-opinion.download": {
|
|
200
|
+
key: "insight.independent-opinion.download",
|
|
201
|
+
method: "GET",
|
|
202
|
+
path: "/application/open-insight/independent-opinion/download/file",
|
|
203
|
+
kind: "download",
|
|
204
|
+
description: "Download foreign independent opinion file",
|
|
205
|
+
},
|
|
206
|
+
// ─── reference ───
|
|
207
|
+
"reference.securities-search": {
|
|
208
|
+
key: "reference.securities-search",
|
|
209
|
+
method: "POST",
|
|
210
|
+
path: "/application/open-reference/securities/search",
|
|
211
|
+
kind: "json",
|
|
212
|
+
description: "Search GTS codes (securities)",
|
|
167
213
|
},
|
|
168
214
|
// ─── quote ───
|
|
169
215
|
"quote.day-kline": {
|
|
@@ -200,35 +246,56 @@ export const ENDPOINTS = {
|
|
|
200
246
|
method: "POST",
|
|
201
247
|
path: "/application/open-fundamental/financial-report/income-statement/accumulated",
|
|
202
248
|
kind: "json",
|
|
203
|
-
description: "Query income statement (accumulated)",
|
|
249
|
+
description: "Query A-share income statement (accumulated)",
|
|
204
250
|
},
|
|
205
251
|
"fundamental.income-statement-quarterly": {
|
|
206
252
|
key: "fundamental.income-statement-quarterly",
|
|
207
253
|
method: "POST",
|
|
208
254
|
path: "/application/open-fundamental/financial-report/income-statement/quarterly",
|
|
209
255
|
kind: "json",
|
|
210
|
-
description: "Query income statement (quarterly)",
|
|
256
|
+
description: "Query A-share income statement (quarterly)",
|
|
211
257
|
},
|
|
212
258
|
"fundamental.balance-sheet": {
|
|
213
259
|
key: "fundamental.balance-sheet",
|
|
214
260
|
method: "POST",
|
|
215
261
|
path: "/application/open-fundamental/financial-report/balance-sheet/accumulated",
|
|
216
262
|
kind: "json",
|
|
217
|
-
description: "Query balance sheet (accumulated)",
|
|
263
|
+
description: "Query A-share balance sheet (accumulated)",
|
|
218
264
|
},
|
|
219
265
|
"fundamental.cash-flow": {
|
|
220
266
|
key: "fundamental.cash-flow",
|
|
221
267
|
method: "POST",
|
|
222
268
|
path: "/application/open-fundamental/financial-report/cash-flow-statement/accumulated",
|
|
223
269
|
kind: "json",
|
|
224
|
-
description: "Query cash flow statement (accumulated)",
|
|
270
|
+
description: "Query A-share cash flow statement (accumulated)",
|
|
225
271
|
},
|
|
226
272
|
"fundamental.cash-flow-quarterly": {
|
|
227
273
|
key: "fundamental.cash-flow-quarterly",
|
|
228
274
|
method: "POST",
|
|
229
275
|
path: "/application/open-fundamental/financial-report/cash-flow-statement/quarterly",
|
|
230
276
|
kind: "json",
|
|
231
|
-
description: "Query cash flow statement (quarterly)",
|
|
277
|
+
description: "Query A-share cash flow statement (quarterly)",
|
|
278
|
+
},
|
|
279
|
+
"fundamental.income-statement-hk": {
|
|
280
|
+
key: "fundamental.income-statement-hk",
|
|
281
|
+
method: "POST",
|
|
282
|
+
path: "/application/open-fundamental/financial-report/income-statement/hk",
|
|
283
|
+
kind: "json",
|
|
284
|
+
description: "Query HK income statement (China GAAP)",
|
|
285
|
+
},
|
|
286
|
+
"fundamental.balance-sheet-hk": {
|
|
287
|
+
key: "fundamental.balance-sheet-hk",
|
|
288
|
+
method: "POST",
|
|
289
|
+
path: "/application/open-fundamental/financial-report/balance-sheet/hk",
|
|
290
|
+
kind: "json",
|
|
291
|
+
description: "Query HK balance sheet (China GAAP)",
|
|
292
|
+
},
|
|
293
|
+
"fundamental.cash-flow-hk": {
|
|
294
|
+
key: "fundamental.cash-flow-hk",
|
|
295
|
+
method: "POST",
|
|
296
|
+
path: "/application/open-fundamental/financial-report/cash-flow-statement/hk",
|
|
297
|
+
kind: "json",
|
|
298
|
+
description: "Query HK cash flow statement (China GAAP)",
|
|
232
299
|
},
|
|
233
300
|
"fundamental.main-business": {
|
|
234
301
|
key: "fundamental.main-business",
|
|
@@ -427,4 +494,33 @@ export const ENDPOINTS = {
|
|
|
427
494
|
kind: "json",
|
|
428
495
|
description: "List WeChat group chatroom IDs",
|
|
429
496
|
},
|
|
497
|
+
"vault.stock-pool.list": {
|
|
498
|
+
key: "vault.stock-pool.list",
|
|
499
|
+
method: "POST",
|
|
500
|
+
path: "/application/open-vault/stock-pool/getPoolList",
|
|
501
|
+
kind: "json",
|
|
502
|
+
description: "List user stock pool IDs and names",
|
|
503
|
+
},
|
|
504
|
+
"vault.stock-pool.stocks": {
|
|
505
|
+
key: "vault.stock-pool.stocks",
|
|
506
|
+
method: "POST",
|
|
507
|
+
path: "/application/open-vault/stock-pool/getStockList",
|
|
508
|
+
kind: "json",
|
|
509
|
+
description: "List securities in stock pool(s)",
|
|
510
|
+
},
|
|
511
|
+
// ─── alternative ───
|
|
512
|
+
"alternative.edb-search": {
|
|
513
|
+
key: "alternative.edb-search",
|
|
514
|
+
method: "POST",
|
|
515
|
+
path: "/application/open-alternative/EDB/search",
|
|
516
|
+
kind: "json",
|
|
517
|
+
description: "Search industry indicator list by keyword",
|
|
518
|
+
},
|
|
519
|
+
"alternative.edb-data": {
|
|
520
|
+
key: "alternative.edb-data",
|
|
521
|
+
method: "POST",
|
|
522
|
+
path: "/application/open-alternative/EDB/getData",
|
|
523
|
+
kind: "json",
|
|
524
|
+
description: "Get industry indicator time-series data by indicator ID list",
|
|
525
|
+
},
|
|
430
526
|
};
|
package/dist/src/core/output.js
CHANGED
|
@@ -100,6 +100,70 @@ export function renderOutput(value, format) {
|
|
|
100
100
|
return renderTable(rows);
|
|
101
101
|
}
|
|
102
102
|
}
|
|
103
|
+
/** Stream large jsonl/csv output row-by-row to avoid building a full string in memory. */
|
|
104
|
+
export async function streamOutputToFile(value, format, outputPath) {
|
|
105
|
+
if (format !== "jsonl" && format !== "csv")
|
|
106
|
+
return false;
|
|
107
|
+
const list = pickListForStreaming(value);
|
|
108
|
+
if (!list)
|
|
109
|
+
return false;
|
|
110
|
+
// Below this row count the join() approach is cheaper than per-row writes.
|
|
111
|
+
if (list.length < 1000)
|
|
112
|
+
return false;
|
|
113
|
+
const { dirname } = await import("node:path");
|
|
114
|
+
const { createWriteStream } = await import("node:fs");
|
|
115
|
+
await fs.mkdir(dirname(outputPath), { recursive: true });
|
|
116
|
+
const stream = createWriteStream(outputPath, { encoding: "utf8" });
|
|
117
|
+
try {
|
|
118
|
+
if (format === "jsonl") {
|
|
119
|
+
for (const item of list) {
|
|
120
|
+
await writeLine(stream, JSON.stringify(item));
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
else {
|
|
124
|
+
const objectRows = list.filter((row) => Boolean(row && typeof row === "object" && !Array.isArray(row)));
|
|
125
|
+
const columns = Array.from(new Set(objectRows.flatMap((row) => Object.keys(row))));
|
|
126
|
+
await writeLine(stream, columns.join(","));
|
|
127
|
+
for (const row of objectRows) {
|
|
128
|
+
const cells = columns.map((column) => csvEscape(formatScalar(row[column])));
|
|
129
|
+
await writeLine(stream, cells.join(","));
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
finally {
|
|
134
|
+
await new Promise((resolve, reject) => {
|
|
135
|
+
stream.end((err) => err ? reject(err) : resolve());
|
|
136
|
+
});
|
|
137
|
+
}
|
|
138
|
+
return true;
|
|
139
|
+
}
|
|
140
|
+
function pickListForStreaming(value) {
|
|
141
|
+
if (Array.isArray(value))
|
|
142
|
+
return value;
|
|
143
|
+
if (value && typeof value === "object") {
|
|
144
|
+
const list = value.list;
|
|
145
|
+
if (Array.isArray(list))
|
|
146
|
+
return list;
|
|
147
|
+
}
|
|
148
|
+
return null;
|
|
149
|
+
}
|
|
150
|
+
function csvEscape(value) {
|
|
151
|
+
let out = value;
|
|
152
|
+
if (/^[=+\-@\t\r]/.test(out))
|
|
153
|
+
out = "'" + out;
|
|
154
|
+
if (/[",\n]/.test(out))
|
|
155
|
+
return `"${out.replaceAll("\"", "\"\"")}"`;
|
|
156
|
+
return out;
|
|
157
|
+
}
|
|
158
|
+
function writeLine(stream, line) {
|
|
159
|
+
return new Promise((resolve, reject) => {
|
|
160
|
+
const ok = stream.write(line + "\n", (err) => err ? reject(err) : undefined);
|
|
161
|
+
if (ok)
|
|
162
|
+
resolve();
|
|
163
|
+
else
|
|
164
|
+
stream.once("drain", () => resolve());
|
|
165
|
+
});
|
|
166
|
+
}
|
|
103
167
|
export async function saveOutputIfNeeded(content, outputPath) {
|
|
104
168
|
if (!outputPath) {
|
|
105
169
|
return;
|