gangtise-openapi-cli 0.43.0 → 0.44.0

package/dist/src/core/client.js

@@ -6,10 +6,10 @@ import { pipeline } from "node:stream/promises";
 import { FormData, request } from "undici";
 import { credentialFingerprint, isTokenCacheValid, normalizeToken, readTokenCache, requireAccessCredentials, writeTokenCache } from "./auth.js";
 import { ApiError, attachEnvelopeTraceId, markStructural, ValidationError } from "./errors.js";
-import { ENDPOINTS, resolveTimeoutMs } from "./endpoints.js";
+import { COSTLY_FETCH_CREDITS, ENDPOINTS, resolveTimeoutMs } from "./endpoints.js";
 import { getLookupData } from "./lookupData/index.js";
 import { stagingPath } from "./output.js";
-import { decodeResponseBody, getDispatcher, isVerbose, logTiming, markRetryable, PAGE_CONCURRENCY, parseRetryAfterMs, quoteBigIntFields, runInOrder, withRetry } from "./transport.js";
+import { decodeResponseBody, getDispatcher, isVerbose, JSON_TOTAL_FACTOR, logTiming, markAuthReplay, PAGE_CONCURRENCY, parseRetryAfterMs, quoteBigIntFields, runInOrder, TRANSFER_TOTAL_FACTOR, withRetry } from "./transport.js";
 import { attachRowSink } from "./rowSink.js";
 import { markIncomplete } from "./exitStatus.js";
 const AUTH_RETRY_CODES = new Set(["8000014", "8000015", "0000001008", "999002"]);
@@ -18,7 +18,6 @@ function hasExpectedShape(expects, payload) {
         return Array.isArray(payload);
     return Boolean(payload && typeof payload === "object" && Array.isArray(payload.list));
 }
-export const COSTLY_FETCH_CREDITS = 1000;
 const PROBE_ABOVE_CREDITS = 50;
 const OFFSET_REFUSAL_CODES = new Set(["140002", "100006"]);
 function readEmptyListAsArray(page) {
@@ -31,12 +30,22 @@ function markTotalCapped(out, why, collected, keepBelow) {
     out.partial = true;
     out.totalCapped = true;
 }
+function stableStringify(value) {
+    if (Array.isArray(value))
+        return `[${value.map(stableStringify).join(",")}]`;
+    if (value && typeof value === "object") {
+        const obj = value;
+        return `{${Object.keys(obj).sort().map((key) => `${JSON.stringify(key)}:${stableStringify(obj[key])}`).join(",")}}`;
+    }
+    return JSON.stringify(value);
+}
 export class GangtiseClient {
     config;
     rowSink;
     allowCostlyFetch = false;
     refreshPromise = null;
     memoCache = null;
+    loginToken = null;
     envTokenInvalidated = false;
     rowSinkClaimed = false;
     constructor(config, rowSink) {
@@ -89,6 +98,7 @@ export class GangtiseClient {
             issuedFor: credentialFingerprint(credentials.accessKey, this.config.baseUrl),
         };
         this.memoCache = cache;
+        this.loginToken = accessToken;
         try {
             await writeTokenCache(this.config.tokenCachePath, cache);
         }
@@ -99,23 +109,45 @@ export class GangtiseClient {
         return accessToken;
     }
     async refreshAuthIfRecoverable(error, useAuth, authState, usedAuthorization) {
-        if (useAuth
-            && !authState.retried
+        if (!(useAuth
             && error instanceof ApiError
             && error.code
             && AUTH_RETRY_CODES.has(error.code)
             && this.config.accessKey
-            && this.config.secretKey) {
-            authState.retried = true;
-            this.envTokenInvalidated = true;
+            && this.config.secretKey))
+            return;
+        if (authState.usedLoginToken)
+            return;
+        this.envTokenInvalidated = true;
+        const replay = () => markAuthReplay(new ApiError(error.message, error.code, error.statusCode, error.details));
+        const tryMemory = () => {
             const memoToken = this.memoCache && isTokenCacheValid(this.memoCache) ? normalizeToken(this.memoCache.accessToken) : null;
-            const alreadyRefreshed = memoToken !== null && usedAuthorization !== undefined && usedAuthorization !== memoToken;
-            if (!alreadyRefreshed) {
-                this.memoCache = null;
-                await this.getAuthorizationHeader(true);
+            if (memoToken === null || usedAuthorization === undefined || memoToken === usedAuthorization)
+                return;
+            if (memoToken === this.loginToken) {
+                authState.usedLoginToken = true;
+                throw replay();
+            }
+            if (!authState.usedUnprovenToken) {
+                authState.usedUnprovenToken = true;
+                throw replay();
+            }
+        };
+        tryMemory();
+        if (!authState.usedUnprovenToken) {
+            const expected = credentialFingerprint(this.config.accessKey, this.config.baseUrl);
+            const disk = await readTokenCache(this.config.tokenCachePath);
+            tryMemory();
+            if (disk && isTokenCacheValid(disk, undefined, expected) && normalizeToken(disk.accessToken) !== usedAuthorization) {
+                this.memoCache = disk;
+                authState.usedUnprovenToken = true;
+                throw replay();
             }
-            throw markRetryable(new ApiError(error.message, error.code, error.statusCode, error.details));
         }
+        this.memoCache = null;
+        await this.getAuthorizationHeader(true);
+        authState.usedLoginToken = true;
+        throw replay();
     }
     buildUrl(path) {
         const base = this.config.baseUrl.endsWith("/") ? this.config.baseUrl : `${this.config.baseUrl}/`;
@@ -184,15 +216,18 @@ export class GangtiseClient {
         }
         const windowRoom = maxWindow === undefined ? Infinity : maxWindow - startFrom;
         const billing = endpoint.billing;
-        const guarded = requestedSize === undefined && billing?.per === "row" && !this.allowCostlyFetch;
+        const guarded = billing?.per === "row" && !this.allowCostlyFetch
+            && (requestedSize === undefined || requestedSize * billing.price > COSTLY_FETCH_CREDITS);
         const refuseIfCostly = (reportedTotal, fetched) => {
             if (!guarded || !billing)
                 return;
-            const rows = Math.min(Math.max(reportedTotal - startFrom, 0), windowRoom);
+            const rows = Math.min(Math.max(reportedTotal - startFrom, 0), windowRoom, requestedSize ?? Infinity);
             const estimatedCredits = Math.round(rows * billing.price * 100) / 100;
             if (estimatedCredits <= COSTLY_FETCH_CREDITS)
                 return;
-            throw new ValidationError(`fetching all ${rows} rows of ${endpoint.key} would cost about ${estimatedCredits} credits (${billing.price} per row), above the ${COSTLY_FETCH_CREDITS}-credit guard for a fetch without --size. Only ${fetched} row(s) were fetched, to learn the total. Pass --size N for a bounded subset, or --yes to fetch them all.`);
+            const what = requestedSize === undefined ? `all ${rows} rows` : `${rows} rows (--size ${requestedSize})`;
+            const escape = requestedSize === undefined ? "Pass --size N for a bounded subset" : "Pass a smaller --size";
+            throw new ValidationError(`fetching ${what} of ${endpoint.key} would cost about ${estimatedCredits} credits (${billing.price} per row), above the ${COSTLY_FETCH_CREDITS}-credit guard. Only ${fetched} row(s) were fetched, to learn the total. ${escape}, or --yes to fetch them all.`);
         };
         let probe;
         let probed = false;
@@ -206,7 +241,7 @@ export class GangtiseClient {
             else {
                 refuseIfCostly(probe.total, probe.list.length);
                 probeRows = probe.list.length;
-                probed = probe.list.length === Math.min(Math.max(probe.total - startFrom, 0), windowRoom);
+                probed = probe.list.length === Math.min(Math.max(probe.total - startFrom, 0), windowRoom, requestedSize ?? Infinity);
             }
         }
         const firstPageSize = probed ? 1 : Math.min(requestedSize === undefined ? maxPageSize : Math.min(maxPageSize, requestedSize), windowRoom);
@@ -245,29 +280,33 @@ export class GangtiseClient {
         const dropRepeats = (rows) => {
             if (!seen || !rowId)
                 return rows;
-            const onThisPage = new Set();
+            const onThisPage = new Map();
             return rows.filter((row) => {
                 const id = row && typeof row === "object" ? row[rowId] : undefined;
                 if (id === undefined || id === null)
                     return true;
-                const digest = createHash("sha1").update(JSON.stringify(row)).digest("base64");
+                const digest = createHash("sha1").update(stableStringify(row)).digest("base64");
                 const key = String(id);
-                const previous = seen.get(key);
-                const earlierOnThisPage = onThisPage.has(key);
-                onThisPage.add(key);
-                if (previous === undefined) {
-                    seen.set(key, digest);
+                const pageVersions = onThisPage.get(key);
+                if (pageVersions && !pageVersions.has(digest))
+                    idIsRowKey = false;
+                if (pageVersions)
+                    pageVersions.add(digest);
+                else
+                    onThisPage.set(key, new Set([digest]));
+                const versions = seen.get(key);
+                if (versions === undefined) {
+                    seen.set(key, new Set([digest]));
                     return true;
                 }
-                if (previous !== digest) {
-                    if (earlierOnThisPage)
-                        idIsRowKey = false;
-                    else
-                        changedRows++;
-                    return true;
+                if (versions.has(digest)) {
+                    duplicateRows++;
+                    return false;
                 }
-                duplicateRows++;
-                return false;
+                versions.add(digest);
+                if (!pageVersions)
+                    changedRows++;
+                return true;
             });
         };
         const keep = async (fetched) => {
@@ -433,7 +472,7 @@ export class GangtiseClient {
         }
         const dispatcher = getDispatcher();
         const url = this.buildUrl(endpoint.path);
-        const authState = { retried: false };
+        const authState = {};
         const timeoutMs = resolveTimeoutMs(this.config.timeoutMs, endpoint);
         return withRetry(async () => {
             const isUpload = endpoint.kind === 'upload';
@@ -454,6 +493,7 @@ export class GangtiseClient {
                 body: endpoint.method === 'GET' ? undefined : (isUpload ? body : JSON.stringify(body ?? {})),
                 headersTimeout: timeoutMs,
                 bodyTimeout: timeoutMs,
+                signal: AbortSignal.timeout(timeoutMs * (isUpload ? TRANSFER_TOTAL_FACTOR : JSON_TOTAL_FACTOR)),
                 dispatcher,
             });
             const encoding = response.headers['content-encoding'];
@@ -510,7 +550,7 @@ export class GangtiseClient {
         }).catch((error) => {
             const isIndicatorFetch = endpoint.key === 'indicator.cross-section' || endpoint.key === 'indicator.time-series' || endpoint.key === 'indicator.screener';
             if (isIndicatorFetch && error instanceof ApiError && error.code === '999999') {
-                throw new ApiError(error.message, error.code, error.statusCode, error.details, error.retryAfterMs, 'EDE 取数故障。先核对参数再重试：参数名以 indicator search 的 parameterList 为准（传错名会报 100003 并指名）、日期匹配指标周期（财务报表类用报告期末如 2025-12-31，PE/PB 等日频估值用交易日）、标的在 scopeList 覆盖内、required 参数已补。注意此码不表示无数据——无数据返回保留行列的 null 单元格；code 或参数名写错报 100003、缺必填参数报 100001，都会指名。');
+                throw new ApiError(error.message, error.code, error.statusCode, error.details, error.retryAfterMs, 'EDE 取数故障。先核对参数再重试：参数名以 indicator search 的 parameterList 为准（传错名会报 100003 并指名）、日期匹配指标周期（财务报表类用报告期末如 2025-12-31，PE/PB 等日频估值用交易日）、标的在 scopeList 覆盖内、required 参数已补。注意此码不表示无数据——无数据返回保留行列的 null 单元格；code 或参数名写错报 100003、缺必填参数报 100001，都会指名（fiscalYear 例外：漏传不报错，返回 null 或默认年度的值）。');
             }
             throw error;
         });
@@ -534,11 +574,12 @@ export class GangtiseClient {
         Object.entries(query).forEach(([key, value]) => {
             url.searchParams.set(key, String(value));
         });
-        const authState = { retried: false };
+        const authState = {};
         const timeoutMs = resolveTimeoutMs(this.config.timeoutMs, endpoint);
         return withRetry(async () => {
             const authorization = await this.getAuthorizationHeader();
             const startedAt = Date.now();
+            const signal = AbortSignal.timeout(timeoutMs * TRANSFER_TOTAL_FACTOR);
             let currentUrl = url;
             let auth = authorization;
             const isPost = endpoint.method === 'POST';
@@ -550,6 +591,7 @@ export class GangtiseClient {
                 body: isPost ? JSON.stringify(body ?? {}) : undefined,
                 headersTimeout: timeoutMs,
                 bodyTimeout: timeoutMs,
+                signal,
                 dispatcher,
             });
             for (let hops = 0; hops < 3 && response.statusCode >= 300 && response.statusCode < 400; hops++) {
@@ -567,6 +609,7 @@ export class GangtiseClient {
                     headers: auth ? { Authorization: auth } : {},
                     headersTimeout: timeoutMs,
                     bodyTimeout: timeoutMs,
+                    signal,
                     dispatcher,
                 });
             }
@@ -662,6 +705,23 @@ export class GangtiseClient {
             },
         });
     }
+    refuseIfEstimatedCostly(endpoint, body) {
+        const billing = endpoint.billing;
+        if (!endpoint.estimateBilledUnits || !billing || this.allowCostlyFetch)
+            return;
+        let units;
+        try {
+            units = endpoint.estimateBilledUnits(body && typeof body === "object" ? body : {});
+        }
+        catch (error) {
+            const why = error instanceof Error ? error.message : String(error);
+            throw new ValidationError(`${endpoint.key} could not be priced before sending (${why}), and it is billed per ${billing.per} with no upper bound. Fix the request, or pass --yes to send it as it is.`);
+        }
+        const credits = Math.ceil(units * billing.price);
+        if (credits <= COSTLY_FETCH_CREDITS)
+            return;
+        throw new ValidationError(`${endpoint.key} would cost about ${credits} credits (${billing.price} per ${billing.per}, about ${units} estimated from the requested date range), above the ${COSTLY_FETCH_CREDITS}-credit guard. Narrow the date range, or pass --yes to send it.`);
+    }
     async call(endpointKey, body, query, options) {
         const endpoint = ENDPOINTS[endpointKey];
         if (!endpoint) {
@@ -673,6 +733,7 @@ export class GangtiseClient {
         if (endpoint.kind === 'download') {
             return this.download(endpoint, query ?? {}, options, body);
         }
+        this.refuseIfEstimatedCostly(endpoint, body);
         if (endpoint.kind === 'json' && endpoint.pagination?.enabled) {
             return this.requestPaginated(endpoint, body);
         }

package/dist/src/core/commandBodies.js

@@ -1,4 +1,4 @@
-import { maybeArray, parseFrom, parseIndicatorParams, parseOptionalNumberOption, parseScreenerIndicators, parseSize } from "./args.js";
+import { maybeArray, parseIndicatorParams, parseOptionalNumberOption, parseScreenerIndicators } from "./args.js";
 import { ValidationError } from "./errors.js";
 import { QUOTE_MAX_LIMIT } from "./quoteSharding.js";
 export function buildQuoteKlineBody(options) {
@@ -10,32 +10,6 @@ export function buildQuoteKlineBody(options) {
         fieldList: maybeArray(options.field),
     };
 }
-export function buildWechatMessageListBody(options) {
-    return {
-        from: parseFrom(options.from),
-        size: parseSize(options.size),
-        startTime: options.startTime,
-        endTime: options.endTime,
-        keyword: options.keyword,
-        securityList: maybeArray(options.security),
-        wechatGroupIdList: maybeArray(options.wechatGroupId),
-        industryIdList: maybeArray(options.industry),
-        categoryList: maybeArray(options.category),
-        tagList: maybeArray(options.tag),
-    };
-}
-export function buildWechatChatroomListBody(options) {
-    return {
-        from: parseFrom(options.from),
-        size: parseSize(options.size),
-        roomName: options.roomName.length > 0 ? options.roomName.join(",") : undefined,
-    };
-}
-export function buildStockPoolStocksBody(options) {
-    return {
-        poolIdList: options.poolId?.length ? options.poolId : ["all"],
-    };
-}
 const DATE_PARAM_KEYS = new Set(["tradeDate", "reportDate"]);
 function assertParamCodesBound(groups, codes) {
     if (!groups?.length)

package/dist/src/core/config.js

@@ -12,14 +12,18 @@ export function loadConfig() {
         process.stderr.write(`[gangtise] warning: GANGTISE_TIMEOUT_MS=${rawTimeout} is not in effect, using ${timeoutMs} ms: expected whole milliseconds from ${MIN_TIMEOUT_MS} to ${MAX_TIMEOUT_MS}\n`);
     }
     return {
-        baseUrl: process.env.GANGTISE_BASE_URL ?? DEFAULT_BASE_URL,
+        baseUrl: envValue("GANGTISE_BASE_URL") ?? DEFAULT_BASE_URL,
         timeoutMs,
-        accessKey: process.env.GANGTISE_ACCESS_KEY,
-        secretKey: process.env.GANGTISE_SECRET_KEY,
-        token: process.env.GANGTISE_TOKEN,
-        tokenCachePath: process.env.GANGTISE_TOKEN_CACHE_PATH ?? DEFAULT_TOKEN_CACHE_PATH,
+        accessKey: envValue("GANGTISE_ACCESS_KEY"),
+        secretKey: envValue("GANGTISE_SECRET_KEY"),
+        token: envValue("GANGTISE_TOKEN"),
+        tokenCachePath: envValue("GANGTISE_TOKEN_CACHE_PATH") ?? DEFAULT_TOKEN_CACHE_PATH,
     };
 }
+function envValue(name) {
+    const value = process.env[name]?.trim();
+    return value ? value : undefined;
+}
 export const MIN_TIMEOUT_MS = 1_000;
 export const MAX_TIMEOUT_MS = 3_600_000;
 export function resolveTimeoutEnv(raw) {

package/dist/src/core/download.js

@@ -84,10 +84,27 @@ export function extFromContentType(contentType) {
     const mime = contentType.split(";")[0].trim().toLowerCase();
     return MIME_EXT[mime] ?? "";
 }
+const KNOWN_EXT = new Set(Object.values(MIME_EXT).concat([".md", ".jpeg", ".htm"]));
+export function extFromUrl(url) {
+    try {
+        const ext = extname(decodeURIComponent(new URL(url).pathname)).toLowerCase();
+        return KNOWN_EXT.has(ext) ? ext : "";
+    }
+    catch {
+        return "";
+    }
+}
+function resultExt(file) {
+    if (file.filename)
+        return extname(file.filename);
+    if (typeof file.url === "string")
+        return extFromUrl(file.url);
+    return extFromContentType(file.contentType);
+}
 export async function resolveTitle(client, result, listEndpoint, idField, idValue, options = {}) {
     const titleField = options.titleField ?? "title";
     const file = result;
-    const serverExt = file.filename ? extname(file.filename) : extFromContentType(file.contentType);
+    const serverExt = resultExt(file);
     function buildFilename(rawTitle) {
         let title = sanitizeFilename(rawTitle).trim();
         if (serverExt && !title.toLowerCase().endsWith(serverExt.toLowerCase())) {
@@ -130,7 +147,7 @@ async function downloadUrlTo(url, outputPath) {
     const { pipeline } = await import("node:stream/promises");
     const { dirname } = await import("node:path");
     const { request } = await import("undici");
-    const { getDispatcher, logTiming, withRetry } = await import("./transport.js");
+    const { getDispatcher, logTiming, TRANSFER_TOTAL_FACTOR, withRetry } = await import("./transport.js");
     const { loadConfig } = await import("./config.js");
     const timeoutMs = loadConfig().timeoutMs;
     await fs.mkdir(dirname(outputPath), { recursive: true });
@@ -150,7 +167,7 @@ async function downloadUrlTo(url, outputPath) {
             method: "GET",
             headersTimeout: timeoutMs,
             bodyTimeout: timeoutMs,
-            signal: AbortSignal.timeout(timeoutMs * 10),
+            signal: AbortSignal.timeout(timeoutMs * TRANSFER_TOTAL_FACTOR),
             dispatcher: getDispatcher(),
         };
         let currentUrl = url;
@@ -189,6 +206,23 @@ async function downloadUrlTo(url, outputPath) {
         throw error;
     }
 }
+export function autoDownloadName(file, fallbackName) {
+    return truncateFilename(file.filename ? sanitizeFilename(file.filename) : sanitizeFilename(fallbackName) + extFromContentType(file.contentType));
+}
+export async function placeStreamedDownload(file, fallbackName, named) {
+    let target;
+    try {
+        target = await uniquePath(truncateFilename(named ?? autoDownloadName(file, fallbackName)));
+        await fs.rename(file.savedPath, target);
+    }
+    catch (error) {
+        if (target)
+            await releaseClaim(target);
+        await fs.unlink(file.savedPath).catch(() => { });
+        throw error;
+    }
+    return target;
+}
 export async function saveDownloadResult(result, fallbackName, output) {
     if (!(result && typeof result === "object")) {
         throw new DownloadError("Unexpected download response");
@@ -200,8 +234,7 @@ export async function saveDownloadResult(result, fallbackName, output) {
         return;
     }
     if (file.data instanceof Uint8Array) {
-        const autoName = (file.filename ? sanitizeFilename(file.filename) : undefined) ?? (safeFallback + extFromContentType(file.contentType));
-        const outputPath = output ?? await uniquePath(truncateFilename(autoName));
+        const outputPath = output ?? await uniquePath(autoDownloadName(file, fallbackName));
         try {
             await saveOutputIfNeeded(file.data, outputPath, false);
         }
@@ -227,12 +260,16 @@ export async function saveDownloadResult(result, fallbackName, output) {
         return;
     }
     if (typeof file.url === "string") {
-        if (output) {
-            await downloadUrlTo(file.url, output);
-            process.stdout.write(`${output}\n`);
-            return;
+        const outputPath = output ?? await uniquePath(truncateFilename(safeFallback + extFromUrl(file.url)));
+        try {
+            await downloadUrlTo(file.url, outputPath);
+        }
+        catch (error) {
+            if (!output)
+                await releaseClaim(outputPath);
+            throw error;
         }
-        process.stdout.write(`${file.url}\n`);
+        process.stdout.write(`${outputPath}\n`);
         return;
     }
     throw new DownloadError("Unexpected download response");