gangtise-openapi-cli 0.41.1 → 0.43.0

package/dist/src/core/client.js

@@ -1,3 +1,4 @@
+import { createHash } from "node:crypto";
 import { createWriteStream } from "node:fs";
 import fs from "node:fs/promises";
 import path from "node:path";
@@ -10,10 +11,30 @@ import { getLookupData } from "./lookupData/index.js";
 import { stagingPath } from "./output.js";
 import { decodeResponseBody, getDispatcher, isVerbose, logTiming, markRetryable, PAGE_CONCURRENCY, parseRetryAfterMs, quoteBigIntFields, runInOrder, withRetry } from "./transport.js";
 import { attachRowSink } from "./rowSink.js";
+import { markIncomplete } from "./exitStatus.js";
 const AUTH_RETRY_CODES = new Set(["8000014", "8000015", "0000001008", "999002"]);
+function hasExpectedShape(expects, payload) {
+    if (expects === "array")
+        return Array.isArray(payload);
+    return Boolean(payload && typeof payload === "object" && Array.isArray(payload.list));
+}
+export const COSTLY_FETCH_CREDITS = 1000;
+const PROBE_ABOVE_CREDITS = 50;
+const OFFSET_REFUSAL_CODES = new Set(["140002", "100006"]);
+function readEmptyListAsArray(page) {
+    const rec = page;
+    if (rec && typeof rec === "object" && rec.total === 0 && rec.list === null)
+        rec.list = [];
+}
+function markTotalCapped(out, why, collected, keepBelow) {
+    process.stderr.write(`[gangtise] warning: ${why}. This export is TRUNCATED at ${collected} rows. Narrow the query (e.g. a shorter time range) until total stays below ${keepBelow}, and fetch in slices.\n`);
+    out.partial = true;
+    out.totalCapped = true;
+}
 export class GangtiseClient {
     config;
     rowSink;
+    allowCostlyFetch = false;
     refreshPromise = null;
     memoCache = null;
     envTokenInvalidated = false;
@@ -162,15 +183,42 @@ export class GangtiseClient {
             throw new ValidationError(`${endpoint.key} serves rows only up to offset ${maxWindow} (from + size ≤ ${maxWindow}); --from ${startFrom} is past it. Narrow the query (e.g. a shorter time range) instead of paging deeper.`);
         }
         const windowRoom = maxWindow === undefined ? Infinity : maxWindow - startFrom;
-        const firstPageSize = Math.min(requestedSize === undefined ? maxPageSize : Math.min(maxPageSize, requestedSize), windowRoom);
-        const firstPage = await this.requestJson(endpoint, {
+        const billing = endpoint.billing;
+        const guarded = requestedSize === undefined && billing?.per === "row" && !this.allowCostlyFetch;
+        const refuseIfCostly = (reportedTotal, fetched) => {
+            if (!guarded || !billing)
+                return;
+            const rows = Math.min(Math.max(reportedTotal - startFrom, 0), windowRoom);
+            const estimatedCredits = Math.round(rows * billing.price * 100) / 100;
+            if (estimatedCredits <= COSTLY_FETCH_CREDITS)
+                return;
+            throw new ValidationError(`fetching all ${rows} rows of ${endpoint.key} would cost about ${estimatedCredits} credits (${billing.price} per row), above the ${COSTLY_FETCH_CREDITS}-credit guard for a fetch without --size. Only ${fetched} row(s) were fetched, to learn the total. Pass --size N for a bounded subset, or --yes to fetch them all.`);
+        };
+        let probe;
+        let probed = false;
+        let probeRows = 0;
+        if (guarded && billing && Math.min(maxPageSize, windowRoom) * billing.price > PROBE_ABOVE_CREDITS) {
+            probe = await this.requestJson(endpoint, { ...initialBody, from: startFrom, size: 1 });
+            readEmptyListAsArray(probe);
+            if (!this.isPaginatedListResponse(probe)) {
+                probed = true;
+            }
+            else {
+                refuseIfCostly(probe.total, probe.list.length);
+                probeRows = probe.list.length;
+                probed = probe.list.length === Math.min(Math.max(probe.total - startFrom, 0), windowRoom);
+            }
+        }
+        const firstPageSize = probed ? 1 : Math.min(requestedSize === undefined ? maxPageSize : Math.min(maxPageSize, requestedSize), windowRoom);
+        const firstPage = probed ? probe : await this.requestJson(endpoint, {
             ...initialBody,
             from: startFrom,
             size: firstPageSize,
         });
+        readEmptyListAsArray(firstPage);
         if (!this.isPaginatedListResponse(firstPage)) {
             process.stderr.write(`[gangtise] warning: ${endpoint.key} is marked paginated but the first page has an unexpected shape (no numeric total + list); returning it as-is\n`);
-            process.exitCode = 3;
+            markIncomplete();
             return firstPage;
         }
         const total = firstPage.total;
@@ -189,7 +237,41 @@ export class GangtiseClient {
             sink.setFieldList(firstPage.fieldList);
         const collected = [];
         let count = 0;
-        const keep = async (rows) => {
+        const rowId = endpoint.rowId;
+        const seen = rowId ? new Map() : undefined;
+        let duplicateRows = 0;
+        let changedRows = 0;
+        let idIsRowKey = true;
+        const dropRepeats = (rows) => {
+            if (!seen || !rowId)
+                return rows;
+            const onThisPage = new Set();
+            return rows.filter((row) => {
+                const id = row && typeof row === "object" ? row[rowId] : undefined;
+                if (id === undefined || id === null)
+                    return true;
+                const digest = createHash("sha1").update(JSON.stringify(row)).digest("base64");
+                const key = String(id);
+                const previous = seen.get(key);
+                const earlierOnThisPage = onThisPage.has(key);
+                onThisPage.add(key);
+                if (previous === undefined) {
+                    seen.set(key, digest);
+                    return true;
+                }
+                if (previous !== digest) {
+                    if (earlierOnThisPage)
+                        idIsRowKey = false;
+                    else
+                        changedRows++;
+                    return true;
+                }
+                duplicateRows++;
+                return false;
+            });
+        };
+        const keep = async (fetched) => {
+            const rows = dropRepeats(fetched);
             const kept = requestedSize === undefined ? rows : rows.slice(0, Math.max(0, requestedSize - count));
             if (kept.length === 0)
                 return;
@@ -203,10 +285,20 @@ export class GangtiseClient {
         const checkTotalCap = async (out) => {
             if (probeFits)
                 return this.flagIfTotalCapped(endpoint, initialBody, total, out, count);
-            process.stderr.write(`[gangtise] note: ${endpoint.key} reported total=${total}, which reaches its ${maxWindow}-row offset window, so whether that is the real count cannot be checked. Narrow the query (e.g. a shorter time range) to keep total below ${maxWindow} if you need to be sure.\n`);
+            markTotalCapped(out, `${endpoint.key} reported total=${total}, which equals its ${maxWindow}-row offset window: rows past it can be neither fetched nor counted`, count, maxWindow);
         };
         const result = () => {
             const out = { ...firstPage, total, list: collected };
+            if (duplicateRows > 0) {
+                process.stderr.write(`[gangtise] warning: ${endpoint.key} returned ${duplicateRows} row(s) more than once — its sort order is not unique, so paging also missed as many rows. The duplicates were dropped and the result is partial. Re-run over a shorter time range to fetch the missing rows.\n`);
+                out.duplicateRows = duplicateRows;
+                out.partial = true;
+            }
+            if (changedRows > 0 && idIsRowKey && !endpoint.rowIdUnverified) {
+                process.stderr.write(`[gangtise] warning: ${endpoint.key} returned ${changedRows} row(s) whose ${rowId} came back on a later page with different content — the list changed while it was being paged, so rows may also have been missed. Both versions were kept (deduplicating by ${rowId} keeps one) and the result is partial. Re-run to fetch a consistent result.\n`);
+                out.changedRows = changedRows;
+                out.partial = true;
+            }
             return sink ? attachRowSink(out, sink) : out;
         };
         await keep(firstPage.list);
@@ -226,6 +318,7 @@ export class GangtiseClient {
                 await checkTotalCap(out);
             return out;
         }
+        refuseIfCostly(total, count + (probed ? 0 : probeRows));
         const MAX_PAGES = 1000;
         let truncatedByPageCap = false;
         const pageRequests = [];
@@ -258,6 +351,7 @@ export class GangtiseClient {
                     from: req.from,
                     size: req.size,
                 });
+                readEmptyListAsArray(page);
                 if (!this.isPaginatedListResponse(page)) {
                     unexpectedShape = true;
                     failedPages.push(req);
@@ -276,7 +370,7 @@ export class GangtiseClient {
                 failedPages.push(req);
                 return [];
             }
-        }, (list) => keep(list));
+        }, (list) => keep(list), PAGE_CONCURRENCY * 4);
         if (unexpectedShape) {
             process.stderr.write(`[gangtise] warning: a page response had unexpected shape; its rows are missing (counted in failedPages)\n`);
         }
@@ -287,9 +381,10 @@ export class GangtiseClient {
             process.stderr.write(`[gangtise] warning: hit the ${MAX_PAGES}-page safety cap; fetched ${count} of ${total} rows. Narrow the query (e.g. a shorter date range) or pass --size to fetch a bounded subset.\n`);
         }
         const short = count < target;
+        const shortByRepeatsOnly = short && count + duplicateRows >= target;
         const out = result();
         flagWindowCut(out);
-        if (requestedSize === undefined && total > 0 && target === wanted && !short && !totalDrift && !truncatedByPageCap && failedPages.length === 0) {
+        if (requestedSize === undefined && total > 0 && target === wanted && (!short || shortByRepeatsOnly) && !totalDrift && !truncatedByPageCap && failedPages.length === 0) {
             await checkTotalCap(out);
         }
         if (short || totalDrift || failedPages.length > 0)
@@ -304,7 +399,7 @@ export class GangtiseClient {
             const skippedHint = aborted ? " A page hit a non-retryable error (e.g. rate limit); remaining pages were skipped." : "";
             process.stderr.write(`[gangtise] warning: ${failedPages.length}/${pageRequests.length} pages not fetched${detail}; results are partial — got ${count}/${total} rows (see failedPages).${skippedHint}\n`);
         }
-        else if (short && !truncatedByPageCap && !totalDrift) {
+        else if (short && !shortByRepeatsOnly && !truncatedByPageCap && !totalDrift) {
             process.stderr.write(`[gangtise] warning: server returned ${count} of ${total} rows (a later page came back short); results may be incomplete\n`);
         }
         return out;
@@ -316,13 +411,14 @@ export class GangtiseClient {
             if (this.isPaginatedListResponse(probe) && probe.list.length > 0)
                 totalCapped = true;
         }
-        catch {
+        catch (error) {
+            if (error instanceof ApiError && error.code !== undefined && OFFSET_REFUSAL_CODES.has(error.code)) {
+                markTotalCapped(out, `${endpoint.key} reported total=${total} and refused the row right after it (${error.code}) — most likely a server-side offset window at total, so rows past it may exist and can be neither fetched nor counted`, collectedLength, total);
+            }
         }
         if (!totalCapped)
             return;
-        process.stderr.write(`[gangtise] warning: ${endpoint.key} reported total=${total} but rows exist past that offset — 'total' is a server-side cap, not the real count. This export is TRUNCATED at ${collectedLength} rows. Narrow the query (date range / filters) and fetch in slices.\n`);
-        out.partial = true;
-        out.totalCapped = true;
+        markTotalCapped(out, `${endpoint.key} reported total=${total} but rows exist past that offset — 'total' is a server-side cap, not the real count`, collectedLength, total);
     }
     async login() {
         if (this.config.token && !this.envTokenInvalidated) {
@@ -393,8 +489,9 @@ export class GangtiseClient {
                     this.throwHttpError(parsed, response.statusCode, retryAfterMs);
                 }
                 const payload = this.unwrapEnvelope(parsed, response.statusCode, retryAfterMs);
-                if (endpoint.expects === "list" && !(payload && typeof payload === "object" && Array.isArray(payload.list))) {
-                    throw markStructural(new ApiError(`${endpoint.key} returned no list payload (got ${payload === null ? "null" : Array.isArray(payload) ? "an array" : typeof payload}) — the response layout may have changed`, undefined, response.statusCode, parsed));
+                if (endpoint.expects && !hasExpectedShape(endpoint.expects, payload)) {
+                    const got = payload === null ? "null" : Array.isArray(payload) ? "an array" : typeof payload;
+                    throw markStructural(new ApiError(`${endpoint.key} returned no ${endpoint.expects} payload (got ${got}) — the response layout may have changed`, undefined, response.statusCode, parsed));
                 }
                 return payload;
             }
@@ -413,7 +510,7 @@ export class GangtiseClient {
         }).catch((error) => {
             const isIndicatorFetch = endpoint.key === 'indicator.cross-section' || endpoint.key === 'indicator.time-series' || endpoint.key === 'indicator.screener';
             if (isIndicatorFetch && error instanceof ApiError && error.code === '999999') {
-                throw new ApiError(error.message, error.code, error.statusCode, error.details, error.retryAfterMs, 'EDE 取数故障。先核对参数再重试：参数名以 indicator search 的 parameterList 为准（传错名是静默失效）、日期匹配指标周期（财务报表类用报告期末如 2025-12-31，PE/PB 等日频估值用交易日）、标的在 scopeList 覆盖内、required 参数已补。注意此码不表示无数据——有效 code 无数据现在返回保留行列的 null 单元格，空表则多半是整个轴的 code 都没被识别。');
+                throw new ApiError(error.message, error.code, error.statusCode, error.details, error.retryAfterMs, 'EDE 取数故障。先核对参数再重试：参数名以 indicator search 的 parameterList 为准（传错名会报 100003 并指名）、日期匹配指标周期（财务报表类用报告期末如 2025-12-31，PE/PB 等日频估值用交易日）、标的在 scopeList 覆盖内、required 参数已补。注意此码不表示无数据——无数据返回保留行列的 null 单元格；code 或参数名写错报 100003、缺必填参数报 100001，都会指名。');
             }
             throw error;
         });

package/dist/src/core/commandBodies.js

@@ -1,11 +1,12 @@
 import { maybeArray, parseFrom, parseIndicatorParams, parseOptionalNumberOption, parseScreenerIndicators, parseSize } from "./args.js";
 import { ValidationError } from "./errors.js";
+import { QUOTE_MAX_LIMIT } from "./quoteSharding.js";
 export function buildQuoteKlineBody(options) {
     return {
         securityList: maybeArray(options.security),
         startDate: options.startDate,
         endDate: options.endDate,
-        limit: parseOptionalNumberOption(options.limit, "--limit", { integer: true, min: 1, max: 10000 }),
+        limit: parseOptionalNumberOption(options.limit, "--limit", { integer: true, min: 1, max: QUOTE_MAX_LIMIT }),
         fieldList: maybeArray(options.field),
     };
 }

package/dist/src/core/config.js

@@ -3,15 +3,28 @@ import path from "node:path";
 export const DEFAULT_BASE_URL = "https://openapi.gangtise.com";
 export const DEFAULT_TIMEOUT_MS = 30_000;
 export const DEFAULT_TOKEN_CACHE_PATH = path.join(os.homedir(), ".config", "gangtise", "token.json");
+let timeoutWarned = false;
 export function loadConfig() {
-    const timeoutValue = process.env.GANGTISE_TIMEOUT_MS;
-    const timeoutMs = timeoutValue ? Number(timeoutValue) : DEFAULT_TIMEOUT_MS;
+    const rawTimeout = process.env.GANGTISE_TIMEOUT_MS?.trim();
+    const timeoutMs = resolveTimeoutEnv(rawTimeout);
+    if (rawTimeout && Number(rawTimeout) !== timeoutMs && !timeoutWarned) {
+        timeoutWarned = true;
+        process.stderr.write(`[gangtise] warning: GANGTISE_TIMEOUT_MS=${rawTimeout} is not in effect, using ${timeoutMs} ms: expected whole milliseconds from ${MIN_TIMEOUT_MS} to ${MAX_TIMEOUT_MS}\n`);
+    }
     return {
         baseUrl: process.env.GANGTISE_BASE_URL ?? DEFAULT_BASE_URL,
-        timeoutMs: Number.isFinite(timeoutMs) && timeoutMs > 0 ? timeoutMs : DEFAULT_TIMEOUT_MS,
+        timeoutMs,
         accessKey: process.env.GANGTISE_ACCESS_KEY,
         secretKey: process.env.GANGTISE_SECRET_KEY,
         token: process.env.GANGTISE_TOKEN,
         tokenCachePath: process.env.GANGTISE_TOKEN_CACHE_PATH ?? DEFAULT_TOKEN_CACHE_PATH,
     };
 }
+export const MIN_TIMEOUT_MS = 1_000;
+export const MAX_TIMEOUT_MS = 3_600_000;
+export function resolveTimeoutEnv(raw) {
+    if (!raw || !/^\d+$/.test(raw.trim()))
+        return DEFAULT_TIMEOUT_MS;
+    const ms = Number(raw);
+    return ms < MIN_TIMEOUT_MS ? DEFAULT_TIMEOUT_MS : Math.min(MAX_TIMEOUT_MS, ms);
+}

package/dist/src/core/download.js

@@ -1,7 +1,7 @@
 import fs from "node:fs/promises";
 import { dirname, extname } from "node:path";
 import { DownloadError } from "./errors.js";
-import { saveOutputIfNeeded, stagingPath } from "./output.js";
+import { saveOutputIfNeeded, stagingPath, trackClaim } from "./output.js";
 import { extractTitles, lookupTitleCache, readTitleCache, TITLE_LOOKUP_SIZE, writeTitleCache } from "./titleCache.js";
 function sanitizeFilename(name) {
     return name.replace(/[/\\:*?"<>|\u0000-\u001f]/g, "_");
@@ -15,6 +15,10 @@ function truncateFilename(name, maxBytes = 200) {
         stem.pop();
     return stem.join("") + ext;
 }
+function trackedClaim(p) {
+    trackClaim(p);
+    return p;
+}
 export async function uniquePath(p) {
     const claim = async (f) => {
         try {
@@ -30,13 +34,13 @@ export async function uniquePath(p) {
     };
     await fs.mkdir(dirname(p) || ".", { recursive: true });
     if (await claim(p))
-        return p;
+        return trackedClaim(p);
     const ext = extname(p);
     const stem = p.slice(0, p.length - ext.length);
     for (let i = 1; i <= 99; i++) {
         const candidate = `${stem}-${i}${ext}`;
         if (await claim(candidate))
-            return candidate;
+            return trackedClaim(candidate);
     }
     throw new DownloadError(`Refusing to overwrite: 100 files already share the name "${p}" — pass --output or clean up the directory`);
 }
@@ -199,7 +203,7 @@ export async function saveDownloadResult(result, fallbackName, output) {
         const autoName = (file.filename ? sanitizeFilename(file.filename) : undefined) ?? (safeFallback + extFromContentType(file.contentType));
         const outputPath = output ?? await uniquePath(truncateFilename(autoName));
         try {
-            await saveOutputIfNeeded(file.data, outputPath);
+            await saveOutputIfNeeded(file.data, outputPath, false);
         }
         catch (error) {
             if (!output)
@@ -212,7 +216,7 @@ export async function saveDownloadResult(result, fallbackName, output) {
     if (typeof file.text === "string") {
         const outputPath = output ?? await uniquePath(truncateFilename(`${safeFallback}.txt`));
         try {
-            await saveOutputIfNeeded(file.text, outputPath);
+            await saveOutputIfNeeded(file.text, outputPath, false);
         }
         catch (error) {
             if (!output)