gangtise-openapi-cli 0.37.0 → 0.38.0

package/dist/src/core/client.js

@@ -4,18 +4,28 @@ import path from "node:path";
 import { pipeline } from "node:stream/promises";
 import { FormData, request } from "undici";
 import { isTokenCacheValid, normalizeToken, readTokenCache, requireAccessCredentials, writeTokenCache } from "./auth.js";
-import { ApiError, attachEnvelopeTraceId, ValidationError } from "./errors.js";
+import { ApiError, attachEnvelopeTraceId, markStructural, ValidationError } from "./errors.js";
 import { ENDPOINTS, resolveTimeoutMs } from "./endpoints.js";
 import { getLookupData } from "./lookupData/index.js";
-import { decodeResponseBody, getDispatcher, isVerbose, logTiming, markRetryable, PAGE_CONCURRENCY, parseRetryAfterMs, quoteBigIntFields, runWithConcurrency, withRetry } from "./transport.js";
+import { decodeResponseBody, getDispatcher, isVerbose, logTiming, markRetryable, PAGE_CONCURRENCY, parseRetryAfterMs, quoteBigIntFields, runInOrder, withRetry } from "./transport.js";
+import { attachRowSink } from "./rowSink.js";
 const AUTH_RETRY_CODES = new Set(["8000014", "8000015", "0000001008", "999002"]);
 export class GangtiseClient {
     config;
+    rowSink;
     refreshPromise = null;
     memoCache = null;
     envTokenInvalidated = false;
-    constructor(config) {
+    rowSinkClaimed = false;
+    constructor(config, rowSink) {
         this.config = config;
+        this.rowSink = rowSink;
+    }
+    claimRowSink() {
+        if (!this.rowSink || this.rowSinkClaimed)
+            return undefined;
+        this.rowSinkClaimed = true;
+        return this.rowSink;
     }
     async getAuthorizationHeader(forceRefresh = false) {
         if (this.config.token && !this.envTokenInvalidated && !forceRefresh) {
@@ -151,27 +161,42 @@ export class GangtiseClient {
             return firstPage;
         }
         const total = firstPage.total;
-        const collected = [...firstPage.list];
         const available = Math.max(total - startFrom, 0);
         const target = requestedSize === undefined ? available : Math.min(requestedSize, available);
+        const sink = this.claimRowSink();
+        if (sink && Array.isArray(firstPage.fieldList))
+            sink.setFieldList(firstPage.fieldList);
+        const collected = [];
+        let count = 0;
+        const keep = async (rows) => {
+            const kept = requestedSize === undefined ? rows : rows.slice(0, Math.max(0, requestedSize - count));
+            if (kept.length === 0)
+                return;
+            count += kept.length;
+            if (sink)
+                await sink.push(kept);
+            else
+                for (const row of kept)
+                    collected.push(row);
+        };
+        const result = () => {
+            const out = { ...firstPage, total, list: collected };
+            return sink ? attachRowSink(out, sink) : out;
+        };
+        await keep(firstPage.list);
         if (firstPage.list.length < firstPageSize) {
-            const out = {
-                ...firstPage,
-                total,
-                list: requestedSize === undefined ? collected : collected.slice(0, requestedSize),
-            };
-            if (collected.length < target) {
-                process.stderr.write(`[gangtise] warning: server returned a short page (${collected.length} rows) but reported total=${total}; treating it as the end of data — results may be incomplete\n`);
+            const out = result();
+            if (count < target) {
+                process.stderr.write(`[gangtise] warning: server returned a short page (${count} rows) but reported total=${total}; treating it as the end of data — results may be incomplete\n`);
                 out.partial = true;
             }
             return out;
         }
-        if (collected.length >= target) {
-            return {
-                ...firstPage,
-                total,
-                list: requestedSize === undefined ? collected : collected.slice(0, requestedSize),
-            };
+        if (count >= target) {
+            const out = result();
+            if (requestedSize === undefined && total > firstPageSize)
+                await this.flagIfTotalCapped(endpoint, initialBody, total, out, count);
+            return out;
         }
         const MAX_PAGES = 1000;
         let truncatedByPageCap = false;
@@ -190,10 +215,11 @@ export class GangtiseClient {
         }
         let unexpectedShape = false;
         let totalDrift = false;
+        let laterPartial = false;
         const failedPages = [];
         let firstError = null;
         let aborted = false;
-        const pages = await runWithConcurrency(pageRequests, PAGE_CONCURRENCY, async (req) => {
+        await runInOrder(pageRequests, PAGE_CONCURRENCY, async (req) => {
             if (aborted) {
                 failedPages.push(req);
                 return [];
@@ -211,6 +237,8 @@ export class GangtiseClient {
                 }
                 if (page.total !== total)
                     totalDrift = true;
+                if (page.partial === true)
+                    laterPartial = true;
                 return page.list;
             }
             catch (error) {
@@ -220,12 +248,7 @@ export class GangtiseClient {
                 failedPages.push(req);
                 return [];
             }
-        });
-        for (const list of pages) {
-            if (list.length === 0)
-                continue;
-            collected.push(...list);
-        }
+        }, (list) => keep(list));
         if (unexpectedShape) {
             process.stderr.write(`[gangtise] warning: a page response had unexpected shape; its rows are missing (counted in failedPages)\n`);
         }
@@ -233,44 +256,45 @@ export class GangtiseClient {
             process.stderr.write(`[gangtise] warning: 'total' changed across pages (data shifted during fetch); rows may be duplicated or missing\n`);
         }
         if (truncatedByPageCap) {
-            process.stderr.write(`[gangtise] warning: hit the ${MAX_PAGES}-page safety cap; fetched ${collected.length} of ${total} rows. Narrow the query (e.g. a shorter date range) or pass --size to fetch a bounded subset.\n`);
+            process.stderr.write(`[gangtise] warning: hit the ${MAX_PAGES}-page safety cap; fetched ${count} of ${total} rows. Narrow the query (e.g. a shorter date range) or pass --size to fetch a bounded subset.\n`);
         }
-        const short = collected.length < target;
-        let totalCapped = false;
+        const short = count < target;
+        const out = result();
         if (requestedSize === undefined && total > 0 && !short && !totalDrift && !truncatedByPageCap && failedPages.length === 0) {
-            try {
-                const probe = await this.requestJson(endpoint, { ...initialBody, from: total, size: 1 });
-                if (this.isPaginatedListResponse(probe) && probe.list.length > 0)
-                    totalCapped = true;
-            }
-            catch {
-            }
-        }
-        if (totalCapped) {
-            process.stderr.write(`[gangtise] warning: ${endpoint.key} reported total=${total} but rows exist past that offset — 'total' is a server-side cap, not the real count. This export is TRUNCATED at ${collected.length} rows. Narrow the query (date range / filters) and fetch in slices.\n`);
-        }
-        const out = {
-            ...firstPage,
-            total,
-            list: requestedSize === undefined ? collected : collected.slice(0, requestedSize),
-        };
-        if (totalCapped) {
-            out.partial = true;
-            out.totalCapped = true;
+            await this.flagIfTotalCapped(endpoint, initialBody, total, out, count);
         }
         if (short || totalDrift || failedPages.length > 0)
             out.partial = true;
+        if (laterPartial) {
+            out.partial = true;
+            process.stderr.write(`[gangtise] warning: a later page reported itself partial; the merged result is marked partial\n`);
+        }
         if (failedPages.length > 0) {
             out.failedPages = failedPages.map((p) => ({ from: p.from, size: p.size }));
             const detail = firstError instanceof Error ? `: ${firstError.message}` : "";
             const skippedHint = aborted ? " A page hit a non-retryable error (e.g. rate limit); remaining pages were skipped." : "";
-            process.stderr.write(`[gangtise] warning: ${failedPages.length}/${pageRequests.length} pages not fetched${detail}; results are partial — got ${collected.length}/${total} rows (see failedPages).${skippedHint}\n`);
+            process.stderr.write(`[gangtise] warning: ${failedPages.length}/${pageRequests.length} pages not fetched${detail}; results are partial — got ${count}/${total} rows (see failedPages).${skippedHint}\n`);
         }
         else if (short && !truncatedByPageCap && !totalDrift) {
-            process.stderr.write(`[gangtise] warning: server returned ${collected.length} of ${total} rows (a later page came back short); results may be incomplete\n`);
+            process.stderr.write(`[gangtise] warning: server returned ${count} of ${total} rows (a later page came back short); results may be incomplete\n`);
         }
         return out;
     }
+    async flagIfTotalCapped(endpoint, initialBody, total, out, collectedLength) {
+        let totalCapped = false;
+        try {
+            const probe = await this.requestJson(endpoint, { ...initialBody, from: total, size: 1 });
+            if (this.isPaginatedListResponse(probe) && probe.list.length > 0)
+                totalCapped = true;
+        }
+        catch {
+        }
+        if (!totalCapped)
+            return;
+        process.stderr.write(`[gangtise] warning: ${endpoint.key} reported total=${total} but rows exist past that offset — 'total' is a server-side cap, not the real count. This export is TRUNCATED at ${collectedLength} rows. Narrow the query (date range / filters) and fetch in slices.\n`);
+        out.partial = true;
+        out.totalCapped = true;
+    }
     async login() {
         const authorization = await this.getAuthorizationHeader();
         const cache = await readTokenCache(this.config.tokenCachePath);
@@ -340,7 +364,11 @@ export class GangtiseClient {
                 if (response.statusCode >= 400) {
                     this.throwHttpError(parsed, response.statusCode, retryAfterMs);
                 }
-                return this.unwrapEnvelope(parsed, response.statusCode, retryAfterMs);
+                const payload = this.unwrapEnvelope(parsed, response.statusCode, retryAfterMs);
+                if (endpoint.expects === "list" && !(payload && typeof payload === "object" && Array.isArray(payload.list))) {
+                    throw markStructural(new ApiError(`${endpoint.key} returned no list payload (got ${payload === null ? "null" : Array.isArray(payload) ? "an array" : typeof payload}) — the response layout may have changed`, undefined, response.statusCode, parsed));
+                }
+                return payload;
             }
             catch (error) {
                 await this.refreshAuthIfRecoverable(error, useAuth, authState, usedAuthorization);
@@ -519,6 +547,6 @@ export class GangtiseClient {
         if (endpoint.kind === 'json' && endpoint.pagination?.enabled) {
             return this.requestPaginated(endpoint, body);
         }
-        return this.requestJson(endpoint, body);
+        return this.requestJson(endpoint, body, endpoint.key !== 'auth.login');
     }
 }

package/dist/src/core/download.js

@@ -1,5 +1,5 @@
 import fs from "node:fs/promises";
-import { extname } from "node:path";
+import { dirname, extname } from "node:path";
 import { DownloadError } from "./errors.js";
 import { saveOutputIfNeeded } from "./output.js";
 import { extractTitles, lookupTitleCache, readTitleCache, TITLE_LOOKUP_SIZE, writeTitleCache } from "./titleCache.js";
@@ -16,18 +16,38 @@ function truncateFilename(name, maxBytes = 200) {
     return stem.join("") + ext;
 }
 export async function uniquePath(p) {
-    const exists = (f) => fs.access(f).then(() => true, () => false);
-    if (!(await exists(p)))
+    const claim = async (f) => {
+        try {
+            const handle = await fs.open(f, "wx");
+            await handle.close();
+            return true;
+        }
+        catch (error) {
+            if (error.code === "EEXIST")
+                return false;
+            throw error;
+        }
+    };
+    await fs.mkdir(dirname(p) || ".", { recursive: true });
+    if (await claim(p))
         return p;
     const ext = extname(p);
     const stem = p.slice(0, p.length - ext.length);
     for (let i = 1; i <= 99; i++) {
         const candidate = `${stem}-${i}${ext}`;
-        if (!(await exists(candidate)))
+        if (await claim(candidate))
             return candidate;
     }
     throw new DownloadError(`Refusing to overwrite: 100 files already share the name "${p}" — pass --output or clean up the directory`);
 }
+export async function releaseClaim(p) {
+    try {
+        if ((await fs.stat(p)).size === 0)
+            await fs.unlink(p);
+    }
+    catch {
+    }
+}
 const MIME_EXT = {
     "application/pdf": ".pdf",
     "application/msword": ".doc",
@@ -178,13 +198,27 @@ export async function saveDownloadResult(result, fallbackName, output) {
     if (file.data instanceof Uint8Array) {
         const autoName = (file.filename ? sanitizeFilename(file.filename) : undefined) ?? (safeFallback + extFromContentType(file.contentType));
         const outputPath = output ?? await uniquePath(truncateFilename(autoName));
-        await saveOutputIfNeeded(file.data, outputPath);
+        try {
+            await saveOutputIfNeeded(file.data, outputPath);
+        }
+        catch (error) {
+            if (!output)
+                await releaseClaim(outputPath);
+            throw error;
+        }
         process.stdout.write(`${outputPath}\n`);
         return;
     }
     if (typeof file.text === "string") {
         const outputPath = output ?? await uniquePath(truncateFilename(`${safeFallback}.txt`));
-        await saveOutputIfNeeded(file.text, outputPath);
+        try {
+            await saveOutputIfNeeded(file.text, outputPath);
+        }
+        catch (error) {
+            if (!output)
+                await releaseClaim(outputPath);
+            throw error;
+        }
         process.stdout.write(`${outputPath}\n`);
         return;
     }

package/dist/src/core/endpoints.js

@@ -273,42 +273,49 @@ const ENDPOINT_DEFS = {
         path: "/application/open-quote/kline/daily",
         kind: "json",
         description: "Query A-share historical daily kline (SH/SZ/BJ)",
+        expects: "list",
     },
     "quote.day-kline-hk": {
         method: "POST",
         path: "/application/open-quote/kline-hk/daily",
         kind: "json",
         description: "Query HK stock historical daily kline (HK)",
+        expects: "list",
     },
     "quote.day-kline-us": {
         method: "POST",
         path: "/application/open-quote/kline-us/daily",
         kind: "json",
         description: "Query US stock historical daily kline (NYSE/NASDAQ/AMEX)",
+        expects: "list",
     },
     "quote.index-day-kline": {
         method: "POST",
         path: "/application/open-quote/index/kline/daily",
         kind: "json",
         description: "Query SH/SZ/BJ index daily kline",
+        expects: "list",
     },
     "quote.minute-kline": {
         method: "POST",
         path: "/application/open-quote/kline/minute",
         kind: "json",
         description: "Query A-share minute kline (SH/SZ/BJ)",
+        expects: "list",
     },
     "quote.realtime": {
         method: "POST",
         path: "/application/open-quote/quote/realtime",
         kind: "json",
         description: "Query realtime quote snapshot (A-share / HK / US)",
+        expects: "list",
     },
     "quote.fund-flow": {
         method: "POST",
         path: "/application/open-quote/fund-flow/daily",
         kind: "json",
         description: "Query A-share daily fund flow (SH/SZ/BJ; small/medium/large/xlarge orders + main net inflow)",
+        expects: "list",
     },
     "fundamental.income-statement": {
         method: "POST",

package/dist/src/core/errors.js

@@ -17,6 +17,14 @@ export function attachEnvelopeTraceId(payload, traceId) {
     }
     return payload;
 }
+const STRUCTURAL = Symbol("gangtise.structural");
+export function markStructural(error) {
+    Object.defineProperty(error, STRUCTURAL, { value: true, enumerable: false, configurable: true });
+    return error;
+}
+export function isStructuralError(error) {
+    return Boolean(error && typeof error === "object" && error[STRUCTURAL] === true);
+}
 const ERROR_HINTS = {
     "999001": "检查 GANGTISE_TOKEN 或 AK/SK 是否已 export。",
     "999002": "有 AK/SK 时 CLI 会自动重新登录重试一次，否则请重新登录。",

package/dist/src/core/normalize.js

@@ -14,6 +14,37 @@ export function zipFieldRow(fields, row, source) {
         return acc;
     }, {});
 }
+export function columnarSchemaValid(fields, list) {
+    if (!fields)
+        return false;
+    if (new Set(fields.map(String)).size !== fields.length)
+        return false;
+    return list.every((row) => !Array.isArray(row) || row.length === fields.length);
+}
+export function flagMissingFields(data, requested, label) {
+    if (!requested?.length || !data || typeof data !== "object" || Array.isArray(data))
+        return;
+    const rec = data;
+    if (!Array.isArray(rec.fieldList))
+        return;
+    const returned = new Set(rec.fieldList.map(String));
+    const missing = requested.filter((field) => !returned.has(field));
+    if (missing.length === 0)
+        return;
+    rec.partial = true;
+    rec.missingFields = missing;
+    process.stderr.write(`[gangtise] warning: ${label} returned no column for ${missing.join(", ")} — the server drops a field name it does not recognise (or no longer serves) without an error. Check the name against references/fields.md; result marked partial (exit 3).\n`);
+}
+export function assertColumnarHeader(fields, source) {
+    if (!fields) {
+        throw new ValidationError(`响应包含数组形式的行但没有 fieldList，无法确定各列的含义，已拒绝输出。这是上游响应结构异常，请报障${traceSuffix(source)}。`);
+    }
+    const names = fields.map(String);
+    const dupes = [...new Set(names.filter((name, i) => names.indexOf(name) !== i))];
+    if (dupes.length > 0) {
+        throw new ValidationError(`响应 fieldList 有重复列名（${dupes.join(", ")}）——按位置拍平时后一列会覆盖前一列，已拒绝输出。这是上游响应结构异常，请报障${traceSuffix(source)}。`);
+    }
+}
 export function normalizeRows(value) {
     if (!value || typeof value !== "object") {
         return value;
@@ -24,12 +55,16 @@ export function normalizeRows(value) {
     const record = value;
     if (Array.isArray(record.fieldList) && Array.isArray(record.list)) {
         const fields = record.fieldList;
+        if (record.list.some(Array.isArray))
+            assertColumnarHeader(fields, record);
         const normalizedList = record.list.map((row) => (Array.isArray(row) ? zipFieldRow(fields, row, record) : row));
         const { fieldList, list, ...meta } = record;
         const hasMeta = Object.keys(meta).length > 0;
         return hasMeta ? { ...meta, list: normalizedList } : normalizedList;
     }
     if (Array.isArray(record.list)) {
+        if (record.list.some(Array.isArray))
+            assertColumnarHeader(undefined, record);
         const { list, ...meta } = record;
         const hasMeta = Object.keys(meta).length > 0;
         return hasMeta ? { ...meta, list } : list;

package/dist/src/core/output.js

@@ -24,7 +24,7 @@ function displayWidth(value) {
     }
     return width;
 }
-function formatScalar(value) {
+export function formatScalar(value) {
     if (value === null || value === undefined) {
         return "";
     }
@@ -104,16 +104,19 @@ function renderCsv(rows) {
     const body = rows.map((row) => columns.map((column) => csvEscape(formatScalar(row[column]))).join(","));
     return [header, ...body].join("\n");
 }
+export function jsonlItems(value) {
+    const list = value && typeof value === "object" && !Array.isArray(value) ? value.list : undefined;
+    return Array.isArray(list) ? list : toRows(value);
+}
+export function countOutputRows(value, format) {
+    return format === "jsonl" ? jsonlItems(value).length : toRows(value).length;
+}
 export function renderOutput(value, format) {
     switch (format) {
         case "json":
             return JSON.stringify(value, null, 2);
-        case "jsonl": {
-            const items = value && typeof value === "object" && !Array.isArray(value) && Array.isArray(value.list)
-                ? value.list
-                : null;
-            return (items ?? toRows(value)).map((item) => JSON.stringify(item)).join("\n");
-        }
+        case "jsonl":
+            return jsonlItems(value).map((item) => JSON.stringify(item)).join("\n");
         case "csv":
             return renderCsv(toRows(value));
         case "markdown":
@@ -147,7 +150,7 @@ export async function streamOutputToFile(value, format, outputPath) {
     stream.on("error", () => { });
     try {
         if (format === "jsonl") {
-            for (const item of list) {
+            for (const item of jsonlItems(value)) {
                 await writeLine(stream, JSON.stringify(item));
             }
         }
@@ -183,7 +186,7 @@ export function pickList(value) {
     }
     return null;
 }
-function csvEscape(value) {
+export function csvEscape(value) {
     let out = value;
     if (/^[=@\t\r]/.test(out) || (/^[+\-]/.test(out) && !Number.isFinite(Number(out))))
         out = "'" + out;
@@ -191,7 +194,7 @@ function csvEscape(value) {
         return `"${out.replaceAll("\"", "\"\"")}"`;
     return out;
 }
-function writeLine(stream, line) {
+export function writeLine(stream, line) {
     return new Promise((resolve, reject) => {
         const ok = stream.write(line + "\n", (err) => err ? reject(err) : undefined);
         if (ok) {

package/dist/src/core/perSecurity.js

@@ -0,0 +1,87 @@
+import { ApiError, markStructural } from "./errors.js";
+import { columnarSchemaValid } from "./normalize.js";
+import { attachRowSink } from "./rowSink.js";
+import { PAGE_CONCURRENCY, runInOrder } from "./transport.js";
+export function estimateTradingDays(startDate, endDate) {
+    if (!startDate)
+        return 262;
+    const start = Date.parse(`${startDate}T00:00:00Z`);
+    const end = endDate ? Date.parse(`${endDate}T00:00:00Z`) : Date.now() + 86_400_000;
+    if (Number.isNaN(start) || Number.isNaN(end) || end < start)
+        return 262;
+    let days = 0;
+    for (let t = start; t <= end; t += 86_400_000) {
+        const weekday = new Date(t).getUTCDay();
+        if (weekday !== 0 && weekday !== 6)
+            days++;
+    }
+    return days;
+}
+export async function callPerSecurity(client, endpointKey, securities, makeBody, cap, label) {
+    const sink = client.claimRowSink?.();
+    let fieldList;
+    let headerSecurity;
+    let emptyFields;
+    const merged = [];
+    let count = 0;
+    const truncated = [];
+    let partial = false;
+    const mergePart = async (part, i) => {
+        const rec = part;
+        if (!(rec && typeof rec === "object" && Array.isArray(rec.list))) {
+            throw markStructural(new ApiError(`${label}: ${securities[i]} returned no list payload — the response layout may have changed`, undefined, undefined, rec));
+        }
+        if (rec.partial === true)
+            partial = true;
+        if (rec.list.length === 0) {
+            if ((typeof rec.total === "number" && rec.total > 0) || rec.partial === true) {
+                const claim = typeof rec.total === "number" && rec.total > 0 ? `reported total=${rec.total}` : "carried a partial marker";
+                throw markStructural(new ApiError(`${label}: ${securities[i]} ${claim} but delivered no rows — the response is inconsistent`, undefined, undefined, rec));
+            }
+            if (!emptyFields && Array.isArray(rec.fieldList))
+                emptyFields = rec.fieldList;
+            return;
+        }
+        const fields = Array.isArray(rec.fieldList) && rec.fieldList.length > 0 ? rec.fieldList : undefined;
+        if (rec.list.some(Array.isArray) && !columnarSchemaValid(fields, rec.list)) {
+            throw markStructural(new ApiError(`${label}: ${securities[i]} returned columnar rows without a usable fieldList (missing, duplicated or mis-sized) — they cannot be read by position`, undefined, undefined, rec));
+        }
+        if (fields && !fieldList) {
+            fieldList = fields;
+            headerSecurity = securities[i];
+            sink?.setFieldList(fieldList);
+        }
+        if ((fields ?? fieldList) && !sameColumns(fieldList, fields)) {
+            throw markStructural(new ApiError(`${label}: ${securities[i]} answered with columns ${JSON.stringify(fields)} while ${headerSecurity} answered ${JSON.stringify(fieldList)} — the parts cannot be merged`, undefined, undefined, rec));
+        }
+        if (rec.list.length >= cap)
+            truncated.push(securities[i]);
+        count += rec.list.length;
+        if (sink)
+            await sink.push(rec.list);
+        else
+            for (const row of rec.list)
+                merged.push(row);
+    };
+    await runInOrder(securities, PAGE_CONCURRENCY, (code) => client.call(endpointKey, makeBody(code)), mergePart);
+    const out = { total: count, list: merged };
+    if (sink)
+        attachRowSink(out, sink);
+    if (fieldList)
+        out.fieldList = fieldList;
+    else if (count === 0 && emptyFields)
+        out.fieldList = emptyFields;
+    if (partial)
+        out.partial = true;
+    if (truncated.length > 0) {
+        out.partial = true;
+        out.truncatedSecurities = truncated;
+        process.stderr.write(`[gangtise] warning: ${label}: ${truncated.join(", ")} returned ${cap} rows = the per-request limit; those securities are likely truncated (see truncatedSecurities). Narrow the date range for them or raise --limit (max 10000).\n`);
+    }
+    return out;
+}
+function sameColumns(a, b) {
+    if (!a || !b)
+        return false;
+    return a.length === b.length && a.every((field, i) => field === b[i]);
+}