gangtise-openapi-cli 0.20.0 → 0.22.0

package/dist/src/core/client.js

@@ -7,8 +7,7 @@ import { isTokenCacheValid, normalizeToken, readTokenCache, requireAccessCredent
 import { ApiError, ValidationError } from "./errors.js";
 import { ENDPOINTS } from "./endpoints.js";
 import { getLookupData } from "./lookupData/index.js";
-import { getDispatcher, isVerbose, logTiming, markRetryable, runWithConcurrency, withRetry } from "./transport.js";
-const PAGINATION_CONCURRENCY = Number(process.env.GANGTISE_PAGE_CONCURRENCY ?? 5) || 5;
+import { getDispatcher, isVerbose, logTiming, markRetryable, PAGE_CONCURRENCY, runWithConcurrency, withRetry } from "./transport.js";
 // Auth errors that warrant a forced re-login + one replay. 8000014/8000015 are
 // AK/SK errors; 0000001008 is a server-side token invalidation (the token still
 // looks valid by local expiry, so only a forced refresh recovers it).
@@ -48,11 +47,28 @@ export class GangtiseClient {
             accessKey: credentials.accessKey,
             secretKey: credentials.secretKey,
         }, false);
+        // Validate the shape before touching it: a missing accessToken used to surface
+        // as a bare TypeError from normalizeToken, hiding the real cause.
+        if (typeof envelope?.accessToken !== "string" || !envelope.accessToken) {
+            throw new ApiError("Login succeeded but the response carried no accessToken", undefined, undefined, envelope);
+        }
         const accessToken = normalizeToken(envelope.accessToken);
-        const expiresAt = Math.floor(Date.now() / 1000) + envelope.expiresIn;
-        const cache = { ...envelope, accessToken, expiresAt };
+        // A non-numeric expiresIn would make expiresAt NaN — the cache would never
+        // validate and every command would silently re-login. Degrade to 0 instead
+        // (token works now, next process logs in again).
+        const expiresIn = Number.isFinite(envelope.expiresIn) ? envelope.expiresIn : 0;
+        const expiresAt = Math.floor(Date.now() / 1000) + expiresIn;
+        const cache = { ...envelope, accessToken, expiresIn, expiresAt };
         this.memoCache = cache;
-        await writeTokenCache(this.config.tokenCachePath, cache);
+        try {
+            await writeTokenCache(this.config.tokenCachePath, cache);
+        }
+        catch (error) {
+            // A read-only HOME or full disk must not fail the request — the in-memory
+            // token is valid; we just can't persist it for the next process.
+            const msg = error instanceof Error ? error.message : String(error);
+            process.stderr.write(`[gangtise] warning: could not persist token cache: ${msg}\n`);
+        }
         return accessToken;
     }
     /**
@@ -62,7 +78,7 @@ export class GangtiseClient {
      * the caller re-throws the original error. `authState` persists across the
      * withRetry attempts so we only refresh once per logical request.
      */
-    async refreshAuthIfRecoverable(error, useAuth, authState) {
+    async refreshAuthIfRecoverable(error, useAuth, authState, usedAuthorization) {
         if (useAuth
             && !authState.retried
             && error instanceof ApiError
@@ -71,12 +87,30 @@ export class GangtiseClient {
             && this.config.accessKey
             && this.config.secretKey) {
             authState.retried = true;
-            this.memoCache = null;
             this.envTokenInvalidated = true;
-            await this.getAuthorizationHeader(true);
+            // If the failed request was still carrying an OLDER token than the one now in
+            // memoCache, another request already refreshed — replay with the fresh token
+            // instead of logging in again (back-to-back logins can kick each other's
+            // sessions server-side, the 0000001008 semantics). If the failed request used
+            // the CURRENT token, that token is genuinely dead: force a new login. A time
+            // window is NOT a valid proxy here — right after the initial login the window
+            // is always "recent", which would skip the refresh exactly when it's needed.
+            const memoToken = this.memoCache && isTokenCacheValid(this.memoCache) ? normalizeToken(this.memoCache.accessToken) : null;
+            const alreadyRefreshed = memoToken !== null && usedAuthorization !== undefined && usedAuthorization !== memoToken;
+            if (!alreadyRefreshed) {
+                this.memoCache = null;
+                await this.getAuthorizationHeader(true);
+            }
             throw markRetryable(new ApiError(error.message, error.code, error.statusCode, error.details));
         }
     }
+    /** `new URL("/a/b", "https://proxy/prefix")` drops "/prefix" — an absolute path
+     * replaces the base's path per the URL spec. Join manually so a reverse-proxy
+     * GANGTISE_BASE_URL with a path prefix keeps working. */
+    buildUrl(path) {
+        const base = this.config.baseUrl.endsWith("/") ? this.config.baseUrl : `${this.config.baseUrl}/`;
+        return new URL(path.replace(/^\//, ""), base);
+    }
     isEnvelope(parsed) {
         if (!parsed || typeof parsed !== 'object')
             return false;
@@ -145,16 +179,25 @@ export class GangtiseClient {
             return firstPage;
         const total = firstPage.total;
         const collected = [...firstPage.list];
-        // Last page reached on first request
+        const available = Math.max(total - startFrom, 0);
+        const target = requestedSize === undefined ? available : Math.min(requestedSize, available);
+        // Last page reached on first request. If `total` promises more rows than the
+        // short page delivered, the server's page cap may be lower than our configured
+        // maxPageSize — say so instead of silently returning a subset as "everything".
         if (firstPage.list.length < firstPageSize) {
-            return {
+            const out = {
                 ...firstPage,
                 total,
                 list: requestedSize === undefined ? collected : collected.slice(0, requestedSize),
             };
+            if (collected.length < target) {
+                process.stderr.write(`[gangtise] warning: server returned a short page (${collected.length} rows) but reported total=${total}; treating it as the end of data — results may be incomplete\n`);
+                // Machine-readable counterpart of the warning: scripts key off partial /
+                // exit code 3, and must not mistake a truncated result for a complete one.
+                out.partial = true;
+            }
+            return out;
         }
-        const available = Math.max(total - startFrom, 0);
-        const target = requestedSize === undefined ? available : Math.min(requestedSize, available);
         if (collected.length >= target) {
             return {
                 ...firstPage,
@@ -162,21 +205,24 @@ export class GangtiseClient {
                 list: requestedSize === undefined ? collected : collected.slice(0, requestedSize),
             };
         }
+        // Build remaining page requests. The cap lives inside the loop: a corrupt
+        // server `total` (e.g. 9e15) must not materialize millions of page objects —
+        // or spin for minutes — before a post-hoc truncation applies.
+        const MAX_PAGES = 1000;
+        let truncatedByPageCap = false;
         const pageRequests = [];
         let nextFrom = startFrom + firstPage.list.length;
         const endFrom = startFrom + target;
         while (nextFrom < endFrom) {
+            if (pageRequests.length + 1 >= MAX_PAGES) {
+                truncatedByPageCap = true;
+                break;
+            }
             const remaining = endFrom - nextFrom;
             const size = Math.min(maxPageSize, remaining);
             pageRequests.push({ from: nextFrom, size });
             nextFrom += size;
         }
-        const MAX_PAGES = 1000;
-        let truncatedByPageCap = false;
-        if (pageRequests.length + 1 > MAX_PAGES) {
-            pageRequests.length = MAX_PAGES - 1;
-            truncatedByPageCap = true;
-        }
         let unexpectedShape = false;
         let totalDrift = false;
         // Fail-soft fan-out: a hard page failure (rate-limit 903301, no-perm, retries
@@ -187,7 +233,7 @@ export class GangtiseClient {
         const failedPages = [];
         let firstError = null;
         let aborted = false;
-        const pages = await runWithConcurrency(pageRequests, PAGINATION_CONCURRENCY, async (req) => {
+        const pages = await runWithConcurrency(pageRequests, PAGE_CONCURRENCY, async (req) => {
             if (aborted) {
                 failedPages.push(req);
                 return [];
@@ -199,7 +245,10 @@ export class GangtiseClient {
                     size: req.size,
                 });
                 if (!this.isPaginatedListResponse(page)) {
+                    // Treat a shape-broken page like a failed page: its rows are missing, so
+                    // the result must carry the partial marker instead of looking complete.
                     unexpectedShape = true;
+                    failedPages.push(req);
                     return [];
                 }
                 if (page.total !== total)
@@ -219,11 +268,11 @@ export class GangtiseClient {
                 continue;
             collected.push(...list);
         }
-        if (unexpectedShape && isVerbose()) {
-            process.stderr.write(`[gangtise] warning: a page response had unexpected shape; results may be incomplete\n`);
+        if (unexpectedShape) {
+            process.stderr.write(`[gangtise] warning: a page response had unexpected shape; its rows are missing (counted in failedPages)\n`);
         }
-        if (totalDrift && isVerbose()) {
-            process.stderr.write(`[gangtise] warning: 'total' changed across pages (data shifted during fetch)\n`);
+        if (totalDrift) {
+            process.stderr.write(`[gangtise] warning: 'total' changed across pages (data shifted during fetch); rows may be duplicated or missing\n`);
         }
         // Always surface a cap-induced truncation (not gated on verbose): the user
         // asked for everything and is silently getting a subset, mirroring the
@@ -240,8 +289,78 @@ export class GangtiseClient {
             out.partial = true;
             out.failedPages = failedPages.map((p) => ({ from: p.from, size: p.size }));
             const detail = firstError instanceof Error ? `: ${firstError.message}` : "";
-            process.stderr.write(`[gangtise] warning: ${failedPages.length}/${pageRequests.length} pages not fetched${detail}; results are partial — got ${collected.length}/${total} rows (see failedPages). A page hit a non-retryable error (e.g. rate limit); remaining pages were skipped.\n`);
+            const skippedHint = aborted ? " A page hit a non-retryable error (e.g. rate limit); remaining pages were skipped." : "";
+            process.stderr.write(`[gangtise] warning: ${failedPages.length}/${pageRequests.length} pages not fetched${detail}; results are partial — got ${collected.length}/${total} rows (see failedPages).${skippedHint}\n`);
+        }
+        return out;
+    }
+    /**
+     * Sequential pagination for endpoints that page by offset but return NO `total`
+     * and use a non-standard list key (e.g. wechat chatroom's `chatRoomList`). We
+     * can't fan out like requestPaginated (no total ⇒ unknown page count), so page
+     * serially until a short page (fewer rows than requested) signals the end.
+     * Returns `{ [listKey]: rows }` so normalize/printer treat it like any list.
+     */
+    async requestSequentialPaginated(endpoint, body) {
+        const initialBody = body && typeof body === 'object' ? { ...body } : {};
+        if ('from' in initialBody && (typeof initialBody.from !== 'number' || !Number.isFinite(initialBody.from) || initialBody.from < 0)) {
+            throw new ValidationError('Invalid from: expected a non-negative number');
+        }
+        if ('size' in initialBody && initialBody.size !== undefined && (typeof initialBody.size !== 'number' || !Number.isFinite(initialBody.size) || initialBody.size <= 0)) {
+            throw new ValidationError('Invalid size: expected a positive number');
+        }
+        const listKey = endpoint.pagination?.listKey ?? 'list';
+        const maxPageSize = endpoint.pagination?.maxPageSize ?? 50;
+        const startFrom = typeof initialBody.from === 'number' && Number.isFinite(initialBody.from) ? initialBody.from : 0;
+        const requestedSize = typeof initialBody.size === 'number' && Number.isFinite(initialBody.size) ? initialBody.size : undefined;
+        const extractList = (page) => {
+            if (!page || typeof page !== 'object')
+                return null;
+            const arr = page[listKey];
+            return Array.isArray(arr) ? arr : null;
+        };
+        const collected = [];
+        let firstPage = null;
+        let from = startFrom;
+        const MAX_PAGES = 1000;
+        let truncatedByPageCap = false;
+        let partialShape = false;
+        for (let page = 0;; page++) {
+            const remaining = requestedSize === undefined ? maxPageSize : requestedSize - collected.length;
+            if (requestedSize !== undefined && remaining <= 0)
+                break;
+            const size = Math.min(maxPageSize, remaining);
+            const pageData = await this.requestJson(endpoint, { ...initialBody, from, size });
+            if (firstPage === null)
+                firstPage = pageData;
+            const list = extractList(pageData);
+            if (list === null) {
+                // First response isn't a paginated-list shape → return it untouched. But a
+                // LATER page losing shape must NOT discard the rows already collected (mirrors
+                // requestPaginated's fail-soft fan-out): stop, keep them, and warn loudly.
+                if (page === 0)
+                    return firstPage;
+                partialShape = true;
+                process.stderr.write(`[gangtise] warning: a page response had unexpected shape; results are partial — ${collected.length} rows fetched.\n`);
+                break;
+            }
+            for (const item of list)
+                collected.push(item);
+            if (list.length < size)
+                break; // short page ⇒ no more rows
+            if (page + 1 >= MAX_PAGES) {
+                truncatedByPageCap = true;
+                break;
+            }
+            from += list.length;
         }
+        if (truncatedByPageCap) {
+            process.stderr.write(`[gangtise] warning: hit the ${MAX_PAGES}-page safety cap; fetched ${collected.length} rows. Pass --size to fetch a bounded subset.\n`);
+        }
+        const rows = requestedSize === undefined ? collected : collected.slice(0, requestedSize);
+        const out = { [listKey]: rows };
+        if (partialShape)
+            out.partial = true;
         return out;
     }
     async login() {
@@ -257,14 +376,18 @@ export class GangtiseClient {
             return this.readLocalLookup(endpoint);
         }
         const dispatcher = getDispatcher();
-        const url = new URL(endpoint.path, this.config.baseUrl);
+        const url = this.buildUrl(endpoint.path);
         const authState = { retried: false };
         return withRetry(async () => {
             const headers = {
                 'content-type': 'application/json',
             };
+            // Keep the header we actually sent: the self-heal check compares it against
+            // the current memoCache token to tell "stale token" from "fresh token died".
+            let usedAuthorization;
             if (useAuth) {
-                headers.Authorization = await this.getAuthorizationHeader();
+                usedAuthorization = await this.getAuthorizationHeader();
+                headers.Authorization = usedAuthorization;
             }
             const startedAt = Date.now();
             const response = await request(url, {
@@ -287,14 +410,16 @@ export class GangtiseClient {
                     : 'Failed to parse API response';
                 throw new ApiError(message, undefined, response.statusCode, text.slice(0, 500));
             }
-            if (response.statusCode >= 400) {
-                this.throwHttpError(parsed, response.statusCode);
-            }
             try {
+                // Auth errors can arrive as HTTP 4xx or as a 200-wrapped error envelope;
+                // both routes must reach the self-heal check below.
+                if (response.statusCode >= 400) {
+                    this.throwHttpError(parsed, response.statusCode);
+                }
                 return this.unwrapEnvelope(parsed, response.statusCode);
             }
             catch (error) {
-                await this.refreshAuthIfRecoverable(error, useAuth, authState);
+                await this.refreshAuthIfRecoverable(error, useAuth, authState, usedAuthorization);
                 throw error;
             }
         }, {
@@ -308,7 +433,7 @@ export class GangtiseClient {
     }
     async download(endpoint, query, options) {
         const dispatcher = getDispatcher();
-        const url = new URL(endpoint.path, this.config.baseUrl);
+        const url = this.buildUrl(endpoint.path);
         Object.entries(query).forEach(([key, value]) => {
             url.searchParams.set(key, String(value));
         });
@@ -316,13 +441,37 @@ export class GangtiseClient {
         return withRetry(async () => {
             const authorization = await this.getAuthorizationHeader();
             const startedAt = Date.now();
-            const response = await request(url, {
+            let currentUrl = url;
+            let auth = authorization;
+            let response = await request(currentUrl, {
                 method: endpoint.method,
                 headers: { Authorization: authorization },
                 headersTimeout: this.config.timeoutMs,
                 bodyTimeout: this.config.timeoutMs,
                 dispatcher,
             });
+            // undici does not follow redirects, and a download endpoint may 302 to a
+            // pre-signed object-store URL — without this the redirect body would be
+            // saved as the "file". Follow up to 3 hops, dropping Authorization once the
+            // redirect leaves the API origin so the bearer never reaches storage hosts.
+            for (let hops = 0; hops < 3 && response.statusCode >= 300 && response.statusCode < 400; hops++) {
+                const locationHeader = response.headers.location;
+                const location = Array.isArray(locationHeader) ? locationHeader[0] : locationHeader;
+                if (!location)
+                    break;
+                await response.body.text().catch(() => { });
+                const next = new URL(location, currentUrl);
+                if (next.origin !== currentUrl.origin)
+                    auth = undefined;
+                currentUrl = next;
+                response = await request(currentUrl, {
+                    method: 'GET',
+                    headers: auth ? { Authorization: auth } : {},
+                    headersTimeout: this.config.timeoutMs,
+                    bodyTimeout: this.config.timeoutMs,
+                    dispatcher,
+                });
+            }
             const contentType = Array.isArray(response.headers['content-type']) ? response.headers['content-type'][0] : response.headers['content-type'];
             if (contentType?.includes('application/json')) {
                 const text = await response.body.text();
@@ -337,15 +486,15 @@ export class GangtiseClient {
                     }
                     return { text, contentType };
                 }
-                if (response.statusCode >= 400) {
-                    this.throwHttpError(parsed, response.statusCode);
-                }
                 let data;
                 try {
+                    if (response.statusCode >= 400) {
+                        this.throwHttpError(parsed, response.statusCode);
+                    }
                     data = this.unwrapEnvelope(parsed, response.statusCode);
                 }
                 catch (error) {
-                    await this.refreshAuthIfRecoverable(error, true, authState);
+                    await this.refreshAuthIfRecoverable(error, true, authState, authorization);
                     throw error;
                 }
                 if (data && typeof data === 'object' && 'url' in data && typeof data.url === 'string') {
@@ -369,7 +518,19 @@ export class GangtiseClient {
             const filenameMatch = Array.isArray(contentDisposition)
                 ? contentDisposition[0]?.match(/filename\*=UTF-8''([^;]+)|filename="?([^";]+)"?/i)
                 : contentDisposition?.match(/filename\*=UTF-8''([^;]+)|filename="?([^";]+)"?/i);
-            const filename = filenameMatch ? decodeURIComponent(filenameMatch[1] || filenameMatch[2]) : undefined;
+            // A plain filename= value with a bare % ("增长100%.pdf") is not valid URI
+            // encoding — decodeURIComponent would throw and fail the whole download over
+            // a cosmetic hint. Fall back to the raw value instead.
+            let filename;
+            if (filenameMatch) {
+                const raw = filenameMatch[1] || filenameMatch[2];
+                try {
+                    filename = decodeURIComponent(raw);
+                }
+                catch {
+                    filename = raw;
+                }
+            }
             // Stream directly to disk when caller already knows the destination
             if (options?.streamTo) {
                 await fs.mkdir(path.dirname(options.streamTo), { recursive: true });
@@ -411,7 +572,9 @@ export class GangtiseClient {
             return this.download(endpoint, query ?? {}, options);
         }
         if (endpoint.kind === 'json' && endpoint.pagination?.enabled) {
-            return this.requestPaginated(endpoint, body);
+            return endpoint.pagination.sequential
+                ? this.requestSequentialPaginated(endpoint, body)
+                : this.requestPaginated(endpoint, body);
         }
         return this.requestJson(endpoint, body);
     }

package/dist/src/core/download.js

@@ -1,12 +1,44 @@
+import fs from "node:fs/promises";
 import { extname } from "node:path";
 import { DownloadError } from "./errors.js";
 import { saveOutputIfNeeded } from "./output.js";
 import { lookupTitleCache, readTitleCache, TITLE_LOOKUP_SIZE } from "./titleCache.js";
-/** Replace filesystem-unsafe characters with `_` so a title or a server-supplied
- * filename can't create stray subdirectories or escape the intended output path.
+/** Replace filesystem-unsafe characters (path separators, wildcards, and control
+ * characters / NUL) with `_` so a title or a server-supplied filename can't create
+ * stray subdirectories, escape the intended output path, or break fs.writeFile.
  * Shared by title-based naming and the download fallback. */
 function sanitizeFilename(name) {
-    return name.replace(/[/\\:*?"<>|]/g, "_");
+    return name.replace(/[/\\:*?"<>|\u0000-\u001f]/g, "_");
+}
+/** Keep auto-derived filenames under 200 UTF-8 bytes, preserving the extension.
+ * ext4 caps directory entries at 255 bytes — long Chinese titles hit that at
+ * ~85 chars (3 bytes each) and fs.writeFile throws ENAMETOOLONG after the body
+ * has already been downloaded. */
+function truncateFilename(name, maxBytes = 200) {
+    if (Buffer.byteLength(name, "utf8") <= maxBytes)
+        return name;
+    const ext = extname(name);
+    let stem = name.slice(0, name.length - ext.length);
+    while (stem.length > 1 && Buffer.byteLength(stem + ext, "utf8") > maxBytes)
+        stem = stem.slice(0, -1);
+    return stem + ext;
+}
+/** Pick a non-existing path by suffixing -1, -2, … before the extension, so batch
+ * downloads whose titles collide ("2025年第一季度报告" from several companies) don't
+ * silently overwrite each other. Only auto-derived names go through this — an
+ * explicit --output path keeps plain overwrite semantics. */
+export async function uniquePath(p) {
+    const exists = (f) => fs.access(f).then(() => true, () => false);
+    if (!(await exists(p)))
+        return p;
+    const ext = extname(p);
+    const stem = p.slice(0, p.length - ext.length);
+    for (let i = 1; i <= 99; i++) {
+        const candidate = `${stem}-${i}${ext}`;
+        if (!(await exists(candidate)))
+            return candidate;
+    }
+    return p;
 }
 const MIME_EXT = {
     "application/pdf": ".pdf",
@@ -48,7 +80,7 @@ export async function resolveTitle(client, result, listEndpoint, idField, idValu
         if (serverExt && !title.toLowerCase().endsWith(serverExt.toLowerCase())) {
             title += serverExt;
         }
-        return title;
+        return truncateFilename(title);
     }
     try {
         const cacheData = await readTitleCache();
@@ -72,11 +104,32 @@ export async function resolveTitle(client, result, listEndpoint, idField, idValu
     }
     return undefined;
 }
+async function downloadUrlTo(url, outputPath) {
+    const { createWriteStream } = await import("node:fs");
+    const { Readable } = await import("node:stream");
+    const { pipeline } = await import("node:stream/promises");
+    const { dirname } = await import("node:path");
+    const response = await fetch(url);
+    if (!response.ok || !response.body) {
+        throw new DownloadError(`Failed to fetch download URL (HTTP ${response.status})`);
+    }
+    await fs.mkdir(dirname(outputPath), { recursive: true });
+    try {
+        await pipeline(Readable.fromWeb(response.body), createWriteStream(outputPath));
+    }
+    catch (error) {
+        await fs.unlink(outputPath).catch(() => { });
+        throw error;
+    }
+}
 export async function saveDownloadResult(result, fallbackName, output) {
     if (!(result && typeof result === "object")) {
         throw new DownloadError("Unexpected download response");
     }
     const file = result;
+    // The fallback prefix embeds a user-supplied id (e.g. --report-id): sanitize it
+    // like server-provided names so an auto filename never interprets input as a path.
+    const safeFallback = sanitizeFilename(fallbackName);
     if (typeof file.savedPath === "string") {
         process.stdout.write(`${file.savedPath}\n`);
         return;
@@ -84,20 +137,24 @@ export async function saveDownloadResult(result, fallbackName, output) {
     if (file.data instanceof Uint8Array) {
         // Sanitize the server-provided filename so a Content-Disposition value with
         // / or : can't write outside the intended path (same rule as buildFilename).
-        const outputPath = output ?? (file.filename ? sanitizeFilename(file.filename) : undefined) ?? (fallbackName + extFromContentType(file.contentType));
+        const autoName = (file.filename ? sanitizeFilename(file.filename) : undefined) ?? (safeFallback + extFromContentType(file.contentType));
+        const outputPath = output ?? await uniquePath(truncateFilename(autoName));
         await saveOutputIfNeeded(file.data, outputPath);
         process.stdout.write(`${outputPath}\n`);
         return;
     }
     if (typeof file.text === "string") {
-        const outputPath = output ?? `${fallbackName}.txt`;
+        const outputPath = output ?? await uniquePath(truncateFilename(`${safeFallback}.txt`));
         await saveOutputIfNeeded(file.text, outputPath);
         process.stdout.write(`${outputPath}\n`);
         return;
     }
     if (typeof file.url === "string") {
         if (output) {
-            await saveOutputIfNeeded(file.url, output);
+            // The server handed us a (typically signed, short-lived) URL instead of the
+            // bytes. The user asked for a file — follow the URL and stream the content
+            // to disk instead of writing the URL string into a fake .pdf.
+            await downloadUrlTo(file.url, output);
             process.stdout.write(`${output}\n`);
             return;
         }