gangtise-openapi-cli 0.11.0 → 0.12.0

package/dist/src/core/client.js

@@ -1,21 +1,35 @@
+import { createWriteStream } from "node:fs";
+import fs from "node:fs/promises";
+import path from "node:path";
+import { pipeline } from "node:stream/promises";
 import { request } from "undici";
 import { isTokenCacheValid, normalizeToken, readTokenCache, requireAccessCredentials, writeTokenCache } from "./auth.js";
 import { ApiError, ValidationError } from "./errors.js";
-import { ENDPOINTS, ENDPOINT_REGISTRY } from "./endpoints.js";
+import { ENDPOINTS } from "./endpoints.js";
 import { getLookupData } from "./lookupData/index.js";
+import { getDispatcher, isVerbose, logTiming, markRetryable, runWithConcurrency, withRetry } from "./transport.js";
+const PAGINATION_CONCURRENCY = Number(process.env.GANGTISE_PAGE_CONCURRENCY ?? 5) || 5;
+const AUTH_RETRY_CODES = new Set(["8000014", "8000015"]);
 export class GangtiseClient {
     config;
     refreshPromise = null;
+    memoCache = null;
     constructor(config) {
         this.config = config;
     }
-    async getAuthorizationHeader() {
-        if (this.config.token) {
+    async getAuthorizationHeader(forceRefresh = false) {
+        if (this.config.token && !forceRefresh) {
             return normalizeToken(this.config.token);
         }
-        const cache = await readTokenCache(this.config.tokenCachePath);
-        if (isTokenCacheValid(cache)) {
-            return normalizeToken(cache.accessToken);
+        if (!forceRefresh) {
+            if (isTokenCacheValid(this.memoCache)) {
+                return normalizeToken(this.memoCache.accessToken);
+            }
+            const cache = await readTokenCache(this.config.tokenCachePath);
+            if (isTokenCacheValid(cache)) {
+                this.memoCache = cache;
+                return normalizeToken(cache.accessToken);
+            }
         }
         if (!this.refreshPromise) {
             this.refreshPromise = this.doTokenRefresh().finally(() => { this.refreshPromise = null; });
@@ -24,21 +38,24 @@ export class GangtiseClient {
     }
     async doTokenRefresh() {
         const credentials = requireAccessCredentials(this.config.accessKey, this.config.secretKey);
-        const envelope = await this.requestJson(ENDPOINTS.authLogin, {
+        const envelope = await this.requestJson(ENDPOINTS["auth.login"], {
             accessKey: credentials.accessKey,
             secretKey: credentials.secretKey,
         }, false);
         const accessToken = normalizeToken(envelope.accessToken);
         const expiresAt = Math.floor(Date.now() / 1000) + envelope.expiresIn;
-        await writeTokenCache(this.config.tokenCachePath, {
-            ...envelope,
-            accessToken,
-            expiresAt,
-        });
+        const cache = { ...envelope, accessToken, expiresAt };
+        this.memoCache = cache;
+        await writeTokenCache(this.config.tokenCachePath, cache);
         return accessToken;
     }
     isEnvelope(parsed) {
-        return Boolean(parsed && typeof parsed === 'object' && 'code' in parsed);
+        if (!parsed || typeof parsed !== 'object')
+            return false;
+        const obj = parsed;
+        if (!('code' in obj))
+            return false;
+        return 'msg' in obj || 'data' in obj || 'success' in obj || 'status' in obj;
     }
     throwHttpError(parsed, statusCode) {
         if (this.isEnvelope(parsed)) {
@@ -95,55 +112,73 @@ export class GangtiseClient {
         const startFrom = typeof initialBody.from === 'number' && Number.isFinite(initialBody.from) ? initialBody.from : 0;
         const requestedSize = typeof initialBody.size === 'number' && Number.isFinite(initialBody.size) ? initialBody.size : undefined;
         const maxPageSize = endpoint.pagination?.maxPageSize ?? requestedSize ?? 20;
-        const collected = [];
-        let firstPage;
-        let total;
-        let nextFrom = startFrom;
+        // First page: serial — we need total before deciding how many more requests to fan out.
+        const firstPageSize = requestedSize === undefined ? maxPageSize : Math.min(maxPageSize, requestedSize);
+        const firstPage = await this.requestJson(endpoint, {
+            ...initialBody,
+            from: startFrom,
+            size: firstPageSize,
+        });
+        if (!this.isPaginatedListResponse(firstPage))
+            return firstPage;
+        const total = firstPage.total;
+        const collected = [...firstPage.list];
+        // Last page reached on first request
+        if (firstPage.list.length < firstPageSize) {
+            return {
+                ...firstPage,
+                total,
+                list: requestedSize === undefined ? collected : collected.slice(0, requestedSize),
+            };
+        }
+        const available = Math.max(total - startFrom, 0);
+        const target = requestedSize === undefined ? available : Math.min(requestedSize, available);
+        if (collected.length >= target) {
+            return {
+                ...firstPage,
+                total,
+                list: requestedSize === undefined ? collected : collected.slice(0, requestedSize),
+            };
+        }
+        const pageRequests = [];
+        let nextFrom = startFrom + firstPage.list.length;
+        const endFrom = startFrom + target;
+        while (nextFrom < endFrom) {
+            const remaining = endFrom - nextFrom;
+            const size = Math.min(maxPageSize, remaining);
+            pageRequests.push({ from: nextFrom, size });
+            nextFrom += size;
+        }
         const MAX_PAGES = 1000;
-        for (let pageCount = 0; pageCount < MAX_PAGES; pageCount++) {
-            const remaining = requestedSize === undefined
-                ? maxPageSize
-                : Math.min(maxPageSize, requestedSize - collected.length);
-            if (requestedSize !== undefined && remaining <= 0) {
-                break;
-            }
+        if (pageRequests.length + 1 > MAX_PAGES) {
+            pageRequests.length = MAX_PAGES - 1;
+        }
+        let unexpectedShape = false;
+        let totalDrift = false;
+        const pages = await runWithConcurrency(pageRequests, PAGINATION_CONCURRENCY, async (req) => {
             const page = await this.requestJson(endpoint, {
                 ...initialBody,
-                from: nextFrom,
-                size: remaining,
+                from: req.from,
+                size: req.size,
             });
             if (!this.isPaginatedListResponse(page)) {
-                if (!firstPage) {
-                    return page;
-                }
-                return {
-                    ...firstPage,
-                    total,
-                    list: requestedSize === undefined ? collected : collected.slice(0, requestedSize),
-                };
-            }
-            if (!firstPage) {
-                firstPage = page;
-                total = page.total;
-            }
-            if (page.list.length === 0) {
-                break;
-            }
-            collected.push(...page.list);
-            nextFrom += page.list.length;
-            const available = total === undefined ? undefined : Math.max(total - startFrom, 0);
-            if (requestedSize !== undefined && collected.length >= requestedSize) {
-                break;
-            }
-            if (available !== undefined && collected.length >= available) {
-                break;
-            }
-            if (page.list.length < remaining) {
-                break;
+                unexpectedShape = true;
+                return [];
             }
+            if (page.total !== total)
+                totalDrift = true;
+            return page.list;
+        });
+        for (const list of pages) {
+            if (list.length === 0)
+                continue;
+            collected.push(...list);
+        }
+        if (unexpectedShape && isVerbose()) {
+            process.stderr.write(`[gangtise] warning: a page response had unexpected shape; results may be incomplete\n`);
         }
-        if (!firstPage) {
-            return { total: 0, list: [] };
+        if (totalDrift && isVerbose()) {
+            process.stderr.write(`[gangtise] warning: 'total' changed across pages (data shifted during fetch)\n`);
         }
         return {
             ...firstPage,
@@ -163,100 +198,153 @@ export class GangtiseClient {
         if (endpoint.path.startsWith('/guide/')) {
             return this.readLocalLookup(endpoint);
         }
-        const headers = {
-            'content-type': 'application/json',
-        };
-        if (useAuth) {
-            headers.Authorization = await this.getAuthorizationHeader();
-        }
-        const response = await request(new URL(endpoint.path, this.config.baseUrl), {
-            method: endpoint.method,
-            headers,
-            body: endpoint.method === 'GET' ? undefined : JSON.stringify(body ?? {}),
-            headersTimeout: this.config.timeoutMs,
-            bodyTimeout: this.config.timeoutMs,
-        });
-        const text = await response.body.text();
-        let parsed;
-        try {
-            parsed = JSON.parse(text);
-        }
-        catch {
-            const message = response.statusCode >= 400
-                ? `API request failed (HTTP ${response.statusCode})`
-                : 'Failed to parse API response';
-            throw new ApiError(message, undefined, response.statusCode, text.slice(0, 500));
-        }
-        if (response.statusCode >= 400) {
-            this.throwHttpError(parsed, response.statusCode);
-        }
-        return this.unwrapEnvelope(parsed, response.statusCode);
-    }
-    async download(endpoint, query) {
-        const authorization = await this.getAuthorizationHeader();
+        const dispatcher = getDispatcher();
         const url = new URL(endpoint.path, this.config.baseUrl);
-        Object.entries(query).forEach(([key, value]) => {
-            url.searchParams.set(key, String(value));
-        });
-        const response = await request(url, {
-            method: endpoint.method,
-            headers: {
-                Authorization: authorization,
-            },
-            headersTimeout: this.config.timeoutMs,
-            bodyTimeout: this.config.timeoutMs,
-        });
-        const contentType = Array.isArray(response.headers['content-type']) ? response.headers['content-type'][0] : response.headers['content-type'];
-        if (contentType?.includes('application/json')) {
+        let authRetried = false;
+        return withRetry(async () => {
+            const headers = {
+                'content-type': 'application/json',
+            };
+            if (useAuth) {
+                headers.Authorization = await this.getAuthorizationHeader();
+            }
+            const startedAt = Date.now();
+            const response = await request(url, {
+                method: endpoint.method,
+                headers,
+                body: endpoint.method === 'GET' ? undefined : JSON.stringify(body ?? {}),
+                headersTimeout: this.config.timeoutMs,
+                bodyTimeout: this.config.timeoutMs,
+                dispatcher,
+            });
             const text = await response.body.text();
+            logTiming(`${endpoint.method} ${endpoint.path}`, Date.now() - startedAt, `${response.statusCode}, ${text.length}B`);
             let parsed;
             try {
                 parsed = JSON.parse(text);
             }
             catch {
-                if (response.statusCode >= 400) {
-                    throw new ApiError('Download failed', undefined, response.statusCode, text);
-                }
-                return { text, contentType };
+                const message = response.statusCode >= 400
+                    ? `API request failed (HTTP ${response.statusCode})`
+                    : 'Failed to parse API response';
+                throw new ApiError(message, undefined, response.statusCode, text.slice(0, 500));
             }
             if (response.statusCode >= 400) {
                 this.throwHttpError(parsed, response.statusCode);
             }
-            const data = this.unwrapEnvelope(parsed, response.statusCode);
-            if (data && typeof data === 'object' && 'url' in data && typeof data.url === 'string') {
-                return { url: String(data.url), contentType };
+            try {
+                return this.unwrapEnvelope(parsed, response.statusCode);
+            }
+            catch (error) {
+                // Auto-recover from auth errors by forcing a token refresh once.
+                if (useAuth
+                    && !authRetried
+                    && error instanceof ApiError
+                    && error.code
+                    && AUTH_RETRY_CODES.has(error.code)
+                    && (this.config.accessKey && this.config.secretKey)) {
+                    authRetried = true;
+                    this.memoCache = null;
+                    await this.getAuthorizationHeader(true);
+                    throw markRetryable(new ApiError(error.message, error.code, error.statusCode, error.details));
+                }
+                throw error;
+            }
+        }, {
+            onRetry: (attempt, error, delay) => {
+                if (!isVerbose())
+                    return;
+                const msg = error instanceof Error ? error.message : String(error);
+                process.stderr.write(`[gangtise] retry ${attempt} after ${delay.toFixed(0)}ms: ${msg.slice(0, 120)}\n`);
+            },
+        });
+    }
+    async download(endpoint, query, options) {
+        const dispatcher = getDispatcher();
+        const url = new URL(endpoint.path, this.config.baseUrl);
+        Object.entries(query).forEach(([key, value]) => {
+            url.searchParams.set(key, String(value));
+        });
+        return withRetry(async () => {
+            const authorization = await this.getAuthorizationHeader();
+            const startedAt = Date.now();
+            const response = await request(url, {
+                method: endpoint.method,
+                headers: { Authorization: authorization },
+                headersTimeout: this.config.timeoutMs,
+                bodyTimeout: this.config.timeoutMs,
+                dispatcher,
+            });
+            const contentType = Array.isArray(response.headers['content-type']) ? response.headers['content-type'][0] : response.headers['content-type'];
+            if (contentType?.includes('application/json')) {
+                const text = await response.body.text();
+                logTiming(`GET ${endpoint.path} (json)`, Date.now() - startedAt, `${response.statusCode}, ${text.length}B`);
+                let parsed;
+                try {
+                    parsed = JSON.parse(text);
+                }
+                catch {
+                    if (response.statusCode >= 400) {
+                        throw new ApiError('Download failed', undefined, response.statusCode, text);
+                    }
+                    return { text, contentType };
+                }
+                if (response.statusCode >= 400) {
+                    this.throwHttpError(parsed, response.statusCode);
+                }
+                const data = this.unwrapEnvelope(parsed, response.statusCode);
+                if (data && typeof data === 'object' && 'url' in data && typeof data.url === 'string') {
+                    return { url: String(data.url), contentType };
+                }
+                return { text: JSON.stringify(data, null, 2), contentType };
+            }
+            if (contentType?.includes('text/plain') || contentType?.includes('text/html')) {
+                const text = await response.body.text();
+                logTiming(`GET ${endpoint.path} (text)`, Date.now() - startedAt, `${response.statusCode}, ${text.length}B`);
+                if (response.statusCode >= 400) {
+                    throw new ApiError('Download failed', undefined, response.statusCode, text);
+                }
+                return { text, contentType };
             }
-            return { text: JSON.stringify(data, null, 2), contentType };
-        }
-        if (contentType?.includes('text/plain') || contentType?.includes('text/html')) {
-            const text = await response.body.text();
             if (response.statusCode >= 400) {
+                const text = await response.body.text();
                 throw new ApiError('Download failed', undefined, response.statusCode, text);
             }
-            return { text, contentType };
-        }
-        if (response.statusCode >= 400) {
-            const text = await response.body.text();
-            throw new ApiError('Download failed', undefined, response.statusCode, text);
-        }
-        const buffer = await response.body.arrayBuffer();
-        const contentDisposition = response.headers['content-disposition'];
-        const filenameMatch = Array.isArray(contentDisposition)
-            ? contentDisposition[0]?.match(/filename\*=UTF-8''([^;]+)|filename="?([^";]+)"?/i)
-            : contentDisposition?.match(/filename\*=UTF-8''([^;]+)|filename="?([^";]+)"?/i);
-        return {
-            data: new Uint8Array(buffer),
-            contentType,
-            filename: filenameMatch ? decodeURIComponent(filenameMatch[1] || filenameMatch[2]) : undefined,
-        };
+            const contentDisposition = response.headers['content-disposition'];
+            const filenameMatch = Array.isArray(contentDisposition)
+                ? contentDisposition[0]?.match(/filename\*=UTF-8''([^;]+)|filename="?([^";]+)"?/i)
+                : contentDisposition?.match(/filename\*=UTF-8''([^;]+)|filename="?([^";]+)"?/i);
+            const filename = filenameMatch ? decodeURIComponent(filenameMatch[1] || filenameMatch[2]) : undefined;
+            // Stream directly to disk when caller already knows the destination
+            if (options?.streamTo) {
+                await fs.mkdir(path.dirname(options.streamTo), { recursive: true });
+                await pipeline(response.body, createWriteStream(options.streamTo));
+                logTiming(`GET ${endpoint.path} (stream)`, Date.now() - startedAt, `${response.statusCode}`);
+                return { contentType, filename, savedPath: options.streamTo };
+            }
+            const buffer = await response.body.arrayBuffer();
+            logTiming(`GET ${endpoint.path} (binary)`, Date.now() - startedAt, `${response.statusCode}, ${buffer.byteLength}B`);
+            return {
+                data: new Uint8Array(buffer),
+                contentType,
+                filename,
+            };
+        }, {
+            onRetry: (attempt, error, delay) => {
+                if (!isVerbose())
+                    return;
+                const msg = error instanceof Error ? error.message : String(error);
+                process.stderr.write(`[gangtise] download retry ${attempt} after ${delay.toFixed(0)}ms: ${msg.slice(0, 120)}\n`);
+            },
+        });
     }
-    async call(endpointKey, body, query) {
-        const endpoint = ENDPOINT_REGISTRY[endpointKey];
+    async call(endpointKey, body, query, options) {
+        const endpoint = ENDPOINTS[endpointKey];
         if (!endpoint) {
             throw new ApiError(`Unknown endpoint key: ${endpointKey}`);
         }
         if (endpoint.kind === 'download') {
-            return this.download(endpoint, query ?? {});
+            return this.download(endpoint, query ?? {}, options);
         }
         if (endpoint.kind === 'json' && endpoint.pagination?.enabled) {
             return this.requestPaginated(endpoint, body);

package/dist/src/core/download.js

@@ -71,6 +71,10 @@ export async function saveDownloadResult(result, fallbackName, output) {
         throw new DownloadError("Unexpected download response");
     }
     const file = result;
+    if (typeof file.savedPath === "string") {
+        process.stdout.write(`${file.savedPath}\n`);
+        return;
+    }
     if (file.data instanceof Uint8Array) {
         const outputPath = output ?? file.filename ?? (fallbackName + extFromContentType(file.contentType));
         await saveOutputIfNeeded(file.data, outputPath);