gangtise-openapi-cli 0.11.0 → 0.12.0

package/dist/src/core/quoteSharding.js

@@ -0,0 +1,81 @@
+import { runWithConcurrency } from "./transport.js";
+const DAY_MS = 86_400_000;
+function parseDate(value) {
+    // Accept yyyy-MM-dd; reject anything else so we can fall back to a single request.
+    if (!/^\d{4}-\d{2}-\d{2}$/.test(value))
+        return null;
+    const d = new Date(`${value}T00:00:00Z`);
+    if (Number.isNaN(d.getTime()))
+        return null;
+    return d;
+}
+function formatDate(d) {
+    return d.toISOString().slice(0, 10);
+}
+function isAllMarket(body) {
+    const list = body.securityList;
+    if (!Array.isArray(list) || list.length !== 1)
+        return false;
+    return list[0] === "all";
+}
+function buildShards(start, end, shardDays) {
+    const shards = [];
+    let cursor = start.getTime();
+    const endTime = end.getTime();
+    while (cursor <= endTime) {
+        const shardEnd = Math.min(cursor + (shardDays - 1) * DAY_MS, endTime);
+        shards.push({
+            startDate: formatDate(new Date(cursor)),
+            endDate: formatDate(new Date(shardEnd)),
+        });
+        cursor = shardEnd + DAY_MS;
+    }
+    return shards;
+}
+/**
+ * For full-market (`--security all`) K-line queries that span more than `shardDays`,
+ * split the date range and run shards in parallel. Each shard is sized so the
+ * combined row count stays under the 10K-row API limit. For small ranges or
+ * single-security queries this is a no-op.
+ */
+export async function callKlineWithSharding(client, endpointKey, body, config) {
+    if (!isAllMarket(body) || !body.startDate || !body.endDate) {
+        return client.call(endpointKey, body);
+    }
+    const start = parseDate(body.startDate);
+    const end = parseDate(body.endDate);
+    if (!start || !end || end < start) {
+        return client.call(endpointKey, body);
+    }
+    const totalDays = Math.floor((end.getTime() - start.getTime()) / DAY_MS) + 1;
+    if (totalDays <= config.shardDays) {
+        return client.call(endpointKey, body);
+    }
+    const shards = buildShards(start, end, config.shardDays);
+    if (process.env.GANGTISE_VERBOSE === "1" || process.env.GANGTISE_VERBOSE === "true") {
+        process.stderr.write(`[gangtise] sharding ${endpointKey} into ${shards.length} requests (${config.shardDays} day(s) each)\n`);
+    }
+    const results = await runWithConcurrency(shards, config.concurrency ?? 5, async (shard) => {
+        return client.call(endpointKey, { ...body, startDate: shard.startDate, endDate: shard.endDate });
+    });
+    let fieldList;
+    let header = null;
+    const merged = [];
+    for (const r of results) {
+        if (!(r && typeof r === "object"))
+            continue;
+        const rec = r;
+        if (!header)
+            header = rec;
+        if (!fieldList && Array.isArray(rec.fieldList))
+            fieldList = rec.fieldList;
+        if (Array.isArray(rec.list))
+            merged.push(...rec.list);
+    }
+    if (!header)
+        return { list: [] };
+    const out = { ...header, list: merged };
+    if (fieldList)
+        out.fieldList = fieldList;
+    return out;
+}

package/dist/src/core/titleCache.js

@@ -4,24 +4,65 @@ import path from "node:path";
 export const DEFAULT_TITLE_CACHE_PATH = path.join(os.homedir(), ".config", "gangtise", "title-cache.json");
 export const TITLE_LOOKUP_SIZE = 200;
 const TITLE_CACHE_TTL_MS = 24 * 60 * 60 * 1000;
-export async function readTitleCache(filePath = DEFAULT_TITLE_CACHE_PATH) {
+// Per-process in-memory snapshot of the cache. We read the file at most once,
+// merge subsequent writes in memory, and flush atomically. This avoids the
+// "read whole file → modify → write whole file" pattern firing on every list
+// command (which got expensive once dozens of endpoints accumulated).
+let memoryCache = null;
+let memoryCachePath = null;
+let pendingWrite = null;
+let dirty = false;
+async function loadInto(filePath) {
+    if (memoryCache && memoryCachePath === filePath)
+        return memoryCache;
     try {
         const content = await fs.readFile(filePath, "utf8");
         const parsed = JSON.parse(content);
-        if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
-            return parsed;
-        }
+        memoryCache = parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : {};
     }
     catch {
-        return {};
+        memoryCache = {};
     }
-    return {};
+    memoryCachePath = filePath;
+    return memoryCache;
 }
-export async function writeTitleCache(endpoint, titles, filePath = DEFAULT_TITLE_CACHE_PATH) {
-    const data = await readTitleCache(filePath);
-    data[endpoint] = { titles, ts: Date.now() };
+export async function readTitleCache(filePath = DEFAULT_TITLE_CACHE_PATH) {
+    return loadInto(filePath);
+}
+async function flush(filePath) {
+    if (!dirty || !memoryCache)
+        return;
+    dirty = false;
+    const snapshot = JSON.stringify(memoryCache);
     await fs.mkdir(path.dirname(filePath), { recursive: true });
-    await fs.writeFile(filePath, JSON.stringify(data), { encoding: "utf8", mode: 0o600 });
+    // Atomic-ish: write to temp file then rename (rename is atomic within a fs).
+    const tmp = `${filePath}.tmp-${process.pid}-${Date.now()}`;
+    await fs.writeFile(tmp, snapshot, { encoding: "utf8", mode: 0o600 });
+    try {
+        await fs.rename(tmp, filePath);
+    }
+    catch (error) {
+        await fs.unlink(tmp).catch(() => { });
+        throw error;
+    }
+}
+export async function writeTitleCache(endpoint, titles, filePath = DEFAULT_TITLE_CACHE_PATH) {
+    const data = await loadInto(filePath);
+    const existing = data[endpoint]?.titles ?? {};
+    data[endpoint] = { titles: { ...existing, ...titles }, ts: Date.now() };
+    dirty = true;
+    // Coalesce concurrent writes: the in-flight flush picks up everything dirty.
+    if (pendingWrite)
+        return pendingWrite;
+    pendingWrite = (async () => {
+        try {
+            await flush(filePath);
+        }
+        finally {
+            pendingWrite = null;
+        }
+    })();
+    return pendingWrite;
 }
 export function lookupTitleCache(data, endpoint, id) {
     const entry = data[endpoint];
@@ -44,3 +85,10 @@ export function extractTitles(items, cache) {
     }
     return titles;
 }
+/** Test-only hook to reset the in-memory snapshot between cases. */
+export function __resetTitleCacheForTests() {
+    memoryCache = null;
+    memoryCachePath = null;
+    pendingWrite = null;
+    dirty = false;
+}

package/dist/src/core/transport.js

@@ -0,0 +1,91 @@
+import { Agent } from "undici";
+import { ApiError } from "./errors.js";
+let cachedDispatcher = null;
+export function getDispatcher() {
+    if (!cachedDispatcher) {
+        cachedDispatcher = new Agent({
+            keepAliveTimeout: 60_000,
+            keepAliveMaxTimeout: 600_000,
+            connections: 16,
+            pipelining: 1,
+        });
+    }
+    return cachedDispatcher;
+}
+export async function runWithConcurrency(items, concurrency, fn) {
+    if (items.length === 0)
+        return [];
+    const limit = Math.max(1, Math.min(concurrency, items.length));
+    const results = new Array(items.length);
+    let next = 0;
+    async function worker() {
+        while (true) {
+            const index = next++;
+            if (index >= items.length)
+                return;
+            results[index] = await fn(items[index], index);
+        }
+    }
+    await Promise.all(Array.from({ length: limit }, () => worker()));
+    return results;
+}
+const RETRYABLE_HTTP_STATUS = new Set([429, 500, 502, 503, 504]);
+const RETRYABLE_NETWORK_CODES = new Set(["ECONNRESET", "ETIMEDOUT", "ENOTFOUND", "EAI_AGAIN", "UND_ERR_SOCKET", "UND_ERR_HEADERS_TIMEOUT", "UND_ERR_BODY_TIMEOUT"]);
+const RETRYABLE_API_CODES = new Set(["999999"]);
+function isRetryableError(error) {
+    if (error && typeof error === "object" && error.__retryable === true) {
+        return true;
+    }
+    if (error instanceof ApiError) {
+        if (error.statusCode != null && RETRYABLE_HTTP_STATUS.has(error.statusCode))
+            return true;
+        if (error.code && RETRYABLE_API_CODES.has(error.code))
+            return true;
+        return false;
+    }
+    if (error && typeof error === "object" && "code" in error) {
+        const code = String(error.code);
+        if (RETRYABLE_NETWORK_CODES.has(code))
+            return true;
+    }
+    if (error instanceof Error && /timeout|ETIMEDOUT|ECONNRESET|socket hang up/i.test(error.message)) {
+        return true;
+    }
+    return false;
+}
+export function markRetryable(error) {
+    return Object.assign(error, { __retryable: true });
+}
+export async function withRetry(fn, options = {}) {
+    const retries = options.retries ?? 2;
+    const baseDelay = options.baseDelayMs ?? 400;
+    const maxDelay = options.maxDelayMs ?? 4_000;
+    let attempt = 0;
+    while (true) {
+        try {
+            return await fn();
+        }
+        catch (error) {
+            if (attempt >= retries || !isRetryableError(error))
+                throw error;
+            const jitter = Math.random() * baseDelay;
+            const delay = Math.min(maxDelay, baseDelay * 2 ** attempt + jitter);
+            options.onRetry?.(attempt + 1, error, delay);
+            await new Promise(resolve => setTimeout(resolve, delay));
+            attempt++;
+        }
+    }
+}
+let verboseEnabled = process.env.GANGTISE_VERBOSE === "1" || process.env.GANGTISE_VERBOSE === "true";
+export function setVerbose(value) {
+    verboseEnabled = value;
+}
+export function isVerbose() {
+    return verboseEnabled;
+}
+export function logTiming(label, durationMs, extra) {
+    if (!verboseEnabled)
+        return;
+    const ms = durationMs.toFixed(0).padStart(5, " ");
+    process.stderr.write(`[gangtise] ${ms}ms ${label}${extra ? ` (${extra})` : ""}\n`);
+}

package/dist/src/version.js

@@ -1,2 +1,2 @@
 // Auto-generated — DO NOT EDIT
-export const CLI_VERSION = "0.11.0";
+export const CLI_VERSION = "0.12.0";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gangtise-openapi-cli",
-  "version": "0.11.0",
+  "version": "0.12.0",
   "description": "CLI for Gangtise OpenAPI",
   "license": "MIT",
   "repository": {