gangtise-openapi-cli 0.42.0 → 0.43.1

package/dist/src/core/output.js

@@ -1,9 +1,32 @@
 import { createHash, randomBytes } from "node:crypto";
+import { rmSync, statSync } from "node:fs";
 import fs from "node:fs/promises";
 import { ConfigError } from "./errors.js";
 const OUTPUT_FORMATS = ["table", "json", "jsonl", "csv", "markdown"];
 export function stagingPath(target, suffix = "part") {
-    return `${target}.${randomBytes(4).toString("hex")}.${suffix}`;
+    const staging = `${target}.${randomBytes(4).toString("hex")}.${suffix}`;
+    handedOut.add(staging);
+    return staging;
+}
+const handedOut = new Set();
+const claimed = new Set();
+export function trackClaim(name) {
+    claimed.add(name);
+}
+export function removeStagingFiles() {
+    for (const staging of handedOut) {
+        try {
+            rmSync(staging, { force: true });
+        }
+        catch { }
+    }
+    for (const name of claimed) {
+        try {
+            if (statSync(name).size === 0)
+                rmSync(name, { force: true });
+        }
+        catch { }
+    }
 }
 export function digestBuffer(content) {
     const buf = typeof content === "string" ? Buffer.from(content, "utf8") : content;
@@ -29,9 +52,13 @@ export function parseOutputFormat(value) {
     throw new ConfigError(`Unsupported format: ${format}`);
 }
 function sanitizeCell(value) {
+    if (!/[\u0000-\u001f\u007f-\u009f]/.test(value))
+        return value;
     return value.replace(/[\r\n]+/g, " ").replace(/[\u0000-\u001f\u007f\u0080-\u009f]/g, "");
 }
 function displayWidth(value) {
+    if (/^[\x20-\x7e]*$/.test(value))
+        return value.length;
     let width = 0;
     for (const ch of value) {
         const cp = ch.codePointAt(0);
@@ -97,11 +124,13 @@ function renderTable(rows) {
     const columns = Array.from(new Set(rows.flatMap((row) => Object.keys(row))));
     const headerCells = columns.map((column) => clampCell(sanitizeCell(column)));
     const matrix = rows.map((row) => columns.map((column) => clampCell(sanitizeCell(formatScalar(row[column])))));
-    const widths = columns.map((_, c) => matrix.reduce((max, cells) => Math.max(max, displayWidth(cells[c])), displayWidth(headerCells[c])));
-    const renderLine = (values) => values.map((value, index) => value + " ".repeat(Math.max(0, widths[index] - displayWidth(value)))).join("  ");
-    const header = renderLine(headerCells);
-    const divider = renderLine(widths.map((width) => "-".repeat(width)));
-    const body = matrix.map((cells) => renderLine(cells));
+    const cellWidths = matrix.map((cells) => cells.map(displayWidth));
+    const headerWidths = headerCells.map(displayWidth);
+    const widths = columns.map((_, c) => cellWidths.reduce((max, cells) => Math.max(max, cells[c]), headerWidths[c]));
+    const renderLine = (values, valueWidths) => values.map((value, index) => value + " ".repeat(Math.max(0, widths[index] - valueWidths[index]))).join("  ");
+    const header = renderLine(headerCells, headerWidths);
+    const divider = renderLine(widths.map((width) => "-".repeat(width)), widths);
+    const body = matrix.map((cells, r) => renderLine(cells, cellWidths[r]));
     return [header, divider, ...body].join("\n");
 }
 function renderMarkdown(rows) {
@@ -242,7 +271,7 @@ export function writeLine(stream, line) {
         stream.once("error", onError);
     });
 }
-export async function saveOutputIfNeeded(content, outputPath) {
+export async function saveOutputIfNeeded(content, outputPath, withDigest = true) {
     if (!outputPath) {
         return;
     }
@@ -256,7 +285,7 @@ export async function saveOutputIfNeeded(content, outputPath) {
         else {
             await fs.writeFile(partPath, content);
         }
-        const digest = digestBuffer(content);
+        const digest = withDigest ? digestBuffer(content) : undefined;
         await fs.rename(partPath, outputPath);
         return digest;
     }

package/dist/src/core/perSecurity.js

@@ -17,7 +17,10 @@ export function estimateTradingDays(startDate, endDate) {
     }
     return days;
 }
-export async function callPerSecurity(client, endpointKey, securities, makeBody, cap, label) {
+export async function callPerSecurity(client, endpointKey, securities, makeBody, cap, label, groupSize = 1) {
+    const groups = [];
+    for (let i = 0; i < securities.length; i += groupSize)
+        groups.push(securities.slice(i, i + groupSize));
     const sink = client.claimRowSink?.();
     let fieldList;
     let headerSecurity;
@@ -29,14 +32,14 @@ export async function callPerSecurity(client, endpointKey, securities, makeBody,
     const mergePart = async (part, i) => {
         const rec = part;
         if (!(rec && typeof rec === "object" && Array.isArray(rec.list))) {
-            throw markStructural(new ApiError(`${label}: ${securities[i]} returned no list payload — the response layout may have changed`, undefined, undefined, rec));
+            throw markStructural(new ApiError(`${label}: ${groups[i].join(", ")} returned no list payload — the response layout may have changed`, undefined, undefined, rec));
         }
         if (rec.partial === true)
             partial = true;
         if (rec.list.length === 0) {
             if ((typeof rec.total === "number" && rec.total > 0) || rec.partial === true) {
                 const claim = typeof rec.total === "number" && rec.total > 0 ? `reported total=${rec.total}` : "carried a partial marker";
-                throw markStructural(new ApiError(`${label}: ${securities[i]} ${claim} but delivered no rows — the response is inconsistent`, undefined, undefined, rec));
+                throw markStructural(new ApiError(`${label}: ${groups[i].join(", ")} ${claim} but delivered no rows — the response is inconsistent`, undefined, undefined, rec));
             }
             if (!emptyFields && Array.isArray(rec.fieldList))
                 emptyFields = rec.fieldList;
@@ -44,26 +47,27 @@ export async function callPerSecurity(client, endpointKey, securities, makeBody,
         }
         const fields = Array.isArray(rec.fieldList) && rec.fieldList.length > 0 ? rec.fieldList : undefined;
         if (rec.list.some(Array.isArray) && !columnarSchemaValid(fields, rec.list)) {
-            throw markStructural(new ApiError(`${label}: ${securities[i]} returned columnar rows without a usable fieldList (missing, duplicated or mis-sized) — they cannot be read by position`, undefined, undefined, rec));
+            throw markStructural(new ApiError(`${label}: ${groups[i].join(", ")} returned columnar rows without a usable fieldList (missing, duplicated or mis-sized) — they cannot be read by position`, undefined, undefined, rec));
         }
         if (fields && !fieldList) {
             fieldList = fields;
-            headerSecurity = securities[i];
+            headerSecurity = groups[i].join(", ");
             sink?.setFieldList(fieldList);
         }
         if ((fields ?? fieldList) && !sameColumns(fieldList, fields)) {
-            throw markStructural(new ApiError(`${label}: ${securities[i]} answered with columns ${JSON.stringify(fields)} while ${headerSecurity} answered ${JSON.stringify(fieldList)} — the parts cannot be merged`, undefined, undefined, rec));
+            throw markStructural(new ApiError(`${label}: ${groups[i].join(", ")} answered with columns ${JSON.stringify(fields)} while ${headerSecurity} answered ${JSON.stringify(fieldList)} — the parts cannot be merged`, undefined, undefined, rec));
         }
         if (rec.list.length >= cap)
-            truncated.push(securities[i]);
-        count += rec.list.length;
+            truncated.push(...groups[i]);
+        const rows = inInputOrder(rec.list, fields ?? fieldList, groups[i], label);
+        count += rows.length;
         if (sink)
-            await sink.push(rec.list);
+            await sink.push(rows);
         else
-            for (const row of rec.list)
+            for (const row of rows)
                 merged.push(row);
     };
-    await runInOrder(securities, PAGE_CONCURRENCY, (code) => client.call(endpointKey, makeBody(code)), mergePart);
+    await runInOrder(groups, PAGE_CONCURRENCY, (codes) => client.call(endpointKey, makeBody(codes)), mergePart);
     const out = { total: count, list: merged };
     if (sink)
         attachRowSink(out, sink);
@@ -80,6 +84,20 @@ export async function callPerSecurity(client, endpointKey, securities, makeBody,
     }
     return out;
 }
+function inInputOrder(rows, fields, group, label) {
+    if (group.length < 2)
+        return rows;
+    const column = fields?.indexOf("securityCode") ?? -1;
+    if (rows.some(Array.isArray) && column < 0) {
+        throw markStructural(new ApiError(`${label}: ${group.join(", ")} answered without a securityCode column — rows of several securities cannot be told apart`));
+    }
+    const rank = new Map(group.map((code, i) => [code.toUpperCase(), i]));
+    const codeOf = (row) => Array.isArray(row) ? row[column] : row?.securityCode;
+    return rows
+        .map((row, i) => ({ row, i, at: rank.get(String(codeOf(row)).toUpperCase()) ?? group.length }))
+        .sort((a, b) => a.at - b.at || a.i - b.i)
+        .map(({ row }) => row);
+}
 function sameColumns(a, b) {
     if (!a || !b)
         return false;

package/dist/src/core/quoteSharding.js

@@ -3,7 +3,8 @@ import { columnarSchemaValid } from "./normalize.js";
 import { attachRowSink } from "./rowSink.js";
 import { isVerbose, PAGE_CONCURRENCY, runInOrder } from "./transport.js";
 const DAY_MS = 86_400_000;
-const ALL_MARKET_LIMIT = 10_000;
+export const QUOTE_MAX_LIMIT = 10_000;
+const ALL_MARKET_LIMIT = QUOTE_MAX_LIMIT;
 function parseDate(value) {
     if (!/^\d{4}-\d{2}-\d{2}$/.test(value))
         return null;
@@ -42,18 +43,21 @@ function isWeekendUtc(d) {
 }
 function buildShards(start, end, shardDays) {
     const shards = [];
-    let cursor = start.getTime();
-    const endTime = end.getTime();
-    while (cursor <= endTime) {
-        const shardEnd = Math.min(cursor + (shardDays - 1) * DAY_MS, endTime);
-        if (!(shardDays === 1 && isWeekendUtc(new Date(cursor)))) {
-            shards.push({
-                startDate: formatDate(new Date(cursor)),
-                endDate: formatDate(new Date(shardEnd)),
-            });
-        }
-        cursor = shardEnd + DAY_MS;
-    }
+    let group = [];
+    const flush = () => {
+        if (group.length === 0)
+            return;
+        shards.push({ startDate: formatDate(new Date(group[0])), endDate: formatDate(new Date(group[group.length - 1])) });
+        group = [];
+    };
+    for (let day = start.getTime(); day <= end.getTime(); day += DAY_MS) {
+        if (isWeekendUtc(new Date(day)))
+            continue;
+        group.push(day);
+        if (group.length === shardDays)
+            flush();
+    }
+    flush();
     return shards;
 }
 export async function callKlineWithSharding(client, endpointKey, body, config) {
@@ -65,7 +69,7 @@ export async function callKlineWithSharding(client, endpointKey, body, config) {
     const perShardLimit = allMarketBody.limit ?? ALL_MARKET_LIMIT;
     const truncationHint = perShardLimit < ALL_MARKET_LIMIT
         ? "raise or omit --limit to fetch the full market"
-        : `a single ${config.shardDays}-day window exceeds the ${ALL_MARKET_LIMIT}-row API cap`;
+        : `a single ${config.shardDays}-weekday window exceeds the ${ALL_MARKET_LIMIT}-row API cap`;
     const isTruncated = (rec) => Array.isArray(rec.list) && rec.list.length >= perShardLimit;
     const callSingle = async () => {
         const single = await client.call(endpointKey, allMarketBody);
@@ -92,7 +96,7 @@ export async function callKlineWithSharding(client, endpointKey, body, config) {
     if (shards.length === 0)
         return { list: [] };
     if (isVerbose()) {
-        process.stderr.write(`[gangtise] sharding ${endpointKey} into ${shards.length} requests (${config.shardDays} day(s) each)\n`);
+        process.stderr.write(`[gangtise] sharding ${endpointKey} into ${shards.length} requests (${config.shardDays} weekday(s) each)\n`);
     }
     const failedShards = [];
     let firstError = null;

package/dist/src/core/rowSink.js

@@ -22,6 +22,7 @@ export class ExportSink {
     columns = [];
     columnSet = new Set();
     objectRows = 0;
+    earliestDates = new Map();
     partPath;
     rowsPath;
     constructor(outputPath, format = "jsonl", cache) {
@@ -45,7 +46,27 @@ export class ExportSink {
     setFieldList(fields) {
         this.fields = fields;
     }
+    watchEarliest(field) {
+        if (!this.earliestDates.has(field))
+            this.earliestDates.set(field, undefined);
+    }
+    earliest(field) {
+        return this.earliestDates.get(field);
+    }
     async push(rows) {
+        for (const [field, current] of this.earliestDates) {
+            const index = this.fields ? this.fields.indexOf(field) : -1;
+            let earliest = current;
+            for (const row of rows) {
+                const value = Array.isArray(row) ? (index >= 0 ? row[index] : undefined) : row && typeof row === "object" ? row[field] : undefined;
+                if (typeof value !== "string")
+                    continue;
+                const day = value.slice(0, 10);
+                if (earliest === undefined || day < earliest)
+                    earliest = day;
+            }
+            this.earliestDates.set(field, earliest);
+        }
         if (this.stream) {
             await this.write(rows);
             return;

package/dist/src/core/titleCache.js

@@ -1,6 +1,7 @@
 import fs from "node:fs/promises";
 import os from "node:os";
 import path from "node:path";
+import { stagingPath } from "./output.js";
 export const DEFAULT_TITLE_CACHE_PATH = path.join(os.homedir(), ".config", "gangtise", "title-cache.json");
 export const TITLE_LOOKUP_SIZE = 200;
 const TITLE_CACHE_TTL_MS = 24 * 60 * 60 * 1000;
@@ -43,6 +44,29 @@ async function loadInto(filePath) {
 export async function readTitleCache(filePath = DEFAULT_TITLE_CACHE_PATH) {
     return loadInto(filePath);
 }
+async function mergeFromDisk(filePath, data) {
+    let disk;
+    try {
+        disk = JSON.parse(await fs.readFile(filePath, "utf8"));
+    }
+    catch {
+        return;
+    }
+    if (!disk || typeof disk !== "object" || Array.isArray(disk))
+        return;
+    for (const [key, entry] of Object.entries(disk)) {
+        if (!entry?.titles || typeof entry.titles !== "object" || typeof entry.ts !== "number")
+            continue;
+        const mine = data[key];
+        if (!mine) {
+            data[key] = { titles: entry.titles, ts: entry.ts };
+            continue;
+        }
+        const titles = { ...entry.titles, ...mine.titles };
+        data[key] = { titles: capTitles(titles, Object.keys(mine.titles), MAX_TITLES_PER_ENDPOINT), ts: Math.max(entry.ts, mine.ts) };
+    }
+    pruneCache(data, "", []);
+}
 async function flush(filePath) {
     for (;;) {
         if (!dirty || !memoryCache) {
@@ -50,9 +74,11 @@ async function flush(filePath) {
             return;
         }
         dirty = false;
-        const snapshot = JSON.stringify(memoryCache);
+        const data = memoryCache;
+        await mergeFromDisk(filePath, data);
+        const snapshot = JSON.stringify(data);
         await fs.mkdir(path.dirname(filePath), { recursive: true });
-        const tmp = `${filePath}.tmp-${process.pid}-${Date.now()}`;
+        const tmp = stagingPath(filePath, "tmp");
         await fs.writeFile(tmp, snapshot, { encoding: "utf8", mode: 0o600 });
         try {
             await fs.rename(tmp, filePath);

package/dist/src/core/transport.js

@@ -1,5 +1,5 @@
+import { createRequire } from "node:module";
 import { gunzipSync } from "node:zlib";
-import { Agent } from "undici";
 import { ApiError } from "./errors.js";
 export function decodeResponseBody(buf, contentEncoding) {
     const enc = (Array.isArray(contentEncoding) ? contentEncoding[0] : contentEncoding)?.toLowerCase().trim();
@@ -34,8 +34,10 @@ function retryAfterFromError(error) {
     return undefined;
 }
 let cachedDispatcher = null;
+const requireLazily = createRequire(import.meta.url);
 export function getDispatcher() {
     if (!cachedDispatcher) {
+        const { Agent } = requireLazily("undici");
         cachedDispatcher = new Agent({
             keepAliveTimeout: 60_000,
             keepAliveMaxTimeout: 600_000,

package/dist/src/core/updateCheck.js

@@ -0,0 +1,51 @@
+import { mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { isVersionNewer } from "./args.js";
+import { stagingPath } from "./output.js";
+export const UPDATE_CHECK_TTL_MS = 24 * 60 * 60 * 1000;
+export const DEFAULT_UPDATE_CHECK_PATH = path.join(os.homedir(), ".config", "gangtise", "update-check.json");
+function saveLatest(cachePath, latest, now) {
+    const staging = stagingPath(cachePath, "tmp");
+    try {
+        mkdirSync(path.dirname(cachePath), { recursive: true });
+        writeFileSync(staging, JSON.stringify({ checkedAt: now, latest }));
+        renameSync(staging, cachePath);
+    }
+    catch {
+        try {
+            rmSync(staging, { force: true });
+        }
+        catch { }
+    }
+}
+export async function latestVersion(cachePath = DEFAULT_UPDATE_CHECK_PATH, timeoutMs = 2000, now = Date.now()) {
+    const cacheable = path.isAbsolute(cachePath);
+    if (cacheable) {
+        try {
+            const cached = JSON.parse(readFileSync(cachePath, "utf8"));
+            const age = typeof cached.checkedAt === "number" ? now - cached.checkedAt : -1;
+            if (typeof cached.latest === "string" && age >= 0 && age < UPDATE_CHECK_TTL_MS)
+                return cached.latest;
+        }
+        catch { }
+    }
+    try {
+        const response = await fetch("https://registry.npmjs.org/gangtise-openapi-cli/latest", { signal: AbortSignal.timeout(timeoutMs) });
+        const latest = (await response.json()).version;
+        if (typeof latest !== "string")
+            return undefined;
+        if (cacheable)
+            saveLatest(cachePath, latest, now);
+        return latest;
+    }
+    catch {
+        return undefined;
+    }
+}
+export async function checkForUpdate(current) {
+    const latest = await latestVersion();
+    if (latest && isVersionNewer(latest, current)) {
+        process.stderr.write(`Update available: ${current} → ${latest}\nRun: npm update -g gangtise-openapi-cli\n`);
+    }
+}

package/dist/src/version.js

@@ -1 +1,19 @@
-export const CLI_VERSION = "0.42.0";
+import fs from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+function packageVersion() {
+    let dir = path.dirname(fileURLToPath(import.meta.url));
+    for (;;) {
+        const file = path.join(dir, "package.json");
+        if (fs.existsSync(file)) {
+            const pkg = JSON.parse(fs.readFileSync(file, "utf8"));
+            if (pkg.name === "gangtise-openapi-cli" && pkg.version)
+                return pkg.version;
+        }
+        const parent = path.dirname(dir);
+        if (parent === dir)
+            throw new Error("gangtise-openapi-cli: its package.json was not found above " + fileURLToPath(import.meta.url));
+        dir = parent;
+    }
+}
+export const CLI_VERSION = packageVersion();