gangtise-openapi-cli 0.37.0 → 0.38.0

package/dist/src/core/printer.js

@@ -1,17 +1,105 @@
+import fs from "node:fs/promises";
+import path from "node:path";
 import { normalizeRows } from "./normalize.js";
-import { pickList, renderOutput, saveOutputIfNeeded, streamOutputToFile } from "./output.js";
+import { countOutputRows, pickList, renderOutput, saveOutputIfNeeded, streamOutputToFile } from "./output.js";
+import { getRowSink } from "./rowSink.js";
 import { extractTitles, writeTitleCache } from "./titleCache.js";
+import { CLI_VERSION } from "../version.js";
 const LARGE_RESULT_ROWS = 50_000;
 function warnIfLargeInMemory(items, format) {
     if (items && items.length >= LARGE_RESULT_ROWS) {
         process.stderr.write(`[gangtise] note: ${items.length} rows in '${format}' is built entirely in memory; stream large exports to a file with --format jsonl --output <path> (or csv).\n`);
     }
 }
+function localIso(d) {
+    const pad = (n) => String(n).padStart(2, "0");
+    const offset = -d.getTimezoneOffset();
+    const sign = offset >= 0 ? "+" : "-";
+    const abs = Math.abs(offset);
+    return `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())}T${pad(d.getHours())}:${pad(d.getMinutes())}:${pad(d.getSeconds())}${sign}${pad(Math.floor(abs / 60))}:${pad(abs % 60)}`;
+}
+const SECRET_KEY = /^(access|secret|api|private)?key$|secret|token|password|credential/i;
+function redactSecrets(value) {
+    if (Array.isArray(value))
+        return value.map(redactSecrets);
+    if (value && typeof value === "object") {
+        const out = {};
+        for (const [key, inner] of Object.entries(value))
+            out[key] = SECRET_KEY.test(key) ? "[redacted]" : redactSecrets(inner);
+        return out;
+    }
+    if (typeof value === "string") {
+        const eq = value.startsWith("--") ? value.indexOf("=") : -1;
+        const head = eq > 0 ? value.slice(0, eq + 1) : "";
+        const raw = eq > 0 ? value.slice(eq + 1) : value;
+        const trimmed = raw.trimStart();
+        if (trimmed.startsWith("{") || trimmed.startsWith("[")) {
+            try {
+                const parsed = JSON.parse(trimmed);
+                if (parsed && typeof parsed === "object")
+                    return head + JSON.stringify(redactSecrets(parsed));
+            }
+            catch {
+            }
+        }
+    }
+    return value;
+}
+async function stageExportMeta(output, format, rows, columns, normalized, complete) {
+    const result = normalized && typeof normalized === "object" && !Array.isArray(normalized) ? { ...normalized } : {};
+    delete result.list;
+    const doc = {
+        file: path.basename(output),
+        format,
+        rows,
+        complete,
+        exitCode: complete ? 0 : 3,
+        command: redactSecrets(process.argv.slice(2)),
+        cliVersion: CLI_VERSION,
+        fetchedAt: localIso(new Date()),
+        timezone: Intl.DateTimeFormat().resolvedOptions().timeZone,
+        columns,
+        result: redactSecrets(result),
+    };
+    const metaPath = `${output}.meta.json`;
+    const partPath = `${metaPath}.part`;
+    await fs.mkdir(path.dirname(output), { recursive: true });
+    try {
+        await fs.writeFile(partPath, `${JSON.stringify(doc, null, 2)}\n`, "utf8");
+    }
+    catch (error) {
+        await fs.unlink(partPath).catch(() => { });
+        throw error;
+    }
+    return {
+        async commit() {
+            try {
+                await fs.rename(partPath, metaPath);
+            }
+            catch (error) {
+                await fs.unlink(partPath).catch(() => { });
+                await fs.unlink(metaPath).catch(() => { });
+                throw error;
+            }
+        },
+        async discard() {
+            await fs.unlink(partPath).catch(() => { });
+        },
+    };
+}
 export async function printData(data, format, output, cache) {
+    const sink = getRowSink(data);
+    if (sink && !sink.opened)
+        data.list = sink.takeBuffer();
+    const streamed = sink?.opened ? sink : undefined;
+    const columns = data && typeof data === "object" && Array.isArray(data.fieldList)
+        ? data.fieldList
+        : undefined;
     const normalized = normalizeRows(data);
     const items = pickList(normalized);
-    if (cache && items) {
-        const titles = extractTitles(items, cache);
+    const showing = streamed ? streamed.rows : (items?.length ?? 0);
+    if (cache) {
+        const titles = { ...(streamed?.titles ?? {}), ...(items ? extractTitles(items, cache) : {}) };
         if (Object.keys(titles).length > 0) {
             writeTitleCache(cache.endpointKey, titles).catch(() => { });
         }
@@ -19,21 +107,32 @@ export async function printData(data, format, output, cache) {
     if (normalized && typeof normalized === "object" && !Array.isArray(normalized)) {
         const meta = normalized;
         if (typeof meta.total === "number" && format !== "json") {
-            const listLen = Array.isArray(meta.list) ? meta.list.length : 0;
-            process.stderr.write(`Total: ${meta.total}, showing: ${listLen}\n`);
+            process.stderr.write(`Total: ${meta.total}, showing: ${showing}\n`);
         }
         if (meta.partial === true) {
             process.exitCode = 3;
         }
     }
+    const complete = process.exitCode !== 3;
     if (output) {
-        if (await streamOutputToFile(normalized, format, output)) {
-            process.stdout.write(`${output}\n`);
-            return;
+        const staged = format === "csv" || format === "jsonl"
+            ? await stageExportMeta(output, format, streamed ? streamed.dataRows : countOutputRows(normalized, format), columns, normalized, complete)
+            : null;
+        try {
+            if (streamed) {
+                await streamed.finish();
+            }
+            else if (!(await streamOutputToFile(normalized, format, output))) {
+                warnIfLargeInMemory(items, format);
+                const content = renderOutput(normalized, format);
+                await saveOutputIfNeeded(format === "csv" ? `\ufeff${content}` : content, output);
+            }
+        }
+        catch (error) {
+            await staged?.discard();
+            throw error;
         }
-        warnIfLargeInMemory(items, format);
-        const content = renderOutput(normalized, format);
-        await saveOutputIfNeeded(format === "csv" ? `\ufeff${content}` : content, output);
+        await staged?.commit();
         process.stdout.write(`${output}\n`);
         return;
     }

package/dist/src/core/quoteSharding.js

@@ -1,4 +1,7 @@
-import { isVerbose, PAGE_CONCURRENCY, runWithConcurrency } from "./transport.js";
+import { isStructuralError } from "./errors.js";
+import { columnarSchemaValid } from "./normalize.js";
+import { attachRowSink } from "./rowSink.js";
+import { isVerbose, PAGE_CONCURRENCY, runInOrder } from "./transport.js";
 const DAY_MS = 86_400_000;
 const ALL_MARKET_LIMIT = 10_000;
 function parseDate(value) {
@@ -12,6 +15,19 @@ function parseDate(value) {
 function formatDate(d) {
     return d.toISOString().slice(0, 10);
 }
+function columnRemap(header, shard) {
+    if (header.length === shard.length && header.every((field, i) => field === shard[i]))
+        return null;
+    const index = new Map(shard.map((field, i) => [String(field), i]));
+    const map = [];
+    for (const field of header) {
+        const i = index.get(String(field));
+        if (i === undefined)
+            return undefined;
+        map.push(i);
+    }
+    return map;
+}
 export function isFullMarket(body, fullMarketValue) {
     const list = body.securityList;
     if (!Array.isArray(list) || list.length !== 1)
@@ -79,7 +95,7 @@ export async function callKlineWithSharding(client, endpointKey, body, config) {
     const failedShards = [];
     let firstError = null;
     let aborted = false;
-    const results = await runWithConcurrency(shards, config.concurrency ?? PAGE_CONCURRENCY, async (shard) => {
+    const fetchShard = async (shard) => {
         if (aborted) {
             failedShards.push(shard);
             return null;
@@ -95,43 +111,106 @@ export async function callKlineWithSharding(client, endpointKey, body, config) {
         catch (error) {
             if (!firstError)
                 firstError = error;
-            aborted = true;
+            if (!isStructuralError(error))
+                aborted = true;
             failedShards.push(shard);
             return null;
         }
-    });
+    };
+    const sink = client.claimRowSink?.();
     let fieldList;
     let header = null;
+    let fallback = null;
     const merged = [];
+    let count = 0;
+    const objectKeys = new Set();
     const truncatedShards = [];
-    for (let i = 0; i < results.length; i++) {
-        const r = results[i];
+    const partialShards = [];
+    const keep = async (rows) => {
+        count += rows.length;
+        if (!fieldList)
+            for (const row of rows)
+                if (row && typeof row === "object" && !Array.isArray(row))
+                    for (const key of Object.keys(row))
+                        objectKeys.add(key);
+        if (sink)
+            await sink.push(rows);
+        else
+            for (const row of rows)
+                merged.push(row);
+    };
+    const mergeShard = async (r, i) => {
         if (!(r && typeof r === "object"))
-            continue;
+            return;
         const rec = r;
-        if (!header)
-            header = rec;
-        if (!fieldList && Array.isArray(rec.fieldList))
-            fieldList = rec.fieldList;
         if (isTruncated(rec))
             truncatedShards.push(shards[i]);
-        if (Array.isArray(rec.list))
-            for (const item of rec.list)
-                merged.push(item);
-    }
+        if (!Array.isArray(rec.list))
+            return;
+        if (rec.list.length === 0) {
+            if ((typeof rec.total === "number" && rec.total > 0) || rec.partial === true) {
+                failedShards.push(shards[i]);
+                const claim = typeof rec.total === "number" && rec.total > 0 ? `reported total=${rec.total}` : "carried a partial marker";
+                process.stderr.write(`[gangtise] warning: shard ${shards[i].startDate}..${shards[i].endDate} ${claim} but delivered no rows; treated as failed (see failedShards)\n`);
+                return;
+            }
+            if (!fallback)
+                fallback = rec;
+            return;
+        }
+        const shardFields = Array.isArray(rec.fieldList) && rec.fieldList.length > 0 ? rec.fieldList : undefined;
+        const columnar = rec.list.some(Array.isArray);
+        if (columnar && !columnarSchemaValid(shardFields, rec.list)) {
+            failedShards.push(shards[i]);
+            process.stderr.write(`[gangtise] warning: shard ${shards[i].startDate}..${shards[i].endDate} returned columnar rows that do not match its own fieldList (missing, duplicated or mis-sized); its rows were dropped (see failedShards)\n`);
+            return;
+        }
+        if (!fieldList && columnar && shardFields) {
+            fieldList = shardFields;
+            sink?.setFieldList(fieldList);
+        }
+        if (!header)
+            header = rec;
+        if (rec.partial === true)
+            partialShards.push(shards[i]);
+        const remap = columnar && fieldList && shardFields ? columnRemap(fieldList, shardFields) : null;
+        if (remap === undefined) {
+            failedShards.push(shards[i]);
+            process.stderr.write(`[gangtise] warning: shard ${shards[i].startDate}..${shards[i].endDate} returned columns that cannot be aligned with the first shard's fieldList; its rows were dropped (see failedShards)\n`);
+            return;
+        }
+        const rows = rec.list;
+        await keep(remap ? rows.map((item) => (Array.isArray(item) ? remap.map((k) => item[k]) : item)) : rows);
+    };
+    await runInOrder(shards, config.concurrency ?? PAGE_CONCURRENCY, fetchShard, mergeShard);
     if (failedShards.length === shards.length) {
         throw firstError ?? new Error(`All ${shards.length} kline shards failed`);
     }
-    if (!header)
-        return { list: [] };
-    const out = { ...header, total: merged.length, list: merged };
-    if (fieldList)
+    const base = header ?? fallback ?? {};
+    const out = { ...base, total: count, list: merged };
+    if (sink)
+        attachRowSink(out, sink);
+    if (fieldList) {
         out.fieldList = fieldList;
+    }
+    else if (count > 0) {
+        out.fieldList = [...objectKeys];
+    }
+    else if (Array.isArray(base.fieldList)) {
+        out.fieldList = base.fieldList;
+    }
+    else {
+        delete out.fieldList;
+    }
     if (failedShards.length > 0) {
         out.partial = true;
         out.failedShards = failedShards;
         process.stderr.write(`[gangtise] warning: ${failedShards.length}/${shards.length} shards failed; results are partial (see failedShards)\n`);
     }
+    if (partialShards.length > 0) {
+        out.partial = true;
+        process.stderr.write(`[gangtise] warning: ${partialShards.length}/${shards.length} shard(s) reported themselves partial; the merged result is marked partial\n`);
+    }
     if (truncatedShards.length > 0) {
         out.partial = true;
         out.truncatedShards = truncatedShards;

package/dist/src/core/rowSink.js

@@ -0,0 +1,196 @@
+import { createReadStream, createWriteStream } from "node:fs";
+import fs from "node:fs/promises";
+import path from "node:path";
+import { assertColumnarHeader, zipFieldRow } from "./normalize.js";
+import { csvEscape, formatScalar, writeLine } from "./output.js";
+import { extractTitles, MAX_TITLES_PER_ENDPOINT } from "./titleCache.js";
+export const ROW_SINK = Symbol("gangtise.rowSink");
+const STREAM_THRESHOLD = 1000;
+export class ExportSink {
+    outputPath;
+    format;
+    cache;
+    rows = 0;
+    titles = {};
+    titleCount = 0;
+    buffer = [];
+    stream = null;
+    fields;
+    headerChecked = false;
+    finished = false;
+    columns = [];
+    columnSet = new Set();
+    objectRows = 0;
+    constructor(outputPath, format = "jsonl", cache) {
+        this.outputPath = outputPath;
+        this.format = format;
+        this.cache = cache;
+    }
+    get opened() {
+        return this.stream !== null;
+    }
+    get count() {
+        return this.rows + this.buffer.length;
+    }
+    get dataRows() {
+        if (this.format === "jsonl")
+            return this.rows;
+        return this.objectRows > 0 ? this.objectRows : this.rows;
+    }
+    get partPath() {
+        return `${this.outputPath}.part`;
+    }
+    get rowsPath() {
+        return `${this.outputPath}.rows.part`;
+    }
+    setFieldList(fields) {
+        this.fields = fields;
+    }
+    async push(rows) {
+        if (this.stream) {
+            await this.write(rows);
+            return;
+        }
+        for (const row of rows)
+            this.buffer.push(row);
+        if (this.buffer.length < STREAM_THRESHOLD)
+            return;
+        await this.open();
+        const buffered = this.buffer;
+        this.buffer = [];
+        await this.write(buffered);
+    }
+    takeBuffer() {
+        const rows = this.buffer;
+        this.buffer = [];
+        return rows;
+    }
+    async finish() {
+        const stream = this.stream;
+        if (!stream || this.finished)
+            return;
+        try {
+            await endStream(stream);
+            if (this.format === "csv")
+                await this.assembleCsv();
+            await fs.rename(this.partPath, this.outputPath);
+            this.finished = true;
+        }
+        catch (error) {
+            await this.abort();
+            throw error;
+        }
+    }
+    async abort() {
+        const stream = this.stream;
+        if (!stream || this.finished)
+            return;
+        this.finished = true;
+        stream.destroy();
+        if (!stream.closed)
+            await new Promise((resolve) => stream.once("close", resolve));
+        await fs.unlink(this.partPath).catch(() => { });
+        await fs.unlink(this.rowsPath).catch(() => { });
+    }
+    async open() {
+        await fs.mkdir(path.dirname(this.outputPath), { recursive: true });
+        const stream = createWriteStream(this.format === "csv" ? this.rowsPath : this.partPath, { encoding: "utf8" });
+        stream.on("error", () => { });
+        this.stream = stream;
+    }
+    async write(rows) {
+        const stream = this.stream;
+        for (const row of rows) {
+            let item = row;
+            if (Array.isArray(row)) {
+                if (!this.headerChecked) {
+                    assertColumnarHeader(this.fields);
+                    this.headerChecked = true;
+                }
+                item = zipFieldRow(this.fields, row);
+            }
+            if (this.format === "csv" && isObjectRow(item)) {
+                this.objectRows++;
+                for (const key of Object.keys(item)) {
+                    if (!this.columnSet.has(key)) {
+                        this.columnSet.add(key);
+                        this.columns.push(key);
+                    }
+                }
+            }
+            await writeLine(stream, JSON.stringify(item));
+            this.rows++;
+            if (this.cache && this.titleCount < MAX_TITLES_PER_ENDPOINT) {
+                for (const [id, title] of Object.entries(extractTitles([item], this.cache))) {
+                    if (!(id in this.titles))
+                        this.titleCount++;
+                    this.titles[id] = title;
+                }
+            }
+        }
+    }
+    async assembleCsv() {
+        const out = createWriteStream(this.partPath, { encoding: "utf8" });
+        out.on("error", () => { });
+        this.stream = out;
+        const objectMode = this.objectRows > 0;
+        const columns = objectMode ? this.columns : ["index", "value"];
+        await writeLine(out, "\ufeff" + columns.map(csvEscape).join(","));
+        const input = createReadStream(this.rowsPath, { encoding: "utf8" });
+        try {
+            let index = 0;
+            const emit = async (line) => {
+                if (!line)
+                    return;
+                const item = JSON.parse(line);
+                if (objectMode) {
+                    if (!isObjectRow(item))
+                        return;
+                    await writeLine(out, columns.map((column) => csvEscape(formatScalar(item[column]))).join(","));
+                }
+                else {
+                    await writeLine(out, `${csvEscape(String(index++))},${csvEscape(formatScalar(item))}`);
+                }
+            };
+            let rest = "";
+            for await (const chunk of input) {
+                const lines = (rest + String(chunk)).split("\n");
+                rest = lines.pop() ?? "";
+                for (const line of lines)
+                    await emit(line);
+            }
+            await emit(rest);
+            await endStream(out);
+        }
+        catch (error) {
+            input.destroy();
+            throw error;
+        }
+        await fs.unlink(this.rowsPath).catch(() => { });
+    }
+}
+function isObjectRow(value) {
+    return Boolean(value && typeof value === "object" && !Array.isArray(value));
+}
+function endStream(stream) {
+    return new Promise((resolve, reject) => {
+        stream.end((err) => err ? reject(err) : resolve());
+    });
+}
+export function attachRowSink(result, sink) {
+    Object.defineProperty(result, ROW_SINK, { value: sink, enumerable: false, configurable: true });
+    return result;
+}
+export function getRowSink(value) {
+    if (!value || typeof value !== "object")
+        return undefined;
+    const sink = value[ROW_SINK];
+    return sink instanceof ExportSink ? sink : undefined;
+}
+export function rowCount(value) {
+    const sink = getRowSink(value);
+    if (sink)
+        return sink.count;
+    const list = value && typeof value === "object" ? value.list : undefined;
+    return Array.isArray(list) ? list.length : 0;
+}

package/dist/src/core/transport.js

@@ -62,6 +62,57 @@ export async function runWithConcurrency(items, concurrency, fn) {
     await Promise.all(Array.from({ length: limit }, () => worker()));
     return results;
 }
+export async function runInOrder(items, concurrency, fn, consume) {
+    const width = Math.max(1, concurrency);
+    const pending = new Map();
+    let next = 0;
+    let failure = null;
+    let chain = Promise.resolve();
+    let waiters = [];
+    const wake = () => {
+        const ws = waiters;
+        waiters = [];
+        for (const w of ws)
+            w();
+    };
+    const drain = () => {
+        chain = chain.then(async () => {
+            while (!failure && pending.has(next)) {
+                const result = pending.get(next);
+                pending.delete(next);
+                await consume(result, next++);
+                wake();
+            }
+        }).catch((error) => {
+            failure ??= { error };
+            wake();
+        });
+    };
+    await runWithConcurrency(items, width, async (item, index) => {
+        if (failure)
+            return;
+        let result;
+        try {
+            result = await fn(item, index);
+        }
+        catch (error) {
+            failure ??= { error };
+            wake();
+            return;
+        }
+        if (failure)
+            return;
+        pending.set(index, result);
+        drain();
+        while (!failure && pending.size >= width) {
+            await new Promise((resolve) => waiters.push(resolve));
+        }
+    });
+    await chain;
+    const failed = failure;
+    if (failed)
+        throw failed.error;
+}
 export function resolvePageConcurrency(raw, fallback = 5, max = 32) {
     const parsed = Math.floor(Number(raw));
     if (!Number.isFinite(parsed) || parsed < 1)

package/dist/src/version.js

@@ -1 +1 @@
-export const CLI_VERSION = "0.37.0";
+export const CLI_VERSION = "0.38.0";