gangtise-openapi-cli 0.51.0 → 0.52.0

package/dist/src/core/etfComponents.js

@@ -1,6 +1,13 @@
 import { beijingDateString } from "./args.js";
 import { normalizeRows } from "./normalize.js";
 import { PAGE_CONCURRENCY, runWithConcurrency } from "./transport.js";
+function weekdayBefore(day) {
+    const d = new Date(`${day}T00:00:00Z`);
+    do
+        d.setUTCDate(d.getUTCDate() - 1);
+    while (d.getUTCDay() === 0 || d.getUTCDay() === 6);
+    return d.toISOString().slice(0, 10);
+}
 const PRICE_TOLERANCE = 0.0005;
 const agrees = (close, price) => price > 0 && Math.abs(close - price) / price <= PRICE_TOLERANCE;
 const CHANGE_TOLERANCE = 0.005;
@@ -138,7 +145,8 @@ export async function fillComponentCodes(rows, lookup) {
         }
     });
     const quotable = [...new Set([...candidates.values()].flat().filter((c) => ["stock", "dr"].includes(c.category) && QUOTABLE.test(c.code)).map((c) => c.code))];
-    const prices = new Map((await runWithConcurrency(inBatches(quotable, QUOTE_BATCH), PAGE_CONCURRENCY, (batch) => splitting(batch, (codes) => lookup.quotes(codes)))).flatMap((quoted) => [...quoted]));
+    const quotes = new Map((await runWithConcurrency(inBatches(quotable, QUOTE_BATCH), PAGE_CONCURRENCY, (batch) => splitting(batch, (codes) => lookup.quotes(codes)))).flatMap((quoted) => [...quoted]));
+    const prices = new Map([...quotes].map(([code, q]) => [code, [q.previous, q.latest].filter((p) => typeof p === "number")]));
     const isQuotable = (c) => ["stock", "dr"].includes(c.category) && QUOTABLE.test(c.code);
     const candidatesOf = (row) => candidates.get(row.componentName) ?? [];
     const marketsOf = (listed) => new Set(listed.map((c) => market(c.code)));
@@ -185,7 +193,30 @@ export async function fillComponentCodes(rows, lookup) {
                 history.set(code, series);
         }
     }
-    const mayBe = (row, c) => !history.has(c.code) || history.get(c.code).some(([, close]) => agrees(row.close, close));
+    const unshown = (close, c) => {
+        const { previous, latest, date } = quotes.get(c.code) ?? {};
+        const onLatest = latest !== undefined && agrees(close, latest);
+        const onPrevious = previous !== undefined && agrees(close, previous);
+        if (!onLatest && !onPrevious)
+            return false;
+        if (date === undefined)
+            return true;
+        const days = (history.get(c.code) ?? []).map(([day]) => day);
+        const reachesQuote = days.some((day) => day >= date);
+        const reachesPrevious = reachesQuote || days.includes(weekdayBefore(date));
+        return (onLatest && !reachesQuote) || (onPrevious && !reachesPrevious);
+    };
+    const marketDays = new Map([...etfs].map(([fundCode, { own }]) => [fundCode, listDates(own, candidates, history)]));
+    const startsLate = (row, c) => {
+        const first = history.get(c.code)?.[0]?.[0];
+        if (first === undefined)
+            return true;
+        const day = marketDays.get(row.fundCode)?.get(market(c.code));
+        if (day !== undefined)
+            return first > day;
+        return candidatesOf(row).some((o) => o.code !== c.code && (history.get(o.code) ?? []).some(([day, price]) => day < first && agrees(row.close, price)));
+    };
+    const mayBe = (row, c) => !history.has(c.code) || history.get(c.code).some(([, close]) => agrees(row.close, close)) || unshown(row.close, c) || startsLate(row, c);
     for (const { own, etf } of etfs.values()) {
         for (const row of own) {
             const matched = unverified.get(row);
@@ -221,9 +252,14 @@ export async function fillComponentCodes(rows, lookup) {
             if (!pool.length || !pool.every((c) => isQuotable(c) && history.has(c.code)))
                 continue;
             const atClose = (c) => history.get(c.code).filter(([, price]) => agrees(close, price));
-            const hit = pool.filter((c) => atClose(c).some(([, , moved]) => moved !== undefined && Math.abs(moved - change) <= CHANGE_TOLERANCE));
-            const unknown = pool.some((c) => !hit.includes(c) && atClose(c).some(([, , moved]) => moved === undefined));
-            if (!hit.length || unknown || marketsOf(hit).size > 1)
+            const fits = (c) => atClose(c).filter(([, , moved]) => moved !== undefined && Math.abs(moved - change) <= CHANGE_TOLERANCE);
+            const hit = pool.filter((c) => fits(c).length);
+            if (!hit.length)
+                continue;
+            const unknown = pool.some((c) => !hit.includes(c) && (atClose(c).some(([, , moved]) => moved === undefined)
+                || unshown(close, c)
+                || startsLate(row, c)));
+            if (unknown || marketsOf(hit).size > 1)
                 continue;
             const code = narrow(hit, etf);
             if (code)
@@ -367,10 +403,11 @@ export function clientLookup(client) {
             const data = normalizeRows(await client.call("quote.realtime", { securityList: codes }));
             const list = Array.isArray(data) ? data : (data?.list ?? []);
             const out = new Map();
+            const price = (value) => (typeof value === "number" ? value : undefined);
             for (const q of list) {
-                const prices = [q.preClose, q.latestPrice].filter((p) => typeof p === "number");
-                if (typeof q.securityCode === "string" && prices.length)
-                    out.set(q.securityCode, prices);
+                const quote = { previous: price(q.preClose), latest: price(q.latestPrice), date: typeof q.tradeDate === "string" && /^\d{4}-\d{2}-\d{2}$/.test(q.tradeDate) ? q.tradeDate : undefined };
+                if (typeof q.securityCode === "string" && (quote.previous !== undefined || quote.latest !== undefined))
+                    out.set(q.securityCode, quote);
             }
             return out;
         },

package/dist/src/core/exitStatus.js

@@ -1,9 +1,13 @@
 export const EXIT_FAILED = 1;
+export const EXIT_REFUSED = 2;
 export const EXIT_INCOMPLETE = 3;
 export const EXIT_SUPERSEDED = 4;
 export function markFailed() {
     process.exitCode = EXIT_FAILED;
 }
+export function markRefused() {
+    process.exitCode = EXIT_REFUSED;
+}
 export function markIncomplete() {
     if (process.exitCode !== EXIT_FAILED)
         process.exitCode = EXIT_INCOMPLETE;

package/dist/src/core/fileParse.js

@@ -3,11 +3,14 @@ import path from "node:path";
 import { isAsyncPending, pollUntilDone } from "./asyncContent.js";
 import { saveDownloadResult } from "./download.js";
 import { ApiError, ValidationError } from "./errors.js";
+import { assertNotStatePath } from "./statePaths.js";
 export const FILE_PARSE_MAX_BYTES = 100 * 1024 * 1024;
 export async function submitFileParse(client, filePath) {
-    const resolved = path.resolve(filePath);
+    const shownAs = path.resolve(filePath);
+    let resolved;
     let stat;
     try {
+        resolved = await fs.realpath(filePath);
         stat = await fs.stat(resolved);
     }
     catch {
@@ -15,17 +18,21 @@ export async function submitFileParse(client, filePath) {
     }
     if (!stat.isFile())
         throw new ValidationError(`Not a file: ${filePath}`);
+    await assertNotStatePath(resolved, "--file");
     if (stat.size === 0)
         throw new ValidationError(`File is empty: ${filePath}`);
     if (stat.size > FILE_PARSE_MAX_BYTES) {
         throw new ValidationError(`File is ${(stat.size / 1024 / 1024).toFixed(1)}MB — the parse API accepts at most 100MB`);
     }
-    if (path.extname(resolved).toLowerCase() !== ".pdf") {
+    if (path.extname(shownAs).toLowerCase() !== ".pdf") {
         throw new ValidationError(`Only PDF files are supported: ${filePath}`);
     }
     const data = await fs.readFile(resolved);
+    if (!data.subarray(0, 1024).toString("latin1").includes("%PDF-")) {
+        throw new ValidationError(`Not a PDF (no %PDF- header in its first 1KB): ${filePath}`);
+    }
     const result = await client.uploadFile("tool.file-parse.submit", {
-        filename: path.basename(resolved),
+        filename: path.basename(shownAs),
         data,
         contentType: "application/pdf",
     });

package/dist/src/core/printer.js

@@ -6,6 +6,7 @@ import { getRowSink } from "./rowSink.js";
 import { extractTitles, writeTitleCache } from "./titleCache.js";
 import { CLI_VERSION } from "../version.js";
 import { decidedExitCode, EXIT_SUPERSEDED, markIncomplete, markSuperseded } from "./exitStatus.js";
+import { describeNoRequest, isDryRun } from "./dryRun.js";
 const LARGE_RESULT_ROWS = 50_000;
 export const SUPERSEDED_EXIT = EXIT_SUPERSEDED;
 export async function warnIfSuperseded(output, digest) {
@@ -101,19 +102,50 @@ async function stageExportMeta(output, format, rows, columns, normalized, comple
         },
     };
 }
+let selectedColumns;
+export function setColumns(names) {
+    selectedColumns = names.length > 0 ? names : undefined;
+}
+export function columnsSelected() {
+    return selectedColumns !== undefined;
+}
+function projectColumns(value) {
+    const names = selectedColumns;
+    if (!names)
+        return value;
+    const isRecord = (row) => Boolean(row) && typeof row === "object" && !Array.isArray(row);
+    const rows = pickList(value) ?? (isRecord(value) ? [value] : []);
+    const present = new Set(rows.filter(isRecord).flatMap((row) => Object.keys(row)));
+    const absent = names.filter((name) => !present.has(name));
+    if (absent.length > 0 && present.size > 0) {
+        process.stderr.write(`[gangtise] warning: --columns names ${absent.join(", ")}, which no row has; the rows have ${[...present].join(", ")}.\n`);
+    }
+    const pick = (row) => (isRecord(row) ? Object.fromEntries(names.map((name) => [name, row[name] ?? null])) : row);
+    if (Array.isArray(value))
+        return value.map(pick);
+    if (!isRecord(value))
+        return value;
+    return Array.isArray(value.list) ? { ...value, list: value.list.map(pick) } : pick(value);
+}
 export async function printData(data, format, output, cache) {
+    if (isDryRun()) {
+        process.stdout.write(`${JSON.stringify(describeNoRequest(output), null, 2)}\n`);
+        return;
+    }
     const sink = getRowSink(data);
     if (sink && !sink.opened)
         data.list = sink.takeBuffer();
     const streamed = sink?.opened ? sink : undefined;
-    const columns = data && typeof data === "object" && Array.isArray(data.fieldList)
+    const columns = selectedColumns ?? (data && typeof data === "object" && Array.isArray(data.fieldList)
         ? data.fieldList
-        : undefined;
-    const normalized = normalizeRows(data);
+        : undefined);
+    const whole = normalizeRows(data);
+    const wholeItems = pickList(whole);
+    const normalized = projectColumns(whole);
     const items = pickList(normalized);
     const showing = streamed ? streamed.rows : (items?.length ?? 0);
     if (cache) {
-        const titles = { ...(streamed?.titles ?? {}), ...(items ? extractTitles(items, cache) : {}) };
+        const titles = { ...(streamed?.titles ?? {}), ...(wholeItems ? extractTitles(wholeItems, cache) : {}) };
         if (Object.keys(titles).length > 0) {
             writeTitleCache(cache.endpointKey, titles).catch(() => { });
         }
@@ -123,13 +155,13 @@ export async function printData(data, format, output, cache) {
         if (typeof meta.total === "number" && format !== "json") {
             process.stderr.write(`Total: ${meta.total}, showing: ${showing}\n`);
         }
-        if (meta.partial === true)
-            markIncomplete();
     }
+    if (whole && typeof whole === "object" && !Array.isArray(whole) && whole.partial === true)
+        markIncomplete();
     const complete = decidedExitCode() === 0;
     if (output) {
         const staged = format === "csv" || format === "jsonl"
-            ? await stageExportMeta(output, format, streamed ? streamed.dataRows : countOutputRows(normalized, format), columns, normalized, complete)
+            ? await stageExportMeta(output, format, streamed ? streamed.dataRows : countOutputRows(normalized, format), columns, whole, complete)
             : null;
         let digest;
         try {

package/dist/src/core/statePaths.js

@@ -0,0 +1,41 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import { DEFAULT_TOKEN_CACHE_PATH, loadConfig } from "./config.js";
+import { ValidationError } from "./errors.js";
+const identity = async (target) => {
+    const stat = await fs.stat(target).catch(() => undefined);
+    return stat ? `${stat.dev}:${stat.ino}` : undefined;
+};
+async function physical(target) {
+    const rest = [];
+    let head = path.isAbsolute(target) ? target : `${process.cwd()}${path.sep}${target}`;
+    for (;;) {
+        const real = await fs.realpath(head).catch(() => undefined);
+        if (real)
+            return path.join(real, ...rest);
+        const parent = path.dirname(head);
+        if (parent === head)
+            return path.resolve(target);
+        rest.unshift(path.basename(head));
+        head = parent;
+    }
+}
+export async function assertNotStatePath(target, option) {
+    const stateDir = path.dirname(DEFAULT_TOKEN_CACHE_PATH);
+    const tokenCache = loadConfig().tokenCachePath;
+    const guarded = new Set((await Promise.all([stateDir, tokenCache].map(identity))).filter((id) => id !== undefined));
+    const resolved = await physical(target);
+    let refused = resolved === await physical(tokenCache);
+    for (let current = resolved; !refused;) {
+        const id = await identity(current);
+        if (id && guarded.has(id))
+            refused = true;
+        const parent = path.dirname(current);
+        if (parent === current)
+            break;
+        current = parent;
+    }
+    if (refused) {
+        throw new ValidationError(`${option} ${target} points at gangtise's own state (${stateDir}, or the token cache ${tokenCache}), which holds the access token: refused. Pass a path outside it.`);
+    }
+}

package/dist/src/core/transport.js

@@ -34,6 +34,13 @@ function retryAfterFromError(error) {
     return undefined;
 }
 let cachedDispatcher = null;
+let requestSent = false;
+export function noteRequestSent() {
+    requestSent = true;
+}
+export function anyRequestSent() {
+    return requestSent;
+}
 const requireLazily = createRequire(import.meta.url);
 export function getDispatcher() {
     if (!cachedDispatcher) {