@johannc-dev/zyrk 0.0.0-stage → 0.2.1

package/dist/local-data.js

@@ -0,0 +1,373 @@
+import { createHash } from "node:crypto";
+import { existsSync, readdirSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+import { batchOf, decompressHour, encodeHour, parseHour, nodeRealm, COLUMNS, MARKET_COLUMNS, ORDER_COLUMNS, TX_COLUMNS, } from "./hourfile.js";
+import { MAX_WINDOW_MS } from "./limits.js";
+export const HOUR_MS = 3_600_000;
+export const READ_FORMAT = "zyrk-read/1";
+export function spec(input) {
+    const withEvents = !!input.withEvents;
+    return {
+        scope: input.scope,
+        key: input.key,
+        side: input.side ?? "any",
+        withSignatures: !!input.withSignatures,
+        withMarket: !!input.withMarket || withEvents,
+        withEvents,
+        withOrder: !!input.withOrder,
+        withTx: !!input.withTx,
+    };
+}
+const flagsOf = (s) => (s.withSignatures ? "s" : "") +
+    (s.withMarket ? "m" : "") +
+    (s.withEvents ? "e" : "") +
+    (s.withOrder ? "o" : "") +
+    (s.withTx ? "t" : "");
+export function readId(s, sinceMs, untilMs) {
+    const flags = flagsOf(s);
+    return `${s.scope}-${s.key}-${s.side}${flags ? "-" + flags : ""}-${sinceMs}-${untilMs}`;
+}
+const sameSpec = (a, b) => a.scope === b.scope &&
+    a.key === b.key &&
+    a.side === b.side &&
+    a.withSignatures === b.withSignatures &&
+    a.withMarket === b.withMarket &&
+    a.withEvents === b.withEvents &&
+    a.withOrder === b.withOrder &&
+    !!a.withTx === !!b.withTx;
+export function exportCommand(s, sinceMs, untilMs, refresh = false) {
+    return [
+        "zyrk export",
+        `--${s.scope} ${s.key}`,
+        `--since ${new Date(sinceMs).toISOString()}`,
+        `--until ${new Date(untilMs).toISOString()}`,
+        s.side !== "any" ? `--side ${s.side}` : "",
+        s.withSignatures ? "--with-signatures" : "",
+        s.withMarket && !s.withEvents ? "--with-market" : "",
+        s.withEvents ? "--with-events" : "",
+        s.withOrder ? "--with-order" : "",
+        s.withTx ? "--with-tx" : "",
+        refresh ? "--refresh" : "",
+    ]
+        .filter(Boolean)
+        .join(" ");
+}
+export class ReadFailed extends Error {
+    failure;
+    constructor(failure) {
+        super(failure.message);
+        this.failure = failure;
+    }
+}
+const failed = (code, message, retryAfterMs) => new ReadFailed({
+    code,
+    message,
+    ...(retryAfterMs === undefined ? {} : { retryAfterMs }),
+});
+const corrupt = (what, error, again) => failed("local_data_corrupt", `${what} is corrupt (${error instanceof Error ? error.message : String(error)}). Export it again: ${again}`);
+export const sha256 = (bytes) => createHash("sha256").update(bytes).digest("hex");
+const served = (c) => {
+    const sealed = c.sealedUntilMs ?? 0;
+    return c.liveUntilMs == null ? sealed : Math.max(c.liveUntilMs, sealed);
+};
+const anyGap = (c, a, b) => c.gaps === undefined ||
+    c.gaps.some((g) => (g.fromMs == null || g.fromMs < b) && (g.toMs == null || g.toMs > a));
+export function windowComplete(c, since, until) {
+    const horizon = c.retentionHorizonMs ?? Number.MAX_SAFE_INTEGER;
+    return since >= horizon && until <= served(c) && !anyGap(c, since, until);
+}
+export function hourComplete(c, hour, since, until) {
+    const a = Math.max(since, hour * HOUR_MS);
+    const b = Math.min(until, (hour + 1) * HOUR_MS);
+    if (a >= b)
+        return windowComplete(c, since, until);
+    return windowComplete(c, a, b);
+}
+function recut(raw, keep) {
+    const b = batchOf(raw);
+    const h = raw.header;
+    const order = [...keep].sort((x, y) => b.line[x] - b.line[y] || b.op[x] - b.op[y]);
+    const dict = new Map();
+    const addresses = [];
+    const map = (index) => {
+        if (index < 0)
+            return index;
+        const a = b.addresses[index];
+        let i = dict.get(a);
+        if (i === undefined) {
+            i = addresses.length;
+            addresses.push(a);
+            dict.set(a, i);
+        }
+        return i;
+    };
+    const n = keep.length;
+    const remap = new Map();
+    const fields = ["mint", "pool", "trader", "quoteMint"];
+    for (const f of fields)
+        remap.set(f, new Int32Array(b.length).fill(-1));
+    const mcols = ["creatorFeeRecipient", "signer"];
+    for (const f of mcols)
+        remap.set(f, new Int32Array(b.length).fill(-1));
+    for (const i of order) {
+        for (const f of fields)
+            remap.get(f)[i] = map(b[f][i]);
+        if (b.market)
+            for (const f of mcols)
+                remap.get(f)[i] = map(b.market[f][i]);
+    }
+    const pick = (source, name) => {
+        const Ctor = source.constructor;
+        const out = new Ctor(n);
+        const re = name ? remap.get(name) : undefined;
+        keep.forEach((i, j) => (out[j] = re ? re[i] : source[i]));
+        return out;
+    };
+    const cols = b;
+    const next = {
+        hour: h.hour,
+        source: h.source,
+        complete: h.complete,
+        skipped: h.skipped,
+        length: n,
+        addresses,
+    };
+    for (const [name] of COLUMNS)
+        next[name] = pick(cols[name], remap.has(name) ? name : undefined);
+    if (b.signatures)
+        next.signatures = keep.map((i) => b.signatures[i] ?? null);
+    if (b.market) {
+        const m = b.market;
+        next.market = Object.fromEntries(MARKET_COLUMNS.map(([name]) => [
+            name,
+            pick(m[name], remap.has(name) ? name : undefined),
+        ]));
+    }
+    if (b.order) {
+        const o = b.order;
+        next.order = Object.fromEntries(ORDER_COLUMNS.map(([name]) => [name, pick(o[name])]));
+    }
+    if (b.tx) {
+        const t = b.tx;
+        next.tx = Object.fromEntries(TX_COLUMNS.map(([name]) => [name, pick(t[name])]));
+    }
+    return parseHour(encodeHour(next));
+}
+const withComplete = (raw, complete) => raw.header.complete === complete
+    ? raw
+    : { header: { ...raw.header, complete }, bytes: raw.bytes };
+export function batchBytes(h) {
+    const perRow = 4 +
+        2 +
+        14 * 8 +
+        4 +
+        4 * 4 +
+        (h.market ? 75 : 0) +
+        (h.order ? 15 : 0) +
+        (h.tx ? 7 : 0);
+    let strings = 0;
+    for (const a of h.addresses)
+        strings += Buffer.byteLength(a) + 16;
+    for (const s of h.signatures ?? [])
+        strings += (s?.length ?? 0) + 16;
+    return h.length * perRow + strings;
+}
+export class LocalData {
+    dir;
+    #reads;
+    #hours = new Map();
+    constructor(dir) {
+        this.dir = dir;
+    }
+    reads() {
+        if (this.#reads)
+            return this.#reads;
+        const root = join(this.dir, "reads");
+        const out = [];
+        if (existsSync(root))
+            for (const id of readdirSync(root).sort()) {
+                const file = join(root, id, "read.json");
+                if (!existsSync(file))
+                    continue;
+                let meta;
+                try {
+                    meta = JSON.parse(readFileSync(file, "utf8"));
+                }
+                catch (e) {
+                    throw corrupt(file, e, `zyrk export ... --refresh (or delete ${join(root, id)})`);
+                }
+                if (meta.format === READ_FORMAT)
+                    out.push(meta);
+            }
+        return (this.#reads = out);
+    }
+    refresh() {
+        this.#reads = undefined;
+        this.#hours.clear();
+    }
+    find(s, since, until) {
+        return this.reads()
+            .filter((r) => sameSpec(r, s) && r.sinceMs <= since && until <= r.untilMs)
+            .sort((a, b) => a.untilMs - a.sinceMs - (b.untilMs - b.sinceMs))[0];
+    }
+    #raw(meta, hour) {
+        const id = readId(meta, meta.sinceMs, meta.untilMs);
+        const key = `${id}/${hour}`;
+        let raw = this.#hours.get(key);
+        if (!raw) {
+            const file = join(this.dir, "reads", id, `${hour}.zhb`);
+            try {
+                const bytes = new Uint8Array(readFileSync(file));
+                const want = meta.hashes?.[String(hour)];
+                if (want !== undefined && sha256(bytes) !== want)
+                    throw new Error("its SHA-256 is not the one recorded by the export");
+                raw = decompressHour(bytes);
+                if (raw.header.hour !== hour)
+                    throw new Error(`it holds hour ${raw.header.hour}`);
+            }
+            catch (e) {
+                throw corrupt(file, e, exportCommand(meta, meta.sinceMs, meta.untilMs, true));
+            }
+            this.#hours.set(key, raw);
+        }
+        return raw;
+    }
+    readHour(s, since, until, allowPartial, afterHour, realm = nodeRealm) {
+        if (since < 0 || until <= since || until - since > MAX_WINDOW_MS)
+            throw failed("invalid_window", `window must be non-empty, start at 0 or later and span at most ${MAX_WINDOW_MS} ms`);
+        const meta = this.find(s, since, until);
+        if (!meta)
+            throw failed("local_data_missing", `No exported data covers this read. Export it with: ${exportCommand(s, since, until)}`);
+        if (!meta.done)
+            throw failed("local_data_missing", `The export of this read is not finished: run again ${exportCommand(meta, meta.sinceMs, meta.untilMs)}`);
+        const cov = meta.coverage ?? {};
+        if (!allowPartial && !windowComplete(cov, since, until)) {
+            if (cov.preparing)
+                throw failed("history_preparing", "History for the requested period is still being prepared. Please retry later, or read with allowPartial.", 30_000);
+            throw failed("partial_coverage", `the window is not fully covered; retry with allowPartial (coverage: ${JSON.stringify(cov)})`);
+        }
+        const exact = since === meta.sinceMs && until === meta.untilMs;
+        const sealedUntil = cov.sealedUntilMs ?? 0;
+        const stale = (source, h) => source === "live" || (h + 1) * HOUR_MS > sealedUntil;
+        const files = new Set(meta.hours);
+        const first = Math.max(Math.floor(since / HOUR_MS) - 1, afterHour === null ? -Infinity : afterHour + 1);
+        const last = Math.floor((until - 1) / HOUR_MS) + 1;
+        const inWindow = (h) => h >= Math.floor(since / HOUR_MS) &&
+            h <= Math.floor((until - 1) / HOUR_MS);
+        for (let hour = first; hour <= last; hour++) {
+            let raw = files.has(hour) ? this.#raw(meta, hour) : undefined;
+            let recuts = false;
+            if (raw && !exact) {
+                const times = raw.header.skippedTimes;
+                const b = batchOf(raw);
+                const keep = [];
+                for (let i = 0; i < b.length; i++) {
+                    const t = b.timeMs[i];
+                    if (t >= since && t < until)
+                        keep.push(i);
+                }
+                if (keep.length !== b.length) {
+                    raw = recut(raw, keep);
+                    recuts = true;
+                }
+                if (times) {
+                    const kept = times.filter((t) => t === null || (t >= since && t < until));
+                    if (kept.length !== raw.header.skipped)
+                        raw = {
+                            header: {
+                                ...raw.header,
+                                skipped: kept.length,
+                                skippedTimes: kept,
+                            },
+                            bytes: raw.bytes,
+                        };
+                }
+                raw = withComplete(raw, raw.header.skipped === 0 && hourComplete(cov, hour, since, until));
+            }
+            if (!raw || (raw.header.length === 0 && raw.header.skipped === 0)) {
+                if (inWindow(hour) && !hourComplete(cov, hour, since, until)) {
+                    const source = raw?.header.source ??
+                        (hour * HOUR_MS < (cov.sealedUntilMs ?? 0) ? "sealed" : "live");
+                    const empty = {
+                        hour,
+                        source,
+                        complete: false,
+                        skipped: 0,
+                        length: 0,
+                        addresses: [],
+                        ...(s.withSignatures ? { signatures: [] } : {}),
+                    };
+                    for (const [name] of COLUMNS)
+                        empty[name] = new Uint8Array(0);
+                    const parsed = parseHour(encodeHour(empty));
+                    return {
+                        batch: batchOf(parsed, realm),
+                        bytes: 0,
+                        recut: false,
+                        stale: stale(source, hour),
+                    };
+                }
+                continue;
+            }
+            if (raw.header.skipped > 0 && !allowPartial)
+                throw failed("partial_coverage", `hour ${hour} holds ${raw.header.skipped} operation(s) of this scope that normalization rejects; retry with allowPartial`);
+            return {
+                batch: batchOf(raw, realm),
+                bytes: batchBytes(raw.header),
+                recut: recuts,
+                stale: stale(raw.header.source, hour) ||
+                    (raw.header.market && !raw.header.virtualSides),
+            };
+        }
+        return null;
+    }
+    #json(file, again) {
+        if (!existsSync(file))
+            return undefined;
+        try {
+            return JSON.parse(readFileSync(file, "utf8"));
+        }
+        catch (e) {
+            throw corrupt(file, e, again);
+        }
+    }
+    #failure(file, again) {
+        const saved = this.#json(join(this.dir, "errors", file), again);
+        if (saved === undefined)
+            return undefined;
+        const { code, message } = saved?.error ?? {};
+        if (typeof code !== "string" || typeof message !== "string")
+            throw corrupt(join(this.dir, "errors", file), new Error("not an error answer"), again);
+        const retry = saved.retryAfterMs;
+        return {
+            code,
+            message,
+            ...(typeof retry === "number" ? { retryAfterMs: retry } : {}),
+        };
+    }
+    totals(mint, quote) {
+        return this.#json(join(this.dir, "totals", totalsName(mint, quote)), `zyrk export --totals ${mint}${quote ? ` --quote ${quote}` : ""}`);
+    }
+    info(mint) {
+        return this.#json(join(this.dir, "info", infoName(mint)), `zyrk export --totals ${mint}`);
+    }
+    totalsFailure(mint, quote) {
+        return this.#failure(join("totals", totalsName(mint, quote)), `zyrk export --totals ${mint}${quote ? ` --quote ${quote}` : ""}`);
+    }
+    infoFailure(mint) {
+        return this.#failure(join("info", infoName(mint)), `zyrk export --totals ${mint}`);
+    }
+    volumeFailure(params) {
+        return this.#failure(join("volume", `${volumeName(params)}.json`), "zyrk export --volume-search '<params>'");
+    }
+    volumePage(params) {
+        const saved = this.#json(join(this.dir, "volume", `${volumeName(params)}.json`), "zyrk export --volume-search '<params>'");
+        return saved?.page;
+    }
+}
+export const totalsName = (mint, quote) => quote ? `${mint}~${quote}.json` : `${mint}.json`;
+export const infoName = (mint) => `${mint}.json`;
+export function volumeName(params) {
+    const canonical = JSON.stringify([...params].sort((a, b) => (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0)));
+    return createHash("sha256").update(canonical).digest("hex").slice(0, 32);
+}

package/dist/local-replay.d.ts

@@ -0,0 +1,40 @@
+import { type LocalData, type ReadMeta } from "./local-data.js";
+import { type HourBatch, type Subscription } from "./sdk.js";
+/** The first 30 s after coverage starts are not trusted (as the service). */
+export declare const LAG_MS = 30000;
+/** Events per delivery, as `delivery_batch` of the service. */
+export declare const DELIVERY_BATCH = 32;
+/** Largest `batchSize` of a replay of the service. */
+export declare const REPLAY_MAX_BATCH = 1000;
+/** History items per source page of the service's replays. */
+export declare const SOURCE_PAGE = 500;
+/**
+ * Fields of the live operation JSON that exported
+ * columns do not hold: absent from replayed operations.
+ */
+export declare const MISSING_OPERATION_FIELDS: readonly ["prices (provider strings: read batch.price… instead)", "usdValue", "finality", "provenance", "market (the provider's raw object)"];
+export interface PendingEvent {
+    readonly event: Record<string, unknown>;
+    /** Time of the event (a gap: its start). */
+    readonly time: number;
+    readonly gap: boolean;
+}
+export interface ReplayProgress {
+    eventsRead: number;
+    eventsDelivered: number;
+    deliveries: number;
+    gaps: number;
+    orderFallbacks: number;
+}
+/** Decimal string of a raw amount, as the live JSON writes it. */
+export declare function formatRaw(raw: bigint, decimals: number): string;
+/** Row `i` as the live publication shapes it (without {@link MISSING_OPERATION_FIELDS}). */
+export declare function operationOf(b: HourBatch, i: number): Record<string, unknown>;
+/**
+ * Every event a subscription receives over `[since, until)`, from the
+ * export `meta` (a read of the trigger's token, or wallet), in delivery
+ * order.
+ */
+export declare function replayEvents(data: LocalData, meta: ReadMeta, sub: Subscription, since: number, until: number, progress: ReplayProgress): PendingEvent[];
+/** `interval`: ticks on the simulated clock, no history read, no gap. */
+export declare function intervalEvents(everyMs: number, since: number, until: number): PendingEvent[];