@johannc-dev/zyrk 0.0.0-stage → 0.2.1

package/dist/local-replay.js

@@ -0,0 +1,343 @@
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import { batchOf, decompressHour } from "./hourfile.js";
+import { HOUR_MS, readId, } from "./local-data.js";
+import { MarketEvent, MarketFlag, NO_EVENT_INDEX, NO_TX_ID, NO_TX_INDEX, TxFlag, txFlagNames, OrderFlag, } from "./sdk.js";
+export const LAG_MS = 30_000;
+export const DELIVERY_BATCH = 32;
+export const REPLAY_MAX_BATCH = 1000;
+export const SOURCE_PAGE = 500;
+export const MISSING_OPERATION_FIELDS = [
+    "prices (provider strings: read batch.price… instead)",
+    "usdValue",
+    "finality",
+    "provenance",
+    "market (the provider's raw object)",
+];
+const ORDER_FLAG_NAMES = Object.entries(OrderFlag);
+const POOL_KINDS = new Map(Object.entries(MarketEvent)
+    .filter(([, v]) => v >= 1 && v <= 5)
+    .map(([k, v]) => [v, k]));
+export function formatRaw(raw, decimals) {
+    if (decimals === 0)
+        return raw.toString();
+    const s = raw.toString().padStart(decimals + 1, "0");
+    const whole = s.slice(0, -decimals);
+    const frac = s.slice(-decimals).replace(/0+$/, "");
+    return frac ? `${whole}.${frac}` : whole;
+}
+const addressAt = (b, index) => index === undefined || index < 0 ? null : (b.addresses[index] ?? null);
+const orNull = (v) => v === undefined || Number.isNaN(v) ? null : v;
+export function operationOf(b, i) {
+    const ord = `${b.hour}:${b.line[i]}:${b.op[i]}`;
+    const o = b.order;
+    const known = o !== undefined && o.txIndex[i] !== NO_TX_INDEX;
+    const flags = o?.flags[i] ?? OrderFlag.unknownNotCovered;
+    const t = b.tx;
+    const txId = t?.id[i] ?? NO_TX_ID;
+    const txKnown = txId !== NO_TX_ID;
+    return {
+        id: ord,
+        mint: addressAt(b, b.mint[i]) ?? "",
+        pool: addressAt(b, b.pool[i]),
+        protocol: b.protocol[i] === 1 ? "pumpswap" : "pumpfun",
+        trader: addressAt(b, b.trader[i]),
+        signature: b.signatures?.[i] ?? null,
+        side: b.side[i] === 1 ? "sell" : "buy",
+        token: {
+            raw: b.tokenRaw[i].toString(),
+            amount: formatRaw(b.tokenRaw[i], b.tokenDecimals[i]),
+            decimals: b.tokenDecimals[i],
+        },
+        quote: {
+            mint: addressAt(b, b.quoteMint[i]) ?? "",
+            raw: b.quoteRaw[i].toString(),
+            amount: formatRaw(b.quoteRaw[i], b.quoteDecimals[i]),
+            decimals: b.quoteDecimals[i],
+        },
+        time: {
+            ms: b.timeMs[i],
+            precision: "provider_timestamp_ms",
+            receivedMs: orNull(b.receivedMs[i]),
+        },
+        order: {
+            slot: b.slot[i],
+            ord,
+            basis: "provider_hour_line_operation",
+            txIndex: known ? o.txIndex[i] : null,
+            eventIndex: known && o.eventIndex[i] !== NO_EVENT_INDEX ? o.eventIndex[i] : null,
+            blockTimeMs: o ? orNull(o.blockTimeMs[i]) : null,
+            flags: ORDER_FLAG_NAMES.filter(([, bit]) => (flags & bit) !== 0).map(([name]) => name),
+        },
+        tx: {
+            id: txKnown ? txId : null,
+            index: txKnown ? t.index[i] : null,
+            flags: txFlagNames(t?.flags[i] ?? TxFlag.unknown),
+        },
+        publishSeq: null,
+        cumulativeQuoteRawAfter: null,
+    };
+}
+function poolEventOf(b, i, kind) {
+    const m = b.market;
+    const flags = m.flags[i];
+    const reserves = (flags & MarketFlag.reserves) !== 0;
+    const vToken = (flags & MarketFlag.virtualTokenReserve) !== 0;
+    const vQuote = (flags & MarketFlag.virtualQuoteReserve) !== 0;
+    const op = operationOf(b, i);
+    return {
+        ...op,
+        type: "poolEvent",
+        kind,
+        code: m.event[i],
+        trader: kind === "add" || kind === "remove" ? op.trader : null,
+        market: {
+            tokenReserveRaw: reserves ? m.tokenReserve[i].toString() : null,
+            quoteReserveRaw: reserves ? m.quoteReserve[i].toString() : null,
+            virtualTokenReserveRaw: vToken
+                ? m.virtualTokenReserve[i].toString()
+                : null,
+            virtualQuoteReserveRaw: vQuote
+                ? m.virtualQuoteReserve[i].toString()
+                : null,
+            signer: addressAt(b, m.signer[i]),
+        },
+    };
+}
+function hourRows(b, since, until) {
+    const rows = [];
+    for (let i = 0; i < b.length; i++) {
+        const t = b.timeMs[i];
+        if (t >= since && t < until)
+            rows.push(i);
+    }
+    const o = b.order;
+    if (!o)
+        return rows.sort((x, y) => b.line[x] - b.line[y] || b.op[x] - b.op[y]);
+    const lineFirst = new Map();
+    for (let i = 0; i < b.length; i++)
+        if (b.side[i] !== 2) {
+            const at = lineFirst.get(b.line[i]);
+            if (at === undefined || b.op[i] < b.op[at])
+                lineFirst.set(b.line[i], i);
+        }
+    const pos = (i) => {
+        const j = b.side[i] === 2 ? (lineFirst.get(b.line[i]) ?? i) : i;
+        return [b.slot[j], o.txIndex[j], o.eventIndex[j]];
+    };
+    return rows.sort((x, y) => {
+        const p = pos(x);
+        const q = pos(y);
+        return (p[0] - q[0] ||
+            p[1] - q[1] ||
+            p[2] - q[2] ||
+            b.line[x] - b.line[y] ||
+            b.op[x] - b.op[y]);
+    });
+}
+class Windows {
+    #covered = null;
+    #series = new Map();
+    reset() {
+        this.#series.clear();
+        this.#covered = null;
+    }
+    push(key, time, seq, raw) {
+        let s = this.#series.get(key);
+        if (!s)
+            this.#series.set(key, (s = { points: [] }));
+        const last = s.points[s.points.length - 1];
+        const t = Math.max(time, last?.time ?? 0, 0);
+        s.points.push({ time: t, seq, cum: (last?.cum ?? 0n) + raw });
+        this.#covered ??= t + LAG_MS;
+    }
+    lookup(key, seq, windowMs) {
+        const s = this.#series.get(key);
+        if (!s || this.#covered === null)
+            return null;
+        const i = s.points.findIndex((p) => p.seq === seq);
+        if (i < 0)
+            return null;
+        const entry = s.points[i];
+        const start = entry.time - windowMs;
+        if (start < this.#covered)
+            return null;
+        let j = 0;
+        while (j < s.points.length && s.points[j].time <= start)
+            j++;
+        const atStart = j === 0 ? 0n : s.points[j - 1].cum;
+        const previous = i === 0 ? 0n : s.points[i - 1].cum;
+        return { before: previous - atStart, after: entry.cum - atStart };
+    }
+}
+const gapEvent = (reason, from, to, after) => ({
+    kind: "gap",
+    reason,
+    fromMs: from,
+    toMs: to,
+    after,
+});
+export function replayEvents(data, meta, sub, since, until, progress) {
+    const t = sub.trigger;
+    const out = [];
+    const cov = meta.coverage ?? {};
+    const gaps = [];
+    const horizon = cov.retentionHorizonMs ?? 0;
+    if (since < horizon)
+        gaps.push([since, Math.min(horizon, until), "retention"]);
+    for (const g of cov.gaps ?? []) {
+        const from = Math.max(g.fromMs ?? since, since);
+        const to = Math.min(g.toMs ?? until, until);
+        if (from < to)
+            gaps.push([from, to, "history_gap"]);
+    }
+    const sealed = cov.sealedUntilMs ?? Number.MAX_SAFE_INTEGER;
+    if (sealed < until)
+        gaps.push([Math.max(sealed, since), until, "not_sealed"]);
+    gaps.sort((a, b) => a[0] - b[0] || a[1] - b[1] || (a[2] < b[2] ? -1 : 1));
+    let seq = 0;
+    const totals = new Map();
+    const windows = t.kind === "volume" && t.windowMs !== undefined ? new Windows() : null;
+    let lastItem = null;
+    let run = null;
+    const closeRun = () => {
+        if (run && lastItem)
+            out.push({
+                event: {
+                    kind: "gap",
+                    reason: "window_not_observed",
+                    after: run.first - 1,
+                    resumeAfter: lastItem.seq,
+                    missed: run.missed,
+                },
+                time: lastItem.time,
+                gap: true,
+            });
+        run = null;
+    };
+    const gapsUntil = (time) => {
+        if (gaps.length && gaps[0][0] <= time)
+            closeRun();
+        while (gaps.length && gaps[0][0] <= time) {
+            const [from, to, reason] = gaps.shift();
+            out.push({
+                event: gapEvent(reason, from, to, seq),
+                time: from,
+                gap: true,
+            });
+            seq++;
+            windows?.reset();
+            for (const v of totals.values())
+                v.complete = false;
+            progress.gaps++;
+        }
+    };
+    const id = readId(meta, meta.sinceMs, meta.untilMs);
+    const first = Math.floor(since / HOUR_MS) - 1;
+    const last = Math.floor((until - 1) / HOUR_MS) + 1;
+    for (const hour of meta.hours) {
+        if (hour < first || hour > last)
+            continue;
+        const b = batchOf(decompressHour(readFileSync(join(data.dir, "reads", id, `${hour}.zhb`))));
+        if (b.skipped > 0)
+            for (const v of totals.values())
+                v.complete = false;
+        for (const i of hourRows(b, since, until)) {
+            const time = b.timeMs[i];
+            gapsUntil(time);
+            seq++;
+            progress.eventsRead++;
+            const event = b.side[i] === 2;
+            const kind = event ? POOL_KINDS.get(b.market?.event[i] ?? 0) : undefined;
+            if (event && !kind)
+                continue;
+            const item = event ? poolEventOf(b, i, kind) : operationOf(b, i);
+            item.publishSeq = seq;
+            const fallback = item.order.txIndex === null;
+            const mint = item.mint;
+            const quote = item.quote.mint;
+            if (!event) {
+                const key = `${mint}/${quote}`;
+                const create = b.market !== undefined &&
+                    (b.market.flags[i] & MarketFlag.creatorInitialBuy) !== 0;
+                if (create && !totals.has(key))
+                    totals.set(key, { total: 0n, complete: true });
+                const v = totals.get(key);
+                if (v)
+                    v.total += b.quoteRaw[i];
+                item.cumulativeQuoteRawAfter = v ? v.total.toString() : null;
+                item.sinceCreationComplete = v?.complete ?? false;
+                item.signalEligible = true;
+                windows?.push(key, time, seq, b.quoteRaw[i]);
+            }
+            const classified = [];
+            if (t.kind === "poolEvent") {
+                if (event &&
+                    (!t.mint || t.mint === mint) &&
+                    (!t.events || t.events.includes(kind)))
+                    classified.push({ kind: "poolEvent", event: item });
+            }
+            else if (!event && t.kind === "trade") {
+                if ((!t.mint || t.mint === mint) &&
+                    (!t.wallet || t.wallet === item.trader))
+                    classified.push({ kind: "trade", operation: item });
+            }
+            else if (!event && t.kind === "volume") {
+                const threshold = BigInt(t.thresholdRaw);
+                if ((!t.mint || t.mint === mint) && (!t.quote || t.quote === quote)) {
+                    if (t.windowMs === undefined) {
+                        const total = item.cumulativeQuoteRawAfter === null
+                            ? null
+                            : BigInt(item.cumulativeQuoteRawAfter);
+                        const amount = b.quoteRaw[i];
+                        if (total !== null &&
+                            item.sinceCreationComplete === true &&
+                            total < threshold + amount &&
+                            threshold <= total)
+                            classified.push({ kind: "volume", operation: item });
+                    }
+                    else {
+                        const w = windows.lookup(`${mint}/${quote}`, seq, t.windowMs);
+                        if (!w) {
+                            run ??= { first: seq, missed: 0 };
+                            run.missed++;
+                        }
+                        else {
+                            closeRun();
+                            if (w.before < threshold && threshold <= w.after)
+                                classified.push({
+                                    kind: "volume",
+                                    operation: item,
+                                    windowMs: t.windowMs,
+                                    windowQuoteRaw: w.after.toString(),
+                                });
+                        }
+                    }
+                }
+            }
+            for (const e of classified) {
+                if (fallback) {
+                    e.orderFallback = true;
+                    progress.orderFallbacks++;
+                }
+                out.push({ event: e, time, gap: false });
+            }
+            lastItem = { seq, time };
+            if (progress.eventsRead % SOURCE_PAGE === 0)
+                closeRun();
+        }
+    }
+    closeRun();
+    gapsUntil(Number.MAX_SAFE_INTEGER);
+    return out;
+}
+export function intervalEvents(everyMs, since, until) {
+    const out = [];
+    for (let at = since; at < until; at += everyMs)
+        out.push({
+            event: { kind: "interval", scheduledAt: at },
+            time: at,
+            gap: false,
+        });
+    return out;
+}

package/dist/local-storage.d.ts

@@ -0,0 +1,76 @@
+/** Defaults of `storageLimits` in the user-storage branch. */
+export declare const localStorageLimits: {
+    readonly maxObjectBytes: number;
+    readonly chunkBytes: number;
+    readonly maxKeyBytes: 512;
+    readonly maxMetadataEntries: 16;
+    readonly maxMetadataBytes: 4096;
+    readonly maxList: 1000;
+    readonly maxUploads: 4;
+    /** Bytes read plus written by one execution (`io_limit`). */
+    readonly ioBytes: number;
+    /** Quota cost floor of one object. */
+    readonly minObjectBytes: 4096;
+    /** Objects per area. */
+    readonly maxObjects: 100000;
+};
+export interface AreaConfig {
+    readonly quotaBytes: number;
+    /** `null`: objects never expire (`durable`). */
+    readonly ttlMs: number | null;
+}
+/** The service's defaults; add an area here to test another one. */
+export declare const defaultAreas: Readonly<Record<string, AreaConfig>>;
+export declare class StorageOpError extends Error {
+    readonly kind: "program" | "limit" | "infrastructure";
+    readonly code: string;
+    constructor(kind: "program" | "limit" | "infrastructure", code: string, message: string);
+}
+/**
+ * The storage areas of one local account. `dir` keeps them between runs
+ * (`<dir>/<area>/index.json` and one file per object); without it they live
+ * in memory.
+ */
+export declare class LocalStorage {
+    #private;
+    readonly dir?: string | undefined;
+    readonly areas: Readonly<Record<string, AreaConfig>>;
+    readonly now: () => number;
+    /**
+     * A replay's space: every area counts in one quota (the account's
+     * `short`), together with `outside()` bytes the account already uses.
+     */
+    readonly shared?: {
+        quotaBytes: number;
+        outside: () => number;
+    } | undefined;
+    constructor(dir?: string | undefined, areas?: Readonly<Record<string, AreaConfig>>, now?: () => number, 
+    /**
+     * A replay's space: every area counts in one quota (the account's
+     * `short`), together with `outside()` bytes the account already uses.
+     */
+    shared?: {
+        quotaBytes: number;
+        outside: () => number;
+    } | undefined);
+    /** Committed and reserved bytes of an area (what its quota counts). */
+    usedBytes(area: string): number;
+    /**
+     * Copies the live objects of `from` in `source` (a replay's space), or
+     * only `keys` (absent ones skipped), to area `to` here, within its quota:
+     * refused as a whole (`quota_exceeded`, nothing copied) when they do not fit.
+     */
+    copyFrom(source: LocalStorage, from: string, to: string, keys?: readonly string[]): {
+        objects: number;
+        bytes: number;
+    };
+    /** Uploads of an execution that did not commit are dropped when it ends. */
+    endExecution(owner: string): void;
+    /**
+     * One `Storage*` host operation of execution `owner`. `io` counts the bytes
+     * moved by the execution (`io_limit`).
+     */
+    op(name: string, a: Record<string, unknown>, owner: string, io: {
+        bytes: number;
+    }): unknown;
+}