@johannc-dev/zyrk 0.0.0-stage → 0.2.0

package/dist/export.d.ts

@@ -0,0 +1,87 @@
+import { type ReadMeta } from "./local-data.js";
+import type { Json, VolumeSearchParams } from "./sdk.js";
+export interface ExportConnection {
+    /** Gateway URL (`TH_API_URL`). */
+    readonly url: string;
+    /** API key (`TH_API_KEY`) of an account with hosted functions. */
+    readonly apiKey: string;
+    /** At most this many requests per minute (default: as fast as the server allows). */
+    readonly rpm?: number;
+    /** Network or 5xx failures tolerated in a row before giving up (default 8). */
+    readonly maxRetries?: number;
+    /** Longest wait accepted on a 429 before stopping (default 5 minutes): a daily quota stops the export. */
+    readonly maxWaitMs?: number;
+    readonly fetch?: typeof fetch;
+    readonly signal?: AbortSignal;
+    readonly onProgress?: (progress: ExportProgress) => void;
+    /** Hours that may still change (open, not sealed), files downloaded again. */
+    readonly onWarning?: (message: string) => void;
+}
+export interface ExportProgress {
+    readonly read: string;
+    readonly hour: number | null;
+    readonly hours: number;
+    readonly done: boolean;
+    readonly bytes: number;
+}
+export interface ExportRead {
+    readonly scope: "token" | "wallet";
+    readonly key: string;
+    readonly sinceMs: number;
+    readonly untilMs: number;
+    readonly side?: "any" | "buy" | "sell";
+    readonly withSignatures?: boolean;
+    readonly withMarket?: boolean;
+    readonly withEvents?: boolean;
+    readonly withOrder?: boolean;
+    readonly withTx?: boolean;
+    /** Download every hour again, even those of a finished export. */
+    readonly refresh?: boolean;
+}
+export declare class ExportError extends Error {
+    readonly code: string;
+    readonly status?: number | undefined;
+    readonly name = "ExportError";
+    constructor(message: string, code: string, status?: number | undefined);
+}
+export interface ExportSummary {
+    readonly reads: readonly ReadMeta[];
+    readonly requests: number;
+    /** Compressed bytes written. */
+    readonly bytes: number;
+}
+/** `[since, until)` in parts of at most 30 days, ending on UTC day boundaries. */
+export declare function parts(sinceMs: number, untilMs: number): [number, number][];
+/** Meta of an export answer (`history_columns::export::hour_response`). */
+interface HourMeta {
+    readonly format?: string;
+    readonly coverage?: ReadMeta["coverage"];
+    readonly batch?: {
+        hour: number;
+        zstdBytes: number;
+    } | null;
+    readonly resumeAfterHour?: number | null;
+    readonly done?: boolean;
+}
+/** `"ZXH1" | u32 LE meta length | meta JSON | exported hour`. */
+export declare function parseFrame(bytes: Uint8Array): {
+    meta: HourMeta;
+    file: Uint8Array;
+};
+/**
+ * The first hour to download again in a finished export: an hour that was
+ * open (`live`) or not sealed yet when it was exported may have changed.
+ * `null`: nothing to refresh.
+ */
+export declare function staleFrom(meta: ReadMeta, dir: string): number | null;
+/** Exports one read (cut into parts of at most 30 days). Resumes an interrupted export. */
+export declare function exportHistory(connection: ExportConnection, out: string, read: ExportRead): Promise<ExportSummary>;
+/** Saves `/v1/tokens/{mint}/totals` for `ctx.data.tokenTotals`. */
+export declare function exportTotals(connection: ExportConnection, out: string, mint: string, quote?: string | null): Promise<Json>;
+/**
+ * Saves every page of `/v1/volume/search` for `params`, following the
+ * cursor exactly as `ctx.data.volumeSearch` does, so that the same call in
+ * a local run finds each page.
+ */
+export declare function exportVolumeSearch(connection: ExportConnection, out: string, params: VolumeSearchParams): Promise<number>;
+export {};

package/dist/export.js

@@ -0,0 +1,338 @@
+import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, writeFileSync, } from "node:fs";
+import { join } from "node:path";
+import { decompressHour } from "./hourfile.js";
+import { HOUR_MS, READ_FORMAT, sha256, readId, spec as readSpec, totalsName, volumeName, } from "./local-data.js";
+import { MAX_WINDOW_MS } from "./limits.js";
+const DAY_MS = 86_400_000;
+const PART_MS = 30 * DAY_MS;
+export class ExportError extends Error {
+    code;
+    status;
+    name = "ExportError";
+    constructor(message, code, status) {
+        super(message);
+        this.code = code;
+        this.status = status;
+    }
+}
+export function parts(sinceMs, untilMs) {
+    if (!(untilMs > sinceMs) || sinceMs < 0)
+        throw new ExportError("--until must be after --since", "invalid_window");
+    const out = [];
+    for (let a = sinceMs; a < untilMs;) {
+        const b = Math.min(untilMs, Math.floor(a / DAY_MS) * DAY_MS + PART_MS);
+        out.push([a, b]);
+        a = b;
+    }
+    return out;
+}
+const sleep = (ms, signal) => new Promise((resolve, reject) => {
+    if (signal?.aborted)
+        return reject(signal.reason);
+    const t = setTimeout(resolve, ms);
+    signal?.addEventListener("abort", () => {
+        clearTimeout(t);
+        reject(signal.reason);
+    }, { once: true });
+});
+const writeAtomic = (file, data) => {
+    const tmp = `${file}.tmp-${process.pid}`;
+    writeFileSync(tmp, data);
+    renameSync(tmp, file);
+};
+class Http {
+    c;
+    requests = 0;
+    #last = 0;
+    constructor(c) {
+        this.c = c;
+    }
+    async raw(path, query) {
+        const f = this.c.fetch ?? fetch;
+        const url = new URL(this.c.url.replace(/\/+$/, "") + path);
+        for (const [k, v] of Object.entries(query))
+            url.searchParams.set(k, v);
+        let failures = 0;
+        for (;;) {
+            if (this.c.rpm) {
+                const gap = 60_000 / this.c.rpm - (Date.now() - this.#last);
+                if (gap > 0)
+                    await sleep(gap, this.c.signal);
+            }
+            this.#last = Date.now();
+            this.requests++;
+            let response;
+            let bytes;
+            try {
+                response = await f(url, {
+                    headers: { authorization: `Bearer ${this.c.apiKey}` },
+                    ...(this.c.signal ? { signal: this.c.signal } : {}),
+                });
+                bytes = new Uint8Array(await response.arrayBuffer());
+            }
+            catch (error) {
+                if (this.c.signal?.aborted)
+                    throw error;
+                if (++failures > (this.c.maxRetries ?? 8))
+                    throw new ExportError(`Network error on ${url.pathname}: ${String(error)}`, "network");
+                await sleep(Math.min(30_000, 500 * 2 ** failures), this.c.signal);
+                continue;
+            }
+            const contentType = response.headers.get("content-type") ?? "";
+            if (response.ok)
+                return { status: response.status, contentType, bytes };
+            let body = null;
+            try {
+                body = JSON.parse(new TextDecoder().decode(bytes));
+            }
+            catch {
+                body = null;
+            }
+            const err = body?.error;
+            const code = typeof err === "string"
+                ? err
+                : typeof err?.code === "string"
+                    ? err.code
+                    : `http_${response.status}`;
+            const message = err?.message ?? code;
+            const after = response.headers.get("retry-after");
+            const header = after === null ? Number.NaN : Number(after);
+            const bodyRetry = body
+                ?.retryAfterMs;
+            const retryMs = typeof bodyRetry === "number"
+                ? bodyRetry
+                : Number.isFinite(header) && header >= 0
+                    ? header * 1000
+                    : undefined;
+            if (response.status === 429) {
+                const wait = retryMs ?? 60_000;
+                if (wait > (this.c.maxWaitMs ?? 300_000))
+                    throw new ExportError(`${code}: ${message} Retry after ${Math.round(wait / 1000)} s: run the same command again then, the export resumes where it stopped.`, code, 429);
+                await sleep(wait, this.c.signal);
+                continue;
+            }
+            if (response.status >= 500 && code !== "history_timeout") {
+                if (++failures > (this.c.maxRetries ?? 8))
+                    throw new ExportError(`${response.status} ${code}: ${message}`, code, response.status);
+                await sleep(retryMs ?? Math.min(30_000, 500 * 2 ** failures), this.c.signal);
+                continue;
+            }
+            throw new ExportError(`${response.status} ${code}: ${message}`, code, response.status);
+        }
+    }
+    async get(path, query) {
+        const answer = await this.raw(path, query);
+        const text = new TextDecoder().decode(answer.bytes);
+        try {
+            return text ? JSON.parse(text) : null;
+        }
+        catch {
+            throw new ExportError(`${path}: the answer is not JSON`, "decode");
+        }
+    }
+}
+export function parseFrame(bytes) {
+    const magic = String.fromCharCode(...bytes.subarray(0, 4));
+    if (bytes.length < 8 || magic !== "ZXH1")
+        throw new ExportError("Unexpected answer from the history route (not an export frame)", "decode");
+    const n = new DataView(bytes.buffer, bytes.byteOffset).getUint32(4, true);
+    if (8 + n > bytes.length)
+        throw new ExportError("Truncated answer of the history route", "decode");
+    const meta = JSON.parse(new TextDecoder().decode(bytes.subarray(8, 8 + n)));
+    if (meta.format !== "zyrk-hour/2")
+        throw new ExportError(`Unexpected answer from the history route (format ${String(meta.format)}): update the SDK`, "decode");
+    const file = bytes.subarray(8 + n);
+    if (meta.batch && file.length !== meta.batch.zstdBytes)
+        throw new ExportError("Truncated answer of the history route", "decode");
+    return { meta, file };
+}
+function loadMeta(file) {
+    if (!existsSync(file))
+        return undefined;
+    try {
+        return JSON.parse(readFileSync(file, "utf8"));
+    }
+    catch {
+        return undefined;
+    }
+}
+function cleanTemporaries(dir) {
+    for (const name of readdirSync(dir))
+        if (/\.tmp-\d+$/.test(name))
+            rmSync(join(dir, name), { force: true });
+}
+export function staleFrom(meta, dir) {
+    const sealed = meta.coverage?.sealedUntilMs ?? 0;
+    let from = sealed < meta.untilMs || meta.coverage?.preparing
+        ? Math.floor(Math.max(sealed, meta.sinceMs) / HOUR_MS) - 1
+        : null;
+    for (const hour of meta.hours) {
+        if (from !== null && hour >= from)
+            break;
+        let live = false;
+        try {
+            live =
+                decompressHour(readFileSync(join(dir, `${hour}.zhb`))).header.source ===
+                    "live";
+        }
+        catch {
+            live = true;
+        }
+        if (live) {
+            from = hour;
+            break;
+        }
+    }
+    return from;
+}
+async function exportPart(http, out, s, sinceMs, untilMs, progress, refresh, onProgress, onWarning) {
+    const id = readId(s, sinceMs, untilMs);
+    const dir = join(out, "reads", id);
+    mkdirSync(dir, { recursive: true });
+    cleanTemporaries(dir);
+    const file = join(dir, "read.json");
+    const fresh = {
+        format: READ_FORMAT,
+        ...s,
+        sinceMs,
+        untilMs,
+        coverage: null,
+        afterHour: null,
+        done: false,
+        hours: [],
+        hashes: {},
+        updatedAt: new Date().toISOString(),
+    };
+    let meta = refresh ? fresh : (loadMeta(file) ?? fresh);
+    if (meta.done) {
+        const from = staleFrom(meta, dir);
+        if (from !== null) {
+            onWarning?.(`${id}: hours from ${new Date(from * HOUR_MS).toISOString()} were open or not sealed when exported; downloading them again`);
+            const hours = meta.hours.filter((h) => h < from);
+            meta = {
+                ...meta,
+                done: false,
+                afterHour: hours.length ? from - 1 : null,
+                hours,
+                hashes: Object.fromEntries(Object.entries(meta.hashes ?? {}).filter(([h]) => Number(h) < from)),
+            };
+        }
+    }
+    const save = () => writeAtomic(file, JSON.stringify(meta, null, 1));
+    save();
+    while (!meta.done) {
+        const query = {
+            since_ms: String(sinceMs),
+            until_ms: String(untilMs),
+            side: s.side,
+            allow_partial: "true",
+            with_signatures: String(s.withSignatures),
+            with_market: String(s.withMarket),
+            with_events: String(s.withEvents),
+            with_order: String(s.withOrder),
+            with_tx: String(!!s.withTx),
+        };
+        if (meta.afterHour !== null)
+            query.after_hour = String(meta.afterHour);
+        const answer = await http.raw(`/v1/functions/history/${s.scope}/${encodeURIComponent(s.key)}`, query);
+        const { meta: m, file: bytes } = parseFrame(answer.bytes);
+        const coverage = m.coverage ?? meta.coverage;
+        if (!m.batch) {
+            if (typeof m.resumeAfterHour === "number") {
+                meta = { ...meta, coverage, afterHour: m.resumeAfterHour };
+                save();
+                continue;
+            }
+            const sealed = coverage?.sealedUntilMs ?? 0;
+            meta = {
+                ...meta,
+                coverage,
+                done: true,
+                updatedAt: new Date().toISOString(),
+            };
+            save();
+            if (sealed < untilMs)
+                onWarning?.(`${id}: history after ${new Date(sealed).toISOString()} was not sealed yet; run zyrk export again later to get its final hours`);
+            onProgress?.({
+                read: id,
+                hour: null,
+                hours: meta.hours.length,
+                done: true,
+                bytes: progress.bytes,
+            });
+            break;
+        }
+        const { hour } = m.batch;
+        let header;
+        try {
+            header = decompressHour(bytes).header;
+        }
+        catch (e) {
+            throw new ExportError(`Hour ${hour}: ${e.message}`, "decode");
+        }
+        if (header.hour !== hour)
+            throw new ExportError(`Hour ${hour}: inconsistent file`, "decode");
+        if (meta.afterHour !== null && hour <= meta.afterHour)
+            throw new ExportError(`Hour ${hour} came twice`, "decode");
+        writeAtomic(join(dir, `${hour}.zhb`), bytes);
+        progress.bytes += bytes.length;
+        meta = {
+            ...meta,
+            coverage,
+            afterHour: hour,
+            hours: [...meta.hours, hour],
+            hashes: { ...meta.hashes, [String(hour)]: sha256(bytes) },
+            updatedAt: new Date().toISOString(),
+        };
+        save();
+        onProgress?.({
+            read: id,
+            hour,
+            hours: meta.hours.length,
+            done: false,
+            bytes: progress.bytes,
+        });
+    }
+    return meta;
+}
+export async function exportHistory(connection, out, read) {
+    const http = new Http(connection);
+    const s = readSpec(read);
+    const progress = { bytes: 0 };
+    const reads = [];
+    for (const [a, b] of parts(read.sinceMs, read.untilMs)) {
+        if (b - a > MAX_WINDOW_MS)
+            throw new ExportError("part too long", "bug");
+        reads.push(await exportPart(http, out, s, a, b, progress, !!read.refresh, connection.onProgress, connection.onWarning));
+    }
+    return { reads, requests: http.requests, bytes: progress.bytes };
+}
+export async function exportTotals(connection, out, mint, quote = null) {
+    const http = new Http(connection);
+    const body = await http.get(`/v1/tokens/${encodeURIComponent(mint)}/totals`, quote ? { quote } : {});
+    mkdirSync(join(out, "totals"), { recursive: true });
+    writeAtomic(join(out, "totals", totalsName(mint, quote)), JSON.stringify(body));
+    return body;
+}
+export async function exportVolumeSearch(connection, out, params) {
+    const http = new Http(connection);
+    mkdirSync(join(out, "volume"), { recursive: true });
+    let p = { ...params };
+    for (let pages = 1;; pages++) {
+        const flat = Object.entries(p)
+            .filter(([, v]) => v !== undefined && v !== null)
+            .map(([k, v]) => [k, String(v)]);
+        const page = await http.get("/v1/volume/search", Object.fromEntries(flat));
+        writeAtomic(join(out, "volume", `${volumeName(flat)}.json`), JSON.stringify({ params: flat, page }));
+        const next = page?.nextCursor;
+        if (next === null || next === undefined)
+            return pages;
+        if (typeof next !== "string" || next === p.cursor)
+            throw new ExportError("Invalid next cursor", "decode");
+        const w = page
+            .window;
+        if (w && typeof w === "object" && "since" in w)
+            p = { ...p, since: w.since, until: w.until };
+        p = { ...p, cursor: next };
+    }
+}

package/dist/globals.d.ts

@@ -0,0 +1,59 @@
+/**
+ * The only globals of a hosted program besides the ECMAScript built-ins.
+ * `zyrk build` type-checks against them; in an editor, add
+ * `/// <reference types="@johannc-dev/zyrk/globals" />` with `"types": []`
+ * and `"lib": ["ES2023"]` in the program's tsconfig.
+ *
+ * Absent on purpose: `fetch` and every network API, timers, Node APIs, DOM.
+ */
+export {};
+
+declare global {
+  /** Each call becomes a `ctx.log` entry of the execution. */
+  interface Console {
+    log(...values: unknown[]): void;
+    info(...values: unknown[]): void;
+    warn(...values: unknown[]): void;
+    error(...values: unknown[]): void;
+    debug(...values: unknown[]): void;
+  }
+  var console: Console;
+
+  interface TextEncoderEncodeIntoResult {
+    read: number;
+    written: number;
+  }
+  interface TextEncoder {
+    readonly encoding: "utf-8";
+    encode(input?: string): Uint8Array;
+    encodeInto(
+      input: string,
+      destination: Uint8Array,
+    ): TextEncoderEncodeIntoResult;
+  }
+  var TextEncoder: { prototype: TextEncoder; new (): TextEncoder };
+
+  interface TextDecoder {
+    readonly encoding: string;
+    readonly fatal: boolean;
+    readonly ignoreBOM: boolean;
+    decode(
+      input?: ArrayBufferView | ArrayBuffer,
+      options?: { stream?: boolean },
+    ): string;
+  }
+  var TextDecoder: {
+    prototype: TextDecoder;
+    new (
+      label?: string,
+      options?: { fatal?: boolean; ignoreBOM?: boolean },
+    ): TextDecoder;
+  };
+
+  function structuredClone<T>(value: T): T;
+
+  interface Crypto {
+    getRandomValues<T extends ArrayBufferView | null>(array: T): T;
+  }
+  var crypto: Crypto;
+}

package/dist/hourfile.d.ts

@@ -0,0 +1,57 @@
+import type { HourBatch } from "./sdk.js";
+export declare const HOUR_MAGIC = "ZHB1";
+export type ColumnType = "u8" | "u16" | "u32" | "i32" | "u64" | "f64";
+export interface HourHeader {
+    readonly hour: number;
+    readonly source: "sealed" | "live";
+    readonly complete: boolean;
+    readonly skipped: number;
+    readonly length: number;
+    readonly addresses: readonly string[];
+    readonly signatures?: readonly (string | null)[];
+    /** Exports: timestamp of each operation counted in `skipped` (`null`: unknown). */
+    readonly skippedTimes?: readonly (number | null)[];
+    readonly market: boolean;
+    readonly order: boolean;
+    /** Absent from files written before transaction ids. */
+    readonly tx?: boolean;
+    readonly columns: readonly (readonly [string, ColumnType, number])[];
+}
+/** Typed array constructors of the realm the batch is built for (a `vm` context, or Node's own). */
+export interface Realm {
+    readonly Uint8Array: Uint8ArrayConstructor;
+    readonly Uint16Array: Uint16ArrayConstructor;
+    readonly Uint32Array: Uint32ArrayConstructor;
+    readonly Int32Array: Int32ArrayConstructor;
+    readonly BigUint64Array: BigUint64ArrayConstructor;
+    readonly Float64Array: Float64ArrayConstructor;
+    readonly Array: ArrayConstructor;
+    readonly Object: ObjectConstructor;
+}
+export declare const nodeRealm: Realm;
+/** Column names and types, in the order of the sandbox's batch object. */
+export declare const COLUMNS: readonly (readonly [string, ColumnType])[];
+export declare const MARKET_COLUMNS: readonly (readonly [string, ColumnType])[];
+export declare const ORDER_COLUMNS: readonly (readonly [string, ColumnType])[];
+export declare const TX_COLUMNS: readonly (readonly [string, ColumnType])[];
+export declare class HourFileError extends Error {
+    readonly name = "HourFileError";
+}
+/** A decoded `ZHB1` buffer: its header and the raw bytes behind its columns. */
+export interface RawHour {
+    readonly header: HourHeader;
+    readonly bytes: Uint8Array;
+}
+export declare function decompressHour(file: Uint8Array): RawHour;
+/** Checks the header and every column, then the address indexes (see {@link checkIndexes}). */
+export declare function parseHour(bytes: Uint8Array): RawHour;
+/**
+ * The batch of a raw hour, every column in its own buffer (as in the
+ * sandbox), arrays of `realm`. Property order is the sandbox's.
+ */
+export declare function batchOf(raw: RawHour, realm?: Realm): HourBatch;
+/** The `ZHB1` buffer of a batch (what the server writes; used by tests and local re-cuts). */
+export declare function encodeHour(batch: HourBatch): Uint8Array;
+/** zstd with the frame checksum, as the server writes exported hours. */
+export declare function compress(raw: Uint8Array): Uint8Array;
+export declare function compressHour(batch: HourBatch): Uint8Array;