@johannc-dev/zyrk 0.0.0-stage → 0.2.1

package/dist/hourfile.js

@@ -0,0 +1,345 @@
+import { constants, zstdCompressSync, zstdDecompressSync } from "node:zlib";
+import { MarketFlag } from "./sdk.js";
+export const HOUR_MAGIC = "ZHB1";
+export const nodeRealm = {
+    Uint8Array,
+    Uint16Array,
+    Uint32Array,
+    Int32Array,
+    BigUint64Array,
+    Float64Array,
+    Array,
+    Object,
+};
+export const COLUMNS = [
+    ["line", "u32"],
+    ["op", "u16"],
+    ["slot", "f64"],
+    ["timeMs", "f64"],
+    ["receivedMs", "f64"],
+    ["side", "u8"],
+    ["protocol", "u8"],
+    ["mint", "u32"],
+    ["pool", "i32"],
+    ["trader", "i32"],
+    ["quoteMint", "u32"],
+    ["tokenRaw", "u64"],
+    ["tokenDecimals", "u8"],
+    ["quoteRaw", "u64"],
+    ["quoteDecimals", "u8"],
+    ["tokenAmount", "f64"],
+    ["quoteAmount", "f64"],
+    ["price", "f64"],
+    ["priceSol", "f64"],
+    ["priceUsd", "f64"],
+    ["quotePriceUsd", "f64"],
+    ["usdValue", "f64"],
+];
+export const MARKET_COLUMNS = [
+    ["tokenReserve", "u64"],
+    ["quoteReserve", "u64"],
+    ["virtualTokenReserve", "u64"],
+    ["virtualQuoteReserve", "u64"],
+    ["poolFeeBps", "f64"],
+    ["routeFeeBps", "f64"],
+    ["holderRewardsBps", "f64"],
+    ["creatorFeeRecipient", "i32"],
+    ["signer", "i32"],
+    ["priorityFeeRaw", "u64"],
+    ["flags", "u16"],
+    ["event", "u8"],
+];
+export const ORDER_COLUMNS = [
+    ["txIndex", "u32"],
+    ["eventIndex", "u16"],
+    ["blockTimeMs", "f64"],
+    ["flags", "u8"],
+];
+export const TX_COLUMNS = [
+    ["id", "u32"],
+    ["index", "u16"],
+    ["flags", "u8"],
+];
+const SIZE = {
+    u8: 1,
+    u16: 2,
+    u32: 4,
+    i32: 4,
+    u64: 8,
+    f64: 8,
+};
+function ctor(realm, type) {
+    switch (type) {
+        case "u8":
+            return realm.Uint8Array;
+        case "u16":
+            return realm.Uint16Array;
+        case "u32":
+            return realm.Uint32Array;
+        case "i32":
+            return realm.Int32Array;
+        case "u64":
+            return realm.BigUint64Array;
+        case "f64":
+            return realm.Float64Array;
+    }
+}
+export class HourFileError extends Error {
+    name = "HourFileError";
+}
+export function decompressHour(file) {
+    let bytes;
+    try {
+        bytes = new Uint8Array(zstdDecompressSync(file));
+    }
+    catch (e) {
+        throw new HourFileError(`Not a valid zstd frame: ${e.message}`);
+    }
+    return parseHour(bytes);
+}
+const isInt = (v, min = 0) => typeof v === "number" && Number.isSafeInteger(v) && v >= min;
+export function parseHour(bytes) {
+    if (bytes.length < 8)
+        throw new HourFileError(`Truncated exported hour (${bytes.length} bytes)`);
+    const magic = String.fromCharCode(...bytes.subarray(0, 4));
+    if (magic !== HOUR_MAGIC)
+        throw new HourFileError(`Not an exported hour (magic ${magic})`);
+    const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+    const n = view.getUint32(4, true);
+    if (8 + n > bytes.length)
+        throw new HourFileError("Truncated header of an exported hour");
+    let header;
+    try {
+        header = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes.subarray(8, 8 + n)));
+    }
+    catch (e) {
+        throw new HourFileError(`Unreadable header: ${e.message}`);
+    }
+    const h = header;
+    const fail = (what) => {
+        throw new HourFileError(`Invalid header: ${what}`);
+    };
+    if (!isInt(h.hour))
+        fail("hour");
+    if (h.source !== "sealed" && h.source !== "live")
+        fail("source");
+    if (typeof h.complete !== "boolean")
+        fail("complete");
+    if (!isInt(h.skipped))
+        fail("skipped");
+    if (!isInt(h.length))
+        fail("length");
+    if (typeof h.market !== "boolean" || typeof h.order !== "boolean")
+        fail("market/order");
+    if (h.tx !== undefined && typeof h.tx !== "boolean")
+        fail("tx");
+    if (!Array.isArray(h.addresses) ||
+        !h.addresses.every((a) => typeof a === "string"))
+        fail("addresses");
+    if (h.signatures !== undefined &&
+        (!Array.isArray(h.signatures) ||
+            h.signatures.length !== header.length ||
+            !h.signatures.every((x) => x === null || typeof x === "string")))
+        fail("signatures");
+    if (h.skippedTimes !== undefined &&
+        (!Array.isArray(h.skippedTimes) ||
+            h.skippedTimes.length !== header.skipped ||
+            !h.skippedTimes.every((x) => x === null || isInt(x))))
+        fail("skippedTimes");
+    if (!Array.isArray(h.columns))
+        fail("columns");
+    const expected = [
+        ...COLUMNS,
+        ...(header.market
+            ? MARKET_COLUMNS.map(([k, t]) => [`market.${k}`, t])
+            : []),
+        ...(header.order
+            ? ORDER_COLUMNS.map(([k, t]) => [`order.${k}`, t])
+            : []),
+        ...(header.tx ? TX_COLUMNS.map(([k, t]) => [`tx.${k}`, t]) : []),
+    ];
+    const at = new Map();
+    for (const c of header.columns) {
+        if (!Array.isArray(c) ||
+            typeof c[0] !== "string" ||
+            !(c[1] in SIZE) ||
+            !isInt(c[2]))
+            fail("column entry");
+        at.set(c[0], c);
+    }
+    for (const [name, type] of expected) {
+        const c = at.get(name);
+        if (!c || c[1] !== type)
+            fail(`column ${name}`);
+        const [, , offset] = c;
+        if (offset % 8 !== 0 ||
+            offset < 8 + n ||
+            offset + header.length * SIZE[type] > bytes.length)
+            throw new HourFileError(`Column ${name} out of the file`);
+    }
+    const raw = { header, bytes };
+    checkIndexes(raw);
+    return raw;
+}
+function checkIndexes(raw) {
+    const { header, bytes } = raw;
+    const n = header.addresses.length;
+    const at = new Map(header.columns.map((c) => [c[0], c]));
+    const check = (name, signed) => {
+        const c = at.get(name);
+        if (!c)
+            return;
+        const view = new DataView(bytes.buffer, bytes.byteOffset + c[2], header.length * 4);
+        for (let i = 0; i < header.length; i++) {
+            const v = signed
+                ? view.getInt32(i * 4, true)
+                : view.getUint32(i * 4, true);
+            if (v >= n || v < (signed ? -1 : 0))
+                throw new HourFileError(`${name}[${i}] = ${v}: not an index of addresses (${n})`);
+        }
+    };
+    check("mint", false);
+    check("quoteMint", false);
+    check("pool", true);
+    check("trader", true);
+    check("market.creatorFeeRecipient", true);
+    check("market.signer", true);
+}
+export function batchOf(raw, realm = nodeRealm) {
+    const { header, bytes } = raw;
+    const at = new Map(header.columns.map((c) => [c[0], c]));
+    const column = (name, type) => {
+        const c = at.get(name);
+        if (!c || c[1] !== type)
+            throw new HourFileError(`Missing column ${name}: ${type}`);
+        const n = header.length * SIZE[type];
+        const copy = new realm.Uint8Array(n);
+        copy.set(bytes.subarray(c[2], c[2] + n));
+        return new (ctor(realm, type))(copy.buffer, 0, header.length);
+    };
+    const strings = (list) => {
+        const out = new realm.Array(list.length);
+        for (let i = 0; i < list.length; i++)
+            out[i] = list[i] ?? null;
+        return out;
+    };
+    const batch = new realm.Object();
+    batch.hour = header.hour;
+    batch.source = header.source;
+    batch.complete = header.complete;
+    batch.skipped = header.skipped;
+    batch.length = header.length;
+    batch.addresses = strings(header.addresses);
+    for (const [name, type] of COLUMNS)
+        batch[name] = column(name, type);
+    if (header.signatures)
+        batch.signatures = strings(header.signatures);
+    if (header.market) {
+        const m = new realm.Object();
+        for (const [name, type] of MARKET_COLUMNS)
+            m[name] = column(`market.${name}`, type);
+        const flags = m.flags;
+        if (!header.virtualSides)
+            for (let i = 0; i < flags.length; i++)
+                if ((flags[i] & MarketFlag.virtualReserves) !== 0)
+                    flags[i] |=
+                        MarketFlag.virtualTokenReserve | MarketFlag.virtualQuoteReserve;
+        batch.market = m;
+    }
+    if (header.order) {
+        const o = new realm.Object();
+        for (const [name, type] of ORDER_COLUMNS)
+            o[name] = column(`order.${name}`, type);
+        batch.order = o;
+    }
+    if (header.tx) {
+        const t = new realm.Object();
+        for (const [name, type] of TX_COLUMNS)
+            t[name] = column(`tx.${name}`, type);
+        batch.tx = t;
+    }
+    return batch;
+}
+export function encodeHour(batch) {
+    const parts = [];
+    const add = (name, type, values) => parts.push({
+        name,
+        type,
+        data: new Uint8Array(values.buffer, values.byteOffset, batch.length * SIZE[type]),
+    });
+    const b = batch;
+    for (const [name, type] of COLUMNS)
+        add(name, type, b[name]);
+    const m = batch.market;
+    if (m)
+        for (const [name, type] of MARKET_COLUMNS)
+            add(`market.${name}`, type, m[name]);
+    const o = batch.order;
+    if (o)
+        for (const [name, type] of ORDER_COLUMNS)
+            add(`order.${name}`, type, o[name]);
+    const t = batch.tx;
+    if (t)
+        for (const [name, type] of TX_COLUMNS)
+            add(`tx.${name}`, type, t[name]);
+    const header = (base) => {
+        let offset = base;
+        const columns = parts.map((p) => {
+            offset = Math.ceil(offset / 8) * 8;
+            const c = [p.name, p.type, offset];
+            offset += p.data.length;
+            return c;
+        });
+        return JSON.stringify({
+            hour: batch.hour,
+            source: batch.source,
+            complete: batch.complete,
+            skipped: batch.skipped,
+            length: batch.length,
+            addresses: batch.addresses,
+            market: !!batch.market,
+            order: !!batch.order,
+            ...(batch.tx ? { tx: true } : {}),
+            columns,
+            ...(batch.signatures ? { signatures: batch.signatures } : {}),
+            ...(batch.skippedTimes
+                ? { skippedTimes: batch.skippedTimes }
+                : {}),
+        });
+    };
+    let base = 0;
+    let text = header(base);
+    for (;;) {
+        const start = Math.ceil((8 + Buffer.byteLength(text)) / 8) * 8;
+        if (start === base)
+            break;
+        base = start;
+        text = header(base);
+    }
+    const head = new TextEncoder().encode(text);
+    let size = base;
+    for (const p of parts)
+        size = Math.ceil(size / 8) * 8 + p.data.length;
+    const out = new Uint8Array(size);
+    out.set(new TextEncoder().encode(HOUR_MAGIC), 0);
+    new DataView(out.buffer).setUint32(4, head.length, true);
+    out.set(head, 8);
+    let offset = base;
+    for (const p of parts) {
+        offset = Math.ceil(offset / 8) * 8;
+        out.set(p.data, offset);
+        offset += p.data.length;
+    }
+    return out;
+}
+export function compress(raw) {
+    return new Uint8Array(zstdCompressSync(raw, {
+        params: {
+            [constants.ZSTD_c_compressionLevel]: 3,
+            [constants.ZSTD_c_checksumFlag]: 1,
+        },
+    }));
+}
+export function compressHour(batch) {
+    return compress(encodeHour(batch));
+}

package/dist/limits.d.ts

@@ -0,0 +1,91 @@
+/**
+ * Limits of hosted functions, mirrored from the server defaults. The server is the
+ * authority; these constants let tools and programs plan around them.
+ *
+ * No dependency: safe to import from guest code.
+ */
+/** Per execution, enforced by the sandbox. */
+export declare const execLimits: {
+    /**
+     * CPU time of the isolate thread (not instructions, not wall time), history
+     * reads excluded: they count in `readMs`.
+     */
+    readonly cpuMs: 10000;
+    /** Wall time, waits (`ctx.jobs.wait`, host operations) included. */
+    readonly wallMs: 60000;
+    readonly heapBytes: number;
+    /** JSON arguments of a direct call or a job. */
+    readonly inputBytes: number;
+    /**
+     * JSON arguments of an event delivery (live or replay): 32 real trade events
+     * are ~64 KB (~2 KB each). A replay `batchSize` multiplies it.
+     */
+    readonly eventInputBytes: number;
+    /** Result + logs + emits of one execution. */
+    readonly outputBytes: number;
+    /** History bytes served to one execution (columns, totals, volume pages). */
+    readonly readBytes: number;
+    /** Calls to `ctx.state`, `ctx.jobs`, `ctx.subscriptions`, `ctx.log`, `ctx.emit`. */
+    readonly hostOps: 10000;
+    /**
+     * CPU time spent by the service serving history reads (`ctx.data.*`:
+     * decompression, filtering) for one execution. Over it: `read_time_limit`.
+     */
+    readonly readMs: 30000;
+    /** User storage bytes read plus written by one execution (`io_limit`). */
+    readonly storageBytes: number;
+};
+/** User storage (`ctx.storage`), per account. Server defaults; operators may change them. */
+export declare const storageLimits: {
+    readonly shortQuotaBytes: number;
+    readonly shortTtlMs: number;
+    readonly longQuotaBytes: number;
+    readonly longTtlMs: number;
+    /** Never expires. */
+    readonly durableQuotaBytes: number;
+    /** Largest object (`too_large`). Reading one whole keeps it in memory (`heapBytes`). */
+    readonly maxObjectBytes: number;
+    /** Bytes per host operation: larger values take several. */
+    readonly chunkBytes: number;
+    readonly maxKeyBytes: 512;
+    readonly maxMetadataEntries: 16;
+    readonly maxMetadataBytes: 4096;
+    /** Items per `list` page. */
+    readonly maxList: 1000;
+    /** `writer()`s open at once in one execution. */
+    readonly maxUploads: 4;
+};
+/** Per account and per workflow, enforced by the orchestrator. */
+export declare const accountLimits: {
+    /** Executions of one account running at once (job grids). */
+    readonly parallelExecutions: 8;
+    readonly queuedExecutions: 1000;
+    /** Jobs started by one execution. */
+    readonly maxChildren: 32;
+    readonly maxDepth: 16;
+    readonly stateBytesPerProgram: number;
+    readonly stateValueBytes: number;
+    readonly maxPrograms: 32;
+    readonly storedCodeBytes: number;
+    readonly bundleBytes: number;
+    readonly maxSubscriptions: 64;
+    readonly maxPriceSubscriptions: 8;
+    readonly workflowCpuMs: 60000;
+    readonly workflowReadBytes: number;
+    /** History read time (`execLimits.readMs`) of a workflow, children included. */
+    readonly workflowReadMs: 180000;
+    readonly workflowAttempts: 256;
+    /** Results are kept this long, or until acknowledged. */
+    readonly resultTtlMs: number;
+    readonly retryAttempts: 3;
+    readonly retryBackoffMs: 250;
+};
+/** Bundle size accepted by `zyrk build` and by the server. */
+export declare const MAX_BUNDLE_BYTES: number;
+/** Longest history window of one read (`[sinceMs, untilMs)`), as the HTTP API. */
+export declare const MAX_WINDOW_MS: number;
+/** Subscription delivery: events per batch and replay cap. */
+export declare const subscriptionLimits: {
+    readonly eventsPerDelivery: 32;
+    readonly maxReplayEvents: 1000;
+};

package/dist/limits.js

@@ -0,0 +1,54 @@
+const KIB = 1024;
+const MIB = 1024 * KIB;
+export const execLimits = {
+    cpuMs: 10_000,
+    wallMs: 60_000,
+    heapBytes: 256 * MIB,
+    inputBytes: 32 * KIB,
+    eventInputBytes: 128 * KIB,
+    outputBytes: 64 * KIB,
+    readBytes: 256 * MIB,
+    hostOps: 10_000,
+    readMs: 30_000,
+    storageBytes: 1024 * MIB,
+};
+export const storageLimits = {
+    shortQuotaBytes: 1024 * MIB,
+    shortTtlMs: 24 * 3_600_000,
+    longQuotaBytes: 500 * MIB,
+    longTtlMs: 7 * 24 * 3_600_000,
+    durableQuotaBytes: 50 * MIB,
+    maxObjectBytes: 64 * MIB,
+    chunkBytes: 4 * MIB,
+    maxKeyBytes: 512,
+    maxMetadataEntries: 16,
+    maxMetadataBytes: 4096,
+    maxList: 1000,
+    maxUploads: 4,
+};
+export const accountLimits = {
+    parallelExecutions: 8,
+    queuedExecutions: 1000,
+    maxChildren: 32,
+    maxDepth: 16,
+    stateBytesPerProgram: 32 * MIB,
+    stateValueBytes: MIB,
+    maxPrograms: 32,
+    storedCodeBytes: 32 * MIB,
+    bundleBytes: MIB,
+    maxSubscriptions: 64,
+    maxPriceSubscriptions: 8,
+    workflowCpuMs: 60_000,
+    workflowReadBytes: 1024 * MIB,
+    workflowReadMs: 180_000,
+    workflowAttempts: 256,
+    resultTtlMs: 15 * 60_000,
+    retryAttempts: 3,
+    retryBackoffMs: 250,
+};
+export const MAX_BUNDLE_BYTES = accountLimits.bundleBytes;
+export const MAX_WINDOW_MS = 31 * 24 * 3_600_000;
+export const subscriptionLimits = {
+    eventsPerDelivery: 32,
+    maxReplayEvents: 1000,
+};

package/dist/local-data.d.ts

@@ -0,0 +1,115 @@
+import { type HourHeader, type Realm } from "./hourfile.js";
+import type { HourBatch, Json } from "./sdk.js";
+export declare const HOUR_MS = 3600000;
+export declare const READ_FORMAT = "zyrk-read/1";
+/** What one exported read selects: the options that shape a batch. */
+export interface ReadSpec {
+    readonly scope: "token" | "wallet";
+    readonly key: string;
+    readonly side: "any" | "buy" | "sell";
+    readonly withSignatures: boolean;
+    /** `withMarket || withEvents`, as the sandbox does. */
+    readonly withMarket: boolean;
+    readonly withEvents: boolean;
+    readonly withOrder: boolean;
+    /** Absent from exports written before transaction ids (false). */
+    readonly withTx?: boolean;
+}
+/** `coverage` of the export window, as the server computed it. */
+export interface Coverage {
+    readonly complete?: boolean;
+    readonly preparing?: boolean;
+    readonly retentionHorizonMs?: number | null;
+    readonly sealedUntilMs?: number | null;
+    readonly liveUntilMs?: number | null;
+    readonly gaps?: readonly {
+        readonly fromMs?: number | null;
+        readonly toMs?: number | null;
+    }[];
+    readonly [field: string]: unknown;
+}
+export interface ReadMeta extends ReadSpec {
+    readonly format: typeof READ_FORMAT;
+    readonly sinceMs: number;
+    readonly untilMs: number;
+    readonly coverage: Coverage | null;
+    /** Last hour received (`after_hour` of the next request); `null` before the first. */
+    readonly afterHour: number | null;
+    readonly done: boolean;
+    /** Hours with a file, in order. */
+    readonly hours: readonly number[];
+    /** SHA-256 (hex) of each hour file, checked before it is read. */
+    readonly hashes?: Readonly<Record<string, string>>;
+    readonly updatedAt: string;
+}
+export declare function spec(input: {
+    scope: "token" | "wallet";
+    key: string;
+    side?: "any" | "buy" | "sell" | undefined;
+    withSignatures?: boolean | undefined;
+    withMarket?: boolean | undefined;
+    withEvents?: boolean | undefined;
+    withOrder?: boolean | undefined;
+    withTx?: boolean | undefined;
+}): ReadSpec;
+/** Directory name of an exported read. */
+export declare function readId(s: ReadSpec, sinceMs: number, untilMs: number): string;
+/** The `zyrk export` command that would produce a read. */
+export declare function exportCommand(s: ReadSpec, sinceMs: number, untilMs: number, refresh?: boolean): string;
+/** An error a read throws in the program: `code` as in production. */
+export interface ReadFailure {
+    readonly code: string;
+    readonly message: string;
+    readonly retryAfterMs?: number;
+}
+export declare class ReadFailed extends Error {
+    readonly failure: ReadFailure;
+    constructor(failure: ReadFailure);
+}
+export declare const sha256: (bytes: Uint8Array) => string;
+export declare function windowComplete(c: Coverage, since: number, until: number): boolean;
+export declare function hourComplete(c: Coverage, hour: number, since: number, until: number): boolean;
+/** What a batch costs against `readBytes`, as the service counts it. */
+export declare function batchBytes(h: HourHeader): number;
+export interface HourRead {
+    readonly batch: HourBatch;
+    readonly bytes: number;
+    /** The hour was re-cut to a narrower window than the export's. */
+    readonly recut: boolean;
+    /**
+     * The hour may have changed since the export: it was open (`live`) or not
+     * sealed yet. `zyrk export` run again downloads such hours again.
+     */
+    readonly stale: boolean;
+}
+export declare class LocalData {
+    #private;
+    readonly dir: string;
+    constructor(dir: string);
+    /** Every exported read (re-scanned after {@link refresh}). */
+    reads(): readonly ReadMeta[];
+    refresh(): void;
+    /** The narrowest export with these options whose window holds `[since, until)`. */
+    find(s: ReadSpec, since: number, until: number): ReadMeta | undefined;
+    /**
+     * The next batch of a read after `afterHour` (`null`: none left), as
+     * the service's reader answers it. Throws {@link ReadFailed}.
+     */
+    readHour(s: ReadSpec, since: number, until: number, allowPartial: boolean, afterHour: number | null, realm?: Realm): HourRead | null;
+    /** Totals of a mint saved by the export, or `undefined`. */
+    totals(mint: string, quote: string | null): Json | undefined;
+    /** Info of a mint saved by the export (`--totals`), or `undefined`. */
+    info(mint: string): Json | undefined;
+    /** The error `tokenTotals` answered at export, or `undefined`. */
+    totalsFailure(mint: string, quote: string | null): ReadFailure | undefined;
+    /** The error `tokenInfo` answered at export, or `undefined`. */
+    infoFailure(mint: string): ReadFailure | undefined;
+    /** The error a volume search page answered at export, or `undefined`. */
+    volumeFailure(params: readonly (readonly [string, string])[]): ReadFailure | undefined;
+    /** A volume search page saved by the export for exactly these parameters. */
+    volumePage(params: readonly (readonly [string, string])[]): Json | undefined;
+}
+export declare const totalsName: (mint: string, quote: string | null) => string;
+export declare const infoName: (mint: string) => string;
+/** Parameters as the sandbox passes them (`[key, string value]`), sorted. */
+export declare function volumeName(params: readonly (readonly [string, string])[]): string;