@johannc-dev/zyrk 0.2.0 → 0.2.2

package/dist/local.js

@@ -6,10 +6,35 @@ import { getHeapStatistics } from "node:v8";
 import vm from "node:vm";
 import { buildProgram } from "./build.js";
 import { DELIVERY_BATCH, REPLAY_MAX_BATCH, MISSING_OPERATION_FIELDS, intervalEvents, replayEvents, } from "./local-replay.js";
-import { LocalData, ReadFailed, spec as readSpec } from "./local-data.js";
+import { LocalData, canonical, ReadFailed, spec as readSpec, } from "./local-data.js";
 import { LocalStorage, StorageOpError, } from "./local-storage.js";
 import { accountLimits, execLimits, subscriptionLimits } from "./limits.js";
 const here = dirname(fileURLToPath(import.meta.url));
+let batchReadsSaid = false;
+function batchReadsFromEnv() {
+    const raw = process.env.FUNRUN_BATCH_READS;
+    const v = (raw ?? "").trim();
+    const on = v === "on" || v === "1" || v === "true";
+    if (raw !== undefined && !batchReadsSaid) {
+        batchReadsSaid = true;
+        process.stderr.write(`zyrk local runner: FUNRUN_BATCH_READS=${JSON.stringify(raw)}: batched history reads ${on ? "on" : "off"} (hosted functions batch only when the service has the same setting; off there by default)\n`);
+    }
+    return on;
+}
+const COMPUTE_OUTPUT_BYTES = 64 * 2 ** 20;
+const computeLine = (r, row) => {
+    const parts = [...r.keys, ...r.measures].map((n) => {
+        const c = r.columns[n];
+        const v = c.values[row];
+        const value = c.type === "f64"
+            ? typeof v === "number" && Number.isFinite(v)
+                ? v
+                : null
+            : v;
+        return `${JSON.stringify(n)}:${JSON.stringify(value)}`;
+    });
+    return `{${parts.join(",")}}\n`;
+};
 let bootstrapText;
 export function sandboxBootstrap() {
     return (bootstrapText ??= readFileSync(join(here, "sandbox-bootstrap.js"), "utf8"));
@@ -60,6 +85,13 @@ const WEBHOOK_PAYLOAD_BYTES = 16 * 1024;
 const WEBHOOK_OVERHEAD_BYTES = 256;
 const WEBHOOK_QUEUE_LIMIT = 1000;
 const MAX_MESSAGE_BYTES = 2048;
+const openWhenExported = (v) => {
+    const c = v?.coverage;
+    return (typeof c === "object" &&
+        c !== null &&
+        (c.unsealed === true ||
+            (c.liveUntilMs !== null && c.liveUntilMs !== undefined)));
+};
 const jsonBytes = (v) => Buffer.byteLength(JSON.stringify(v) ?? "null");
 const truncate = (m) => Buffer.byteLength(m) <= MAX_MESSAGE_BYTES
     ? m
@@ -212,6 +244,7 @@ export class LocalRunner {
             pending: 0,
             io: { bytes: 0 },
             startedAt: 0,
+            readWaitMs: 0,
             done,
             finish,
         };
@@ -352,7 +385,7 @@ export class LocalRunner {
             if (e.usage.outputBytes + size > this.limits.exec.outputBytes * e.scale)
                 throw limit("output_limit", `Result, logs and emits went over ${this.limits.exec.outputBytes * e.scale} bytes`);
             e.usage.outputBytes += size;
-            if (performance.now() - e.startedAt > this.limits.exec.wallMs)
+            if (this.#wallUsed(e) > this.limits.exec.wallMs)
                 throw limit("wall_limit", `The execution ran over ${this.limits.exec.wallMs} ms (measured locally, not preempted)`);
             if (e.cancelled)
                 throw new HostError("permission", "cancelled", "Its parent failed: the job was cancelled");
@@ -428,6 +461,38 @@ export class LocalRunner {
         };
         const fromRead = (f) => new HostError(kindOfSdkCode(f.code), f.code, f.message, f.retryAfterMs);
         const guestJson = (v) => realm.JSON.parse(JSON.stringify(v) ?? "null");
+        const computeOutputs = new Map();
+        let computeNext = 0;
+        const computeSummary = (r) => ({
+            length: r.length,
+            keys: r.keys,
+            measures: r.measures,
+            coverage: r.coverage,
+            complete: r.complete,
+            skipped: r.skipped,
+            stats: r.stats,
+        });
+        const computeObject = (r) => {
+            const o = guestJson(computeSummary(r));
+            const columns = new realm.Object();
+            for (const n of [...r.keys, ...r.measures]) {
+                const c = r.columns[n];
+                columns[n] =
+                    c.type === "f64"
+                        ? realm.Float64Array.from(c.values.map((v) => v === "NaN"
+                            ? NaN
+                            : v === "Infinity"
+                                ? Infinity
+                                : v === "-Infinity"
+                                    ? -Infinity
+                                    : Number(v)))
+                        : c.type === "int"
+                            ? realm.Array.from(c.values.map((v) => (v === null ? null : BigInt(String(v)))))
+                            : realm.Array.from(c.values);
+            }
+            o.columns = columns;
+            return o;
+        };
         const ms = (v) => typeof v === "number" && Number.isFinite(v) && Math.abs(v) < 9e15
             ? Math.trunc(v)
             : undefined;
@@ -454,13 +519,43 @@ export class LocalRunner {
                 wf.readBytes > this.limits.account.workflowReadBytes)
                 throw readLimit();
         };
+        const waited = (t0) => {
+            e.readWaitMs += Math.min(performance.now() - t0, this.limits.exec.readCallWaitMs);
+        };
         const data = () => {
             if (!this.data)
                 throw sdkError(program("local_data_missing", "No data directory: export history with zyrk export, then pass --data (or { data })."));
             return this.data;
         };
+        const saved = (answer, failure, missing) => {
+            let value;
+            let failed;
+            const t0 = performance.now();
+            try {
+                value = answer();
+                failed = value === undefined ? failure() : undefined;
+            }
+            catch (error) {
+                if (error instanceof ReadFailed)
+                    throw sdkError(fromRead(error.failure));
+                throw error;
+            }
+            finally {
+                waited(t0);
+            }
+            if (failed)
+                throw sdkError(fromRead(failed));
+            if (value === undefined)
+                throw sdkError(program("local_data_missing", missing));
+            if (openWhenExported(value))
+                this.#warn("Totals or volume search pages were exported while an hour was still open: the service may now answer otherwise. Run zyrk export again (--totals, --volume-search) to download them again.");
+            charge(jsonBytes(value));
+            return guestJson(value);
+        };
         return {
             maxDepth: this.limits.account.maxDepth,
+            batchReads: batchReadsFromEnv(),
+            hint: () => undefined,
             fail: (code, message) => sdkError(new HostError(kindOfSdkCode(String(code)), String(code), String(message))),
             op: (text, awaited) => this.#op(e, text, !!awaited, realm, mode, sdkError, guestJson, sample),
             read: (kind, key, since, until, side, allowPartial, withSignatures, after, withMarket, withEvents, withOrder, withTx) => {
@@ -500,12 +595,13 @@ export class LocalRunner {
                 }
                 finally {
                     e.usage.readMs += performance.now() - t0;
+                    waited(t0);
                     sample();
                 }
                 if (hour === null)
                     return null;
                 if (hour.stale)
-                    this.#warn("Some hours were open or not sealed when they were exported: the service may now answer otherwise. Run zyrk export again to download them again.");
+                    this.#warn("Some hours were open or not sealed when they were exported, or were exported by an older version: the service may now answer otherwise. Run zyrk export again to download them again.");
                 if (hour.recut)
                     this.#warn("A read used a narrower window than its export: its edge hours were cut locally. Values are the service's; the address indexes and market.flags (chainBreak, chainUnchecked) of a cut hour may differ. Export the exact window for byte-identical batches.");
                 charge(hour.bytes);
@@ -517,47 +613,77 @@ export class LocalRunner {
                 if (typeof mint !== "string" ||
                     (quote !== null && typeof quote !== "string"))
                     throw sdkError(program("invalid_argument", "Expected a mint address"));
-                const value = data().totals(mint, quote);
-                if (value === undefined)
-                    throw sdkError(program("local_data_missing", `No exported totals for ${mint}: zyrk export --totals ${mint}${quote ? ` --quote ${quote}` : ""}`));
-                charge(jsonBytes(value));
-                return guestJson(value);
+                return saved(() => data().totals(mint, quote), () => data().totalsFailure(mint, quote), `No exported totals for ${mint}: zyrk export --totals ${mint}${quote ? ` --quote ${quote}` : ""}`);
             },
             info: (mint) => {
                 readAllowed();
                 e.usage.readCalls++;
                 if (typeof mint !== "string")
                     throw sdkError(program("invalid_argument", "Expected a mint address"));
-                const t = data().totals(mint, null);
-                if (t === undefined)
-                    throw sdkError(program("local_data_missing", `No exported totals for ${mint}: zyrk export --totals ${mint}`));
-                const creation = (t.creation ?? null);
-                const metadata = (t.metadata ?? null);
-                const created = creation !== null &&
-                    typeof creation === "object" &&
-                    !Array.isArray(creation) &&
-                    typeof creation.timestamp === "number"
-                    ? creation.timestamp
-                    : (t.createdMs ?? null);
-                const value = {
-                    mint,
-                    known: creation !== null || metadata !== null || created !== null,
-                    createdMs: created,
-                    creation,
-                    metadata,
-                };
-                charge(jsonBytes(value));
-                return guestJson(value);
+                return saved(() => data().info(mint), () => data().infoFailure(mint), `No exported info for ${mint} (exports by older versions saved only its totals): zyrk export --totals ${mint}`);
+            },
+            computeAsk: () => realm.Promise.resolve(),
+            compute: (text, toStorage) => {
+                readAllowed();
+                e.usage.readCalls++;
+                let plan;
+                try {
+                    plan = JSON.parse(String(text));
+                }
+                catch {
+                    throw sdkError(program("invalid_plan", "The plan is not JSON"));
+                }
+                const answer = saved(() => data().computeResult(plan), () => data().computeFailure(plan), `No exported result for this plan: zyrk export --compute '${canonical(plan)}'`);
+                if (toStorage === true) {
+                    if (computeOutputs.size > 0)
+                        throw sdkError(program("invalid_argument", "One ctx.compute output at a time per execution: await the one being written first"));
+                    if (Array.isArray(answer))
+                        throw sdkError(program("invalid_plan", "output writes the result of one plan"));
+                    const result = answer;
+                    let bytes = 0;
+                    for (let row = 0; row < result.length; row++)
+                        bytes += Buffer.byteLength(computeLine(result, row), "utf8");
+                    if (bytes > COMPUTE_OUTPUT_BYTES)
+                        throw sdkError(limit("compute_result_limit", `the result is ${Math.ceil(bytes / 2 ** 20)} MiB of JSON lines (${result.length} rows), more than the ${COMPUTE_OUTPUT_BYTES / 2 ** 20} MiB of one storage object: add a limit, a top or a having, group less finely, or split the range into several calls`));
+                    const handle = computeNext++;
+                    computeOutputs.set(handle, {
+                        result: answer,
+                        row: 0,
+                    });
+                    return guestJson({
+                        handle,
+                        summary: computeSummary(answer),
+                    });
+                }
+                return Array.isArray(answer)
+                    ? realm.Array.from(answer.map((a) => computeObject(a)))
+                    : computeObject(answer);
+            },
+            computeChunk: (handle) => {
+                const o = computeOutputs.get(Number(handle));
+                if (!o)
+                    throw sdkError(program("invalid_argument", "Unknown compute output"));
+                const r = o.result;
+                if (o.row >= r.length)
+                    return null;
+                let text = "";
+                while (o.row < r.length && text.length < 4 * 1024 * 1024 - 64 * 1024) {
+                    text += computeLine(r, o.row);
+                    o.row++;
+                }
+                const bytes = Buffer.from(text, "utf8");
+                const out = new realm.Uint8Array(bytes.length);
+                out.set(bytes);
+                return out;
+            },
+            computeDrop: (handle) => {
+                computeOutputs.delete(Number(handle));
             },
             volume: (text) => {
                 readAllowed();
                 e.usage.readCalls++;
                 const params = JSON.parse(String(text));
-                const page = data().volumePage(params);
-                if (page === undefined)
-                    throw sdkError(program("local_data_missing", `No exported volume page for these parameters: zyrk export --volume-search '${JSON.stringify(Object.fromEntries(params.filter(([k]) => k !== "cursor")))}'`));
-                charge(jsonBytes(page));
-                return guestJson(page);
+                return saved(() => data().volumePage(params), () => data().volumeFailure(params), `No exported volume page for these parameters: zyrk export --volume-search '${JSON.stringify(Object.fromEntries(params.filter(([k]) => k !== "cursor")))}'`);
             },
             encode: (text) => {
                 const bytes = Buffer.from(String(text), "utf8");
@@ -588,8 +714,13 @@ export class LocalRunner {
             },
         };
     }
+    #wallUsed(e) {
+        return (performance.now() -
+            e.startedAt -
+            Math.min(e.readWaitMs, this.limits.exec.readWaitMs));
+    }
     #checkWall(e) {
-        if (performance.now() - e.startedAt > this.limits.exec.wallMs) {
+        if (this.#wallUsed(e) > this.limits.exec.wallMs) {
             e.fatal ??= limit("wall_limit", `The execution ran over ${this.limits.exec.wallMs} ms`);
             throw e.fatal;
         }
@@ -781,7 +912,7 @@ export class LocalRunner {
                 });
                 if (this.#waiting >= account.parallelExecutions - 1)
                     throw limit("job_wait_limit", `At most ${account.parallelExecutions - 1} executions wait at once`);
-                const left = this.limits.exec.wallMs - (performance.now() - e.startedAt);
+                const left = this.limits.exec.wallMs - this.#wallUsed(e);
                 const timeout = Math.max(0, Math.min(typeof a.timeout_ms === "number" ? a.timeout_ms : left, left));
                 this.#waiting++;
                 let timer;

package/dist/sandbox-bootstrap.js

@@ -299,25 +299,25 @@
   const summary = (r) => freeze({ key: r.key, size: r.size, contentType: r.contentType, lastModified: r.lastModified, expiresAt: r.expiresAt });
   const storageArea = (area) => {
     const read = async (key, offset, length) => {
-      const first = await call({ StorageGet: { area, key, offset, length: length > CHUNK ? CHUNK : length } });
-      if (first === null) return null;
-      const end = Math.min(first.size, offset + length);
-      const total = Math.max(0, end - first.offset);
+      const first2 = await call({ StorageGet: { area, key, offset, length: length > CHUNK ? CHUNK : length } });
+      if (first2 === null) return null;
+      const end = Math.min(first2.size, offset + length);
+      const total = Math.max(0, end - first2.offset);
       const out = new U8(total);
       let at = 0;
-      let r = first;
+      let r = first2;
       for (; ; ) {
         const n = N.b64decodeInto(r.data, out, at);
         if (n < 0) throw fail("unavailable", "Corrupt storage chunk");
         at += n;
         if (at >= total || n === 0) break;
         r = await call({
-          StorageGet: { area, key, offset: first.offset + at, length: Math.min(CHUNK, total - at) }
+          StorageGet: { area, key, offset: first2.offset + at, length: Math.min(CHUNK, total - at) }
         });
-        if (r === null || r.etag !== first.etag)
+        if (r === null || r.etag !== first2.etag)
           throw fail("value_changed", "The object changed while it was read: read it again");
       }
-      return { first, body: at === total ? out : out.subarray(0, at) };
+      return { first: first2, body: at === total ? out : out.subarray(0, at) };
     };
     const get = async (key, range) => {
       keyOf(key);
@@ -427,6 +427,65 @@
     long: storageArea("long"),
     durable: storageArea("durable")
   });
+  const computeAreas = ["short", "long", "durable"];
+  let outputTurn = PromiseCtor.resolve();
+  const compute = async (plan, options) => {
+    if (plan === null || typeof plan !== "object")
+      throw fail("invalid_plan", "Expected a plan object (or an array of plans)");
+    let output;
+    if (options !== void 0 && options !== null) {
+      if (typeOf(options) !== "object") throw fail("invalid_argument", "Expected options { output }");
+      for (const k of keys(options))
+        if (k !== "output") throw fail("invalid_argument", "Unknown compute option: " + k);
+      output = options.output;
+      if (output !== void 0 && (typeOf(output) !== "object" || !computeAreas.includes(output.area) || typeof output.key !== "string"))
+        throw fail("invalid_argument", 'output is { area: "short" | "long" | "durable", key }');
+    }
+    let text;
+    try {
+      text = stringify(plan);
+    } catch {
+      throw fail("invalid_plan", "The plan is not JSON (write integers beyond 2^53 as decimal strings)");
+    }
+    await N.computeAsk(text);
+    if (output === void 0) return N.compute(text, false);
+    const before = outputTurn;
+    let release;
+    outputTurn = new PromiseCtor((resolve) => {
+      release = resolve;
+    });
+    try {
+      await before;
+      return await computeOutput(text, output);
+    } finally {
+      release();
+    }
+  };
+  const computeOutput = async (text, output) => {
+    const r = N.compute(text, true);
+    let w;
+    try {
+      w = await storage[output.area].writer(output.key, { contentType: "application/x-ndjson" });
+      let bytes = 0;
+      for (; ; ) {
+        const chunk = N.computeChunk(r.handle);
+        if (chunk === null) break;
+        bytes += chunk.length;
+        await w.write(chunk);
+      }
+      const head2 = await w.close();
+      return freeze({
+        ...r.summary,
+        output: freeze({ area: output.area, key: output.key, rows: r.summary.length, bytes, etag: head2.etag })
+      });
+    } catch (e) {
+      if (w) await w.abort().catch(() => {
+      });
+      throw e;
+    } finally {
+      N.computeDrop(r.handle);
+    }
+  };
   const start = (functionName, argsText, executionText) => {
     const p = program();
     let depth = 0;
@@ -449,7 +508,7 @@
         version: execution.version,
         attempt: execution.attempt
       }),
-      data: freeze({
+      data: batching ? batchedData() : freeze({
         walletTrades: trades("wallet"),
         tokenTrades: trades("token"),
         tokenTotals: async (mint, o) => {
@@ -487,6 +546,7 @@
         continue: (args) => call({ Continue: { args } })
       }),
       storage,
+      compute,
       subscriptions: freeze({
         set: (name, subscription) => call({ Subscribe: { name, subscription } }),
         remove: (name) => call({ Unsubscribe: { name } }),
@@ -515,5 +575,167 @@
       return text === void 0 ? "null" : text;
     })();
   };
+  const batching = N.batchReads === true;
+  const BATCH = 64;
+  const MAX_REENTRY = 1e3;
+  const reflectApply = Reflect.apply;
+  const promiseThen = PromiseCtor.prototype.then;
+  const ready = PromiseCtor.resolve();
+  Object.defineProperty(ready, "constructor", { value: void 0 });
+  let open = false;
+  let direct = false;
+  let reentry = 0;
+  let first = null;
+  let last = null;
+  let epoch = 0;
+  const wait = (run, hint) => new PromiseCtor((resolve, reject) => {
+    const q = { run, hint, next: null, resolve, reject };
+    if (last === null) first = q;
+    else last.next = q;
+    last = q;
+  });
+  const checkpoint = () => {
+    let q = first;
+    first = null;
+    last = null;
+    direct = reentry >= MAX_REENTRY;
+    while (q !== null) {
+      epoch++;
+      let n = 0;
+      let hinted = 0;
+      for (let r = q; r !== null && n < BATCH; r = r.next) {
+        n++;
+        if (r.hint !== null && r.hint(epoch)) hinted++;
+      }
+      if (hinted > 0) N.hint();
+      for (; n > 0; n--) {
+        const c = q;
+        q = c.next;
+        let value;
+        try {
+          value = c.run();
+        } catch (e) {
+          c.reject(e);
+          continue;
+        }
+        c.resolve(value);
+      }
+    }
+    direct = false;
+    if (first === null) {
+      reentry = 0;
+      open = false;
+    } else {
+      reentry++;
+      reflectApply(promiseThen, ready, [checkpoint]);
+    }
+  };
+  const openBatch = () => {
+    open = true;
+    reflectApply(promiseThen, ready, [checkpoint]);
+  };
+  const waiting = () => open && !direct;
+  const tradesBatched = (kind) => (key, w) => {
+    if (typeof key !== "string") throw fail("invalid_argument", "Expected an address string");
+    const { since, until, side, allowPartial, withSignatures, withMarket, withEvents, withOrder, withTx } = checkWindow(w);
+    return freeze({
+      [asyncIterator]() {
+        let after = null;
+        let done = false;
+        let closed = false;
+        let mark = 0;
+        const end = () => ({ done: true, value: void 0 });
+        const read = () => {
+          if (done) return end();
+          const batch = N.read(kind, key, since, until, side, allowPartial, withSignatures, after, withMarket, withEvents, withOrder, withTx);
+          if (batch === null) {
+            done = true;
+            return end();
+          }
+          after = batch.hour;
+          return { done: false, value: batch };
+        };
+        const hint = (e) => {
+          if (done || mark === e) return false;
+          mark = e;
+          N.hint(kind, key, since, until, side, allowPartial, withSignatures, after, withMarket, withEvents, withOrder, withTx);
+          return true;
+        };
+        const nextNow = async () => {
+          if (done || closed) return end();
+          if (!direct) openBatch();
+          return read();
+        };
+        const returnNow = async (value) => {
+          closed = true;
+          return { done: true, value };
+        };
+        return {
+          next() {
+            if (!waiting()) return nextNow();
+            if (closed) return wait(end, null);
+            return wait(read, hint);
+          },
+          return(value) {
+            if (!waiting()) return returnNow(value);
+            closed = true;
+            return wait(() => ({ done: true, value }), null);
+          },
+          [asyncIterator]() {
+            return this;
+          }
+        };
+      }
+    });
+  };
+  const totalsNow = async (mint, o) => {
+    if (typeof mint !== "string") throw fail("invalid_argument", "Expected a mint address");
+    const rawQuote = o ? o.quote : void 0;
+    const quote = rawQuote !== void 0 ? StringCtor(rawQuote) : null;
+    return N.totals(mint, quote);
+  };
+  const infoNow = async (mint) => {
+    if (typeof mint !== "string") throw fail("invalid_argument", "Expected a mint address");
+    return N.info(mint);
+  };
+  const batchedData = () => freeze({
+    walletTrades: tradesBatched("wallet"),
+    tokenTrades: tradesBatched("token"),
+    tokenTotals: (mint, o) => {
+      if (!waiting()) return totalsNow(mint, o);
+      let quote;
+      try {
+        if (typeof mint !== "string") throw fail("invalid_argument", "Expected a mint address");
+        const rawQuote = o ? o.quote : void 0;
+        quote = rawQuote !== void 0 ? StringCtor(rawQuote) : null;
+      } catch (e) {
+        return PromiseCtor.reject(e);
+      }
+      return wait(() => N.totals(mint, quote), null);
+    },
+    tokenInfo: (mint) => {
+      if (!waiting()) return infoNow(mint);
+      if (typeof mint !== "string") return PromiseCtor.reject(fail("invalid_argument", "Expected a mint address"));
+      return wait(() => N.info(mint), null);
+    },
+    volumeSearch: async function* (params) {
+      let p = { ...params };
+      for (; ; ) {
+        let page;
+        if (waiting()) {
+          const text = flatParams(p);
+          page = await wait(() => N.volume(text), null);
+        } else page = N.volume(flatParams(p));
+        yield page;
+        const next = page === null || typeof page !== "object" ? null : page.nextCursor;
+        if (next === null || next === void 0) return;
+        if (typeof next !== "string" || next === p.cursor)
+          throw fail("history_unavailable", "Invalid next cursor");
+        const w = page.window;
+        if (w && typeof w === "object" && "since" in w) p = { ...p, since: w.since, until: w.until };
+        p = { ...p, cursor: next };
+      }
+    }
+  });
   return freeze({ start, describe });
 });