@opendatalabs/personal-server-ts-server 1.16.0 → 1.17.0

package/dist/jobs/raw-envelope-stream.js

@@ -0,0 +1,371 @@
+import { Readable, Transform } from "node:stream";
+import { DataFileEnvelopeSchema } from "@opendatalabs/vana-sdk/browser";
+const decoder = new TextDecoder();
+const encoder = new TextEncoder();
+const QUOTE = 0x22;
+const BACKSLASH = 0x5c;
+const MAX_METADATA_TOKEN_BYTES = 64 * 1024;
+const OUTPUT_CHUNK_BYTES = 64 * 1024;
+/** Open twice: once for bounded metadata inspection, then for streamed output. */
+export async function openRedactedEnvelopeStream(open) {
+    const plan = await scanEnvelope(await open());
+    const input = Readable.fromWeb((await open()));
+    const redactor = new EnvelopeRedactor(plan);
+    input.on("error", (error) => redactor.destroy(error));
+    redactor.on("close", () => input.destroy());
+    input.pipe(redactor);
+    return Readable.toWeb(redactor);
+}
+async function scanEnvelope(stream) {
+    const scanner = new EnvelopeScanner();
+    for await (const chunk of Readable.fromWeb(stream)) {
+        scanner.write(chunk);
+    }
+    return scanner.finish();
+}
+class EnvelopeScanner {
+    stack = [];
+    inString = false;
+    escaped = false;
+    stringIsKey = false;
+    stringBytes = [];
+    captureHeaderValue = false;
+    headerValueBytes = [];
+    metadata = {};
+    sawData = false;
+    storedLineage = false;
+    binary = false;
+    metadataIsObject = false;
+    metadataHasKeptProperty = false;
+    write(chunk) {
+        for (const byte of chunk)
+            this.writeByte(byte);
+    }
+    finish() {
+        if (this.inString || this.stack.length !== 0 || !this.sawData) {
+            throw new Error("Stored data envelope is not complete JSON");
+        }
+        const envelope = DataFileEnvelopeSchema.parse({
+            ...this.metadata,
+            data: {},
+        });
+        const { data: _data, ...header } = envelope;
+        const headerJson = JSON.stringify(header);
+        return {
+            prefix: encoder.encode(`${headerJson.slice(0, -1)},"data":`),
+            storedLineage: this.storedLineage,
+            binary: this.binary,
+            metadataIsObject: this.metadataIsObject,
+            metadataHasKeptProperty: this.metadataHasKeptProperty,
+        };
+    }
+    writeByte(byte) {
+        if (this.inString) {
+            if (this.stringIsKey || this.captureHeaderValue) {
+                const target = this.stringIsKey
+                    ? this.stringBytes
+                    : this.headerValueBytes;
+                target.push(byte);
+                if (target.length > MAX_METADATA_TOKEN_BYTES) {
+                    throw new Error("Stored data envelope metadata token is too large");
+                }
+            }
+            if (this.escaped) {
+                this.escaped = false;
+            }
+            else if (byte === BACKSLASH) {
+                this.escaped = true;
+            }
+            else if (byte === QUOTE) {
+                this.endString();
+            }
+            return;
+        }
+        if (isWhitespace(byte))
+            return;
+        const frame = this.stack.at(-1);
+        if (byte === QUOTE) {
+            this.inString = true;
+            this.stringIsKey = frame?.type === "object" && frame.expectKey;
+            this.stringBytes.length = 0;
+            this.headerValueBytes.length = 0;
+            if (this.stringIsKey)
+                this.stringBytes.push(byte);
+            this.captureHeaderValue =
+                !this.stringIsKey &&
+                    frame?.type === "object" &&
+                    frame.path === "root" &&
+                    isHeaderKey(frame.currentKey);
+            if (this.captureHeaderValue)
+                this.headerValueBytes.push(byte);
+            return;
+        }
+        if (byte === 0x7b) {
+            const path = childObjectPath(frame);
+            if (path === "data")
+                this.sawData = true;
+            if (path === "metadata")
+                this.metadataIsObject = true;
+            this.stack.push({
+                type: "object",
+                path,
+                suppress: false,
+                expectKey: true,
+                keepCurrent: true,
+                wroteProperty: false,
+            });
+            return;
+        }
+        if (byte === 0x5b) {
+            this.stack.push({ type: "array", path: "other", suppress: false });
+            return;
+        }
+        if (byte === 0x7d || byte === 0x5d) {
+            this.stack.pop();
+            return;
+        }
+        if (byte === 0x2c) {
+            if (frame?.type === "object") {
+                frame.expectKey = true;
+                frame.currentKey = undefined;
+            }
+            return;
+        }
+        if (frame?.type === "object" &&
+            frame.path === "data" &&
+            frame.currentKey === "$binary" &&
+            byte === 0x74) {
+            this.binary = true;
+        }
+    }
+    endString() {
+        const frame = this.stack.at(-1);
+        if (this.stringIsKey && frame?.type === "object") {
+            const key = JSON.parse(decoder.decode(Uint8Array.from(this.stringBytes)));
+            frame.currentKey = key;
+            frame.expectKey = false;
+            if (frame.path === "data" && key === "$lineage")
+                this.storedLineage = true;
+            if (frame.path === "metadata" && key !== "lineage") {
+                this.metadataHasKeptProperty = true;
+            }
+        }
+        else if (this.captureHeaderValue &&
+            frame?.type === "object" &&
+            frame.currentKey) {
+            this.metadata[frame.currentKey] = JSON.parse(decoder.decode(Uint8Array.from(this.headerValueBytes)));
+        }
+        this.inString = false;
+        this.stringIsKey = false;
+        this.captureHeaderValue = false;
+    }
+}
+class EnvelopeRedactor extends Transform {
+    plan;
+    stack = [];
+    inString = false;
+    escaped = false;
+    stringIsKey = false;
+    stringSuppressed = false;
+    stringBytes = [];
+    pending = [];
+    constructor(plan) {
+        super();
+        this.plan = plan;
+    }
+    _transform(chunk, _encoding, callback) {
+        try {
+            for (const byte of chunk)
+                this.writeByte(byte);
+            this.flushPending();
+            callback();
+        }
+        catch (error) {
+            callback(error);
+        }
+    }
+    writeByte(byte) {
+        if (this.inString) {
+            if (this.stringIsKey) {
+                this.stringBytes.push(byte);
+                if (this.stringBytes.length > MAX_METADATA_TOKEN_BYTES) {
+                    throw new Error("Stored data envelope property name is too large");
+                }
+            }
+            if (!this.stringSuppressed && !this.stringIsKey)
+                this.pushByte(byte);
+            if (this.escaped) {
+                this.escaped = false;
+            }
+            else if (byte === BACKSLASH) {
+                this.escaped = true;
+            }
+            else if (byte === QUOTE) {
+                if (this.stringIsKey)
+                    this.endKey();
+                this.inString = false;
+            }
+            return;
+        }
+        if (isWhitespace(byte))
+            return;
+        const frame = this.stack.at(-1);
+        if (byte === QUOTE) {
+            this.inString = true;
+            this.stringIsKey = frame?.type === "object" && frame.expectKey;
+            this.stringBytes.length = 0;
+            if (this.stringIsKey)
+                this.stringBytes.push(byte);
+            this.stringSuppressed = this.stringIsKey || isSuppressed(frame);
+            if (!this.stringSuppressed)
+                this.pushByte(byte);
+            return;
+        }
+        if (byte === 0x7b) {
+            const path = childObjectPath(frame);
+            const suppress = isSuppressed(frame);
+            if (path === "root") {
+                // The validated canonical prefix is emitted when the data value starts.
+            }
+            else if (path === "data") {
+                this.pushBytes(this.plan.prefix);
+                this.pushByte(byte);
+            }
+            else if (!suppress) {
+                this.pushByte(byte);
+            }
+            this.stack.push({
+                type: "object",
+                path,
+                suppress,
+                expectKey: true,
+                keepCurrent: true,
+                wroteProperty: false,
+            });
+            return;
+        }
+        if (byte === 0x5b) {
+            const suppress = isSuppressed(frame);
+            if (!suppress)
+                this.pushByte(byte);
+            this.stack.push({ type: "array", path: "other", suppress });
+            return;
+        }
+        if (byte === 0x7d || byte === 0x5d) {
+            const closing = this.stack.pop();
+            if (closing?.type === "object" && closing.path === "root") {
+                this.pushByte(byte);
+            }
+            else if (closing && !closing.suppress) {
+                this.pushByte(byte);
+            }
+            return;
+        }
+        if (byte === 0x2c) {
+            if (frame?.type === "object") {
+                frame.expectKey = true;
+                frame.currentKey = undefined;
+                frame.keepCurrent = true;
+            }
+            else if (!isSuppressed(frame)) {
+                this.pushByte(byte);
+            }
+            return;
+        }
+        if (byte === 0x3a) {
+            if (frame?.type === "object" &&
+                frame.path !== "root" &&
+                frame.keepCurrent &&
+                !frame.suppress) {
+                this.pushByte(byte);
+            }
+            return;
+        }
+        if (!isSuppressed(frame))
+            this.pushByte(byte);
+    }
+    endKey() {
+        const frame = this.stack.at(-1);
+        if (frame?.type !== "object")
+            return;
+        const keyBytes = Uint8Array.from(this.stringBytes);
+        const key = JSON.parse(decoder.decode(keyBytes));
+        frame.currentKey = key;
+        frame.expectKey = false;
+        frame.keepCurrent = shouldKeepProperty(frame.path, key, this.plan);
+        if (frame.path === "root")
+            return;
+        if (frame.keepCurrent && !frame.suppress) {
+            if (frame.wroteProperty)
+                this.pushByte(0x2c);
+            this.pushBytes(keyBytes);
+            frame.wroteProperty = true;
+        }
+    }
+    pushByte(byte) {
+        this.pending.push(byte);
+        if (this.pending.length === OUTPUT_CHUNK_BYTES)
+            this.flushPending();
+    }
+    pushBytes(bytes) {
+        for (const byte of bytes)
+            this.pushByte(byte);
+    }
+    flushPending() {
+        if (this.pending.length === 0)
+            return;
+        this.push(Buffer.from(this.pending));
+        this.pending.length = 0;
+    }
+}
+function childObjectPath(frame) {
+    if (!frame)
+        return "root";
+    if (frame.type !== "object")
+        return "other";
+    if (frame.path === "root" && frame.currentKey === "data")
+        return "data";
+    if (frame.path === "data" && frame.currentKey === "metadata") {
+        return "metadata";
+    }
+    return "other";
+}
+function shouldKeepProperty(path, key, plan) {
+    if (path === "root")
+        return key === "data";
+    if (path === "metadata") {
+        return !(plan.storedLineage && plan.binary && key === "lineage");
+    }
+    if (path !== "data")
+        return true;
+    if (key === "$writtenBy" || key === "$lineage")
+        return false;
+    if (plan.storedLineage && !plan.binary && key === "lineage")
+        return false;
+    if (plan.storedLineage &&
+        plan.binary &&
+        key === "metadata" &&
+        plan.metadataIsObject &&
+        !plan.metadataHasKeptProperty) {
+        return false;
+    }
+    return true;
+}
+function isSuppressed(frame) {
+    if (!frame)
+        return false;
+    if (frame.suppress)
+        return true;
+    return frame.type === "object" && !frame.keepCurrent;
+}
+function isHeaderKey(key) {
+    return (key === "$schema" ||
+        key === "version" ||
+        key === "scope" ||
+        key === "schemaId" ||
+        key === "collectedAt");
+}
+function isWhitespace(byte) {
+    return byte === 0x20 || byte === 0x09 || byte === 0x0a || byte === 0x0d;
+}
+//# sourceMappingURL=raw-envelope-stream.js.map

package/dist/jobs/raw-envelope-stream.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"raw-envelope-stream.js","sourceRoot":"","sources":["../../src/jobs/raw-envelope-stream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,SAAS,EAA0B,MAAM,aAAa,CAAC;AAC1E,OAAO,EAAE,sBAAsB,EAAE,MAAM,gCAAgC,CAAC;AAExE,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;AAClC,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;AAClC,MAAM,KAAK,GAAG,IAAI,CAAC;AACnB,MAAM,SAAS,GAAG,IAAI,CAAC;AACvB,MAAM,wBAAwB,GAAG,EAAE,GAAG,IAAI,CAAC;AAC3C,MAAM,kBAAkB,GAAG,EAAE,GAAG,IAAI,CAAC;AA4BrC,kFAAkF;AAClF,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAC9C,IAA+C;IAE/C,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;IAC9C,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,MAAM,IAAI,EAAE,CAAU,CAAC,CAAC;IACxD,MAAM,QAAQ,GAAG,IAAI,gBAAgB,CAAC,IAAI,CAAC,CAAC;IAC5C,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;IACtD,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;IAC5C,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACrB,OAAO,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAA0C,CAAC;AAC3E,CAAC;AAED,KAAK,UAAU,YAAY,CACzB,MAAkC;IAElC,MAAM,OAAO,GAAG,IAAI,eAAe,EAAE,CAAC;IACtC,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,QAAQ,CAAC,OAAO,CAAC,MAAe,CAAC,EAAE,CAAC;QAC5D,OAAO,CAAC,KAAK,CAAC,KAAmB,CAAC,CAAC;IACrC,CAAC;IACD,OAAO,OAAO,CAAC,MAAM,EAAE,CAAC;AAC1B,CAAC;AAED,MAAM,eAAe;IACF,KAAK,GAAY,EAAE,CAAC;IAC7B,QAAQ,GAAG,KAAK,CAAC;IACjB,OAAO,GAAG,KAAK,CAAC;IAChB,WAAW,GAAG,KAAK,CAAC;IACX,WAAW,GAAa,EAAE,CAAC;IACpC,kBAAkB,GAAG,KAAK,CAAC;IAClB,gBAAgB,GAAa,EAAE,CAAC;IAChC,QAAQ,GAA4B,EAAE,CAAC;IAChD,OAAO,GAAG,KAAK,CAAC;IAChB,aAAa,GAAG,KAAK,CAAC;IACtB,MAAM,GAAG,KAAK,CAAC;IACf,gBAAgB,GAAG,KAAK,CAAC;IACzB,uBAAuB,GAAG,KAAK,CAAC;IAExC,KAAK,CAAC,KAAiB;QACrB,KAAK,MAAM,IAAI,IAAI,KAAK;YAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACjD,CAAC;IAED,MAAM;QACJ,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAC9D,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;QAC/D,CAAC;QACD,MAAM,QAAQ,GAAG,sBAAsB,CAAC,KAAK,CAAC;YAC5C,GAAG,IAAI,CAAC,QAAQ;YAChB,IAAI,EAAE,EAAE;SACT,CAAC,CAAC;QACH,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,EAAE,GAAG,QAAQ,CAAC;QAC5C,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QAC1C,OAAO;YACL,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC;YAC5D,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;YACvC,uBAAuB,EAAE,IAAI,CAAC,uBAAuB;SACtD,CAAC;IACJ,CAAC;IAEO,SAAS,CAAC,IAAY;QAC5B,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;gBAChD,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW;oBAC7B,CAAC,CAAC,IAAI,CAAC,WAAW;oBAClB,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC;gBAC1B,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAClB,IAAI,MAAM,CAAC,MAAM,GAAG,wBAAwB,EAAE,CAAC;oBAC7C,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;gBACtE,CAAC;YACH,CAAC;YACD,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gBACjB,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;YACvB,CAAC;iBAAM,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;gBAC9B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;YACtB,CAAC;iBAAM,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;gBAC1B,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,CAAC;YACD,OAAO;QACT,CAAC;QACD,IAAI,YAAY,CAAC,IAAI,CAAC;YAAE,OAAO;QAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAChC,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;YACnB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;YACrB,IAAI,CAAC,WAAW,GAAG,KAAK,EAAE,IAAI,KAAK,QAAQ,IAAI,KAAK,CAAC,SAAS,CAAC;YAC/D,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC;YAC5B,IAAI,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC;YACjC,IAAI,IAAI,CAAC,WAAW;gBAAE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAClD,IAAI,CAAC,kBAAkB;gBACrB,CAAC,IAAI,CAAC,WAAW;oBACjB,KAAK,EAAE,IAAI,KAAK,QAAQ;oBACxB,KAAK,CAAC,IAAI,KAAK,MAAM;oBACrB,WAAW,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YAChC,IAAI,IAAI,CAAC,kBAAkB;gBAAE,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC9D,OAAO;QACT,CAAC;QACD,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;YAClB,MAAM,IAAI,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;YACpC,IAAI,IAAI,KAAK,MAAM;gBAAE,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;YACzC,IAAI,IAAI,KAAK,UAAU;gBAAE,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;YACtD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;gBACd,IAAI,EAAE,QAAQ;gBACd,IAAI;gBACJ,QAAQ,EAAE,KAAK;gBACf,SAAS,EAAE,IAAI;gBACf,WAAW,EAAE,IAAI;gBACjB,aAAa,EAAE,KAAK;aACrB,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QACD,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;YAClB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;YACnE,OAAO;QACT,CAAC;QACD,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;YACnC,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;YACjB,OAAO;QACT,CAAC;QACD,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;YAClB,IAAI,KAAK,EAAE,IAAI,KAAK,QAAQ,EAAE,CAAC;gBAC7B,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC;gBACvB,KAAK,CAAC,UAAU,GAAG,SAAS,CAAC;YAC/B,CAAC;YACD,OAAO;QACT,CAAC;QACD,IACE,KAAK,EAAE,IAAI,KAAK,QAAQ;YACxB,KAAK,CAAC,IAAI,KAAK,MAAM;YACrB,KAAK,CAAC,UAAU,KAAK,SAAS;YAC9B,IAAI,KAAK,IAAI,EACb,CAAC;YACD,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACrB,CAAC;IACH,CAAC;IAEO,SAAS;QACf,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAChC,IAAI,IAAI,CAAC,WAAW,IAAI,KAAK,EAAE,IAAI,KAAK,QAAQ,EAAE,CAAC;YACjD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CACpB,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CACxC,CAAC;YACZ,KAAK,CAAC,UAAU,GAAG,GAAG,CAAC;YACvB,KAAK,CAAC,SAAS,GAAG,KAAK,CAAC;YACxB,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,IAAI,GAAG,KAAK,UAAU;gBAC7C,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;YAC5B,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;gBACnD,IAAI,CAAC,uBAAuB,GAAG,IAAI,CAAC;YACtC,CAAC;QACH,CAAC;aAAM,IACL,IAAI,CAAC,kBAAkB;YACvB,KAAK,EAAE,IAAI,KAAK,QAAQ;YACxB,KAAK,CAAC,UAAU,EAChB,CAAC;YACD,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,KAAK,CAC1C,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAC5C,CAAC;QACf,CAAC;QACD,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QACtB,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;QACzB,IAAI,CAAC,kBAAkB,GAAG,KAAK,CAAC;IAClC,CAAC;CACF;AAED,MAAM,gBAAiB,SAAQ,SAAS;IAST;IARZ,KAAK,GAAY,EAAE,CAAC;IAC7B,QAAQ,GAAG,KAAK,CAAC;IACjB,OAAO,GAAG,KAAK,CAAC;IAChB,WAAW,GAAG,KAAK,CAAC;IACpB,gBAAgB,GAAG,KAAK,CAAC;IAChB,WAAW,GAAa,EAAE,CAAC;IAC3B,OAAO,GAAa,EAAE,CAAC;IAExC,YAA6B,IAAkB;QAC7C,KAAK,EAAE,CAAC;QADmB,SAAI,GAAJ,IAAI,CAAc;IAE/C,CAAC;IAEQ,UAAU,CACjB,KAAa,EACb,SAAyB,EACzB,QAA2B;QAE3B,IAAI,CAAC;YACH,KAAK,MAAM,IAAI,IAAI,KAAK;gBAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YAC/C,IAAI,CAAC,YAAY,EAAE,CAAC;YACpB,QAAQ,EAAE,CAAC;QACb,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,QAAQ,CAAC,KAAc,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;IAEO,SAAS,CAAC,IAAY;QAC5B,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;gBACrB,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC5B,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,wBAAwB,EAAE,CAAC;oBACvD,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;gBACrE,CAAC;YACH,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,gBAAgB,IAAI,CAAC,IAAI,CAAC,WAAW;gBAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YACrE,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gBACjB,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;YACvB,CAAC;iBAAM,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;gBAC9B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;YACtB,CAAC;iBAAM,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;gBAC1B,IAAI,IAAI,CAAC,WAAW;oBAAE,IAAI,CAAC,MAAM,EAAE,CAAC;gBACpC,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;YACxB,CAAC;YACD,OAAO;QACT,CAAC;QACD,IAAI,YAAY,CAAC,IAAI,CAAC;YAAE,OAAO;QAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAChC,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;YACnB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;YACrB,IAAI,CAAC,WAAW,GAAG,KAAK,EAAE,IAAI,KAAK,QAAQ,IAAI,KAAK,CAAC,SAAS,CAAC;YAC/D,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC;YAC5B,IAAI,IAAI,CAAC,WAAW;gBAAE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAClD,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,WAAW,IAAI,YAAY,CAAC,KAAK,CAAC,CAAC;YAChE,IAAI,CAAC,IAAI,CAAC,gBAAgB;gBAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YAChD,OAAO;QACT,CAAC;QACD,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;YAClB,MAAM,IAAI,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;YACpC,MAAM,QAAQ,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;YACrC,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;gBACpB,wEAAwE;YAC1E,CAAC;iBAAM,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;gBAC3B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACjC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YACtB,CAAC;iBAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACrB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YACtB,CAAC;YACD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;gBACd,IAAI,EAAE,QAAQ;gBACd,IAAI;gBACJ,QAAQ;gBACR,SAAS,EAAE,IAAI;gBACf,WAAW,EAAE,IAAI;gBACjB,aAAa,EAAE,KAAK;aACrB,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QACD,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;YAClB,MAAM,QAAQ,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;YACrC,IAAI,CAAC,QAAQ;gBAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YACnC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;YAC5D,OAAO;QACT,CAAC;QACD,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;YACnC,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;YACjC,IAAI,OAAO,EAAE,IAAI,KAAK,QAAQ,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBAC1D,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YACtB,CAAC;iBAAM,IAAI,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;gBACxC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YACtB,CAAC;YACD,OAAO;QACT,CAAC;QACD,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;YAClB,IAAI,KAAK,EAAE,IAAI,KAAK,QAAQ,EAAE,CAAC;gBAC7B,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC;gBACvB,KAAK,CAAC,UAAU,GAAG,SAAS,CAAC;gBAC7B,KAAK,CAAC,WAAW,GAAG,IAAI,CAAC;YAC3B,CAAC;iBAAM,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;gBAChC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YACtB,CAAC;YACD,OAAO;QACT,CAAC;QACD,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;YAClB,IACE,KAAK,EAAE,IAAI,KAAK,QAAQ;gBACxB,KAAK,CAAC,IAAI,KAAK,MAAM;gBACrB,KAAK,CAAC,WAAW;gBACjB,CAAC,KAAK,CAAC,QAAQ,EACf,CAAC;gBACD,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YACtB,CAAC;YACD,OAAO;QACT,CAAC;QACD,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC;YAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAChD,CAAC;IAEO,MAAM;QACZ,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAChC,IAAI,KAAK,EAAE,IAAI,KAAK,QAAQ;YAAE,OAAO;QACrC,MAAM,QAAQ,GAAG,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACnD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAW,CAAC;QAC3D,KAAK,CAAC,UAAU,GAAG,GAAG,CAAC;QACvB,KAAK,CAAC,SAAS,GAAG,KAAK,CAAC;QACxB,KAAK,CAAC,WAAW,GAAG,kBAAkB,CAAC,KAAK,CAAC,IAAI,EAAE,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;QACnE,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM;YAAE,OAAO;QAClC,IAAI,KAAK,CAAC,WAAW,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;YACzC,IAAI,KAAK,CAAC,aAAa;gBAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YAC7C,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;YACzB,KAAK,CAAC,aAAa,GAAG,IAAI,CAAC;QAC7B,CAAC;IACH,CAAC;IAEO,QAAQ,CAAC,IAAY;QAC3B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxB,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,kBAAkB;YAAE,IAAI,CAAC,YAAY,EAAE,CAAC;IACtE,CAAC;IAEO,SAAS,CAAC,KAAiB;QACjC,KAAK,MAAM,IAAI,IAAI,KAAK;YAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAChD,CAAC;IAEO,YAAY;QAClB,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QACtC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;QACrC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;IAC1B,CAAC;CACF;AAED,SAAS,eAAe,CAAC,KAAwB;IAC/C,IAAI,CAAC,KAAK;QAAE,OAAO,MAAM,CAAC;IAC1B,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ;QAAE,OAAO,OAAO,CAAC;IAC5C,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,IAAI,KAAK,CAAC,UAAU,KAAK,MAAM;QAAE,OAAO,MAAM,CAAC;IACxE,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,IAAI,KAAK,CAAC,UAAU,KAAK,UAAU,EAAE,CAAC;QAC7D,OAAO,UAAU,CAAC;IACpB,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,kBAAkB,CACzB,IAAyB,EACzB,GAAW,EACX,IAAkB;IAElB,IAAI,IAAI,KAAK,MAAM;QAAE,OAAO,GAAG,KAAK,MAAM,CAAC;IAC3C,IAAI,IAAI,KAAK,UAAU,EAAE,CAAC;QACxB,OAAO,CAAC,CAAC,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,MAAM,IAAI,GAAG,KAAK,SAAS,CAAC,CAAC;IACnE,CAAC;IACD,IAAI,IAAI,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC;IACjC,IAAI,GAAG,KAAK,YAAY,IAAI,GAAG,KAAK,UAAU;QAAE,OAAO,KAAK,CAAC;IAC7D,IAAI,IAAI,CAAC,aAAa,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,GAAG,KAAK,SAAS;QAAE,OAAO,KAAK,CAAC;IAC1E,IACE,IAAI,CAAC,aAAa;QAClB,IAAI,CAAC,MAAM;QACX,GAAG,KAAK,UAAU;QAClB,IAAI,CAAC,gBAAgB;QACrB,CAAC,IAAI,CAAC,uBAAuB,EAC7B,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,YAAY,CAAC,KAAwB;IAC5C,IAAI,CAAC,KAAK;QAAE,OAAO,KAAK,CAAC;IACzB,IAAI,KAAK,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAC;IAChC,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC;AACvD,CAAC;AAED,SAAS,WAAW,CAAC,GAAuB;IAC1C,OAAO,CACL,GAAG,KAAK,SAAS;QACjB,GAAG,KAAK,SAAS;QACjB,GAAG,KAAK,OAAO;QACf,GAAG,KAAK,UAAU;QAClB,GAAG,KAAK,aAAa,CACtB,CAAC;AACJ,CAAC;AAED,SAAS,YAAY,CAAC,IAAY;IAChC,OAAO,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,CAAC;AAC1E,CAAC"}

package/dist/jobs/types.d.ts

@@ -0,0 +1,17 @@
+import type { Hex } from "viem";
+export type JobFailureCode = "AUTH_INVALID" | "BUILDER_MISMATCH" | "OWNER_MISMATCH" | "GRANT_REVOKED" | "GRANT_INVALID" | "SIGNED_ARTIFACT_MISSING" | "SIGNED_ARTIFACT_INVALID" | "SERVER_NOT_REGISTERED" | "SCOPE_NOT_FOUND" | "VERSION_MISMATCH" | "DEADLINE_PASSED" | "RESULT_SIGNING_REFUSED" | "RESULT_UPLOAD_FAILED" | "RESULT_TOO_LARGE" | "INTERNAL";
+export interface JobExecuteResponse {
+    /** Object key in vana-storage: `jobresults/{chainId}/{jobId}`. */
+    resultObjectKey: string;
+    /** SHA-256 of ciphertext bytes. */
+    resultHash: Hex;
+    resultSize: number;
+}
+export interface JobExecuteError {
+    error: {
+        code: JobFailureCode;
+        message: string;
+        retryable: boolean;
+    };
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/jobs/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/jobs/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,MAAM,CAAC;AAGhC,MAAM,MAAM,cAAc,GACtB,cAAc,GACd,kBAAkB,GAClB,gBAAgB,GAChB,eAAe,GACf,eAAe,GACf,yBAAyB,GACzB,yBAAyB,GACzB,uBAAuB,GACvB,iBAAiB,GACjB,kBAAkB,GAClB,iBAAiB,GACjB,wBAAwB,GACxB,sBAAsB,GACtB,kBAAkB,GAClB,UAAU,CAAC;AAEf,MAAM,WAAW,kBAAkB;IACjC,kEAAkE;IAClE,eAAe,EAAE,MAAM,CAAC;IACxB,mCAAmC;IACnC,UAAU,EAAE,GAAG,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE;QACL,IAAI,EAAE,cAAc,CAAC;QACrB,OAAO,EAAE,MAAM,CAAC;QAChB,SAAS,EAAE,OAAO,CAAC;KACpB,CAAC;CACH"}

package/dist/jobs/types.js

@@ -0,0 +1,3 @@
+export {};
+// JOB_EXECUTE_WIRE_END
+//# sourceMappingURL=types.js.map

package/dist/jobs/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/jobs/types.ts"],"names":[],"mappings":";AAmCA,uBAAuB"}

package/dist/jobs/worker.d.ts

@@ -0,0 +1,43 @@
+import type { AccessLogWriter } from "@opendatalabs/personal-server-ts-core/logging/access-log";
+import type { DataStoragePort, ProtocolGatewayPort } from "@opendatalabs/personal-server-ts-core/ports";
+import type { ScopeDeletionTracker } from "@opendatalabs/personal-server-ts-core/sync";
+import { type DataPortabilityGatewayConfig } from "@opendatalabs/vana-sdk/node";
+import { type JobRequestEnvelope } from "@opendatalabs/vana-sdk/protocol/jobs";
+import type { ECIESProvider } from "@opendatalabs/vana-sdk/crypto/ecies/interface";
+import type { Logger } from "pino";
+import { type Address, type Hex } from "viem";
+import type { JobExecuteResponse, JobFailureCode } from "./types.js";
+export declare const DEFAULT_JOB_RESULT_MAX_BYTES: number;
+export interface JobWorkerDeps {
+    serverOwner: Address;
+    serverAddress: Address;
+    serverPublicKey: Hex;
+    authAudience: string;
+    gateway: ProtocolGatewayPort;
+    gatewayConfig: DataPortabilityGatewayConfig;
+    storage: DataStoragePort;
+    ecies: ECIESProvider;
+    now?: () => Date;
+    logger: Logger;
+    accessLogWriter: AccessLogWriter;
+    scopeDeletions?: ScopeDeletionTracker;
+    /** Refresh one exact scope before retrying a local read selection. */
+    hydrateScope?: (scope: string) => Promise<void>;
+    resultMaxBytes?: number;
+    resultTempDir?: string;
+    resultUpload: {
+        storageEndpoint: string;
+        agentEndpoint: string;
+        accessToken: string;
+        chainId: number;
+        fetch?: typeof fetch;
+    };
+}
+export declare class JobFailure extends Error {
+    readonly code: JobFailureCode;
+    readonly retryable: boolean;
+    constructor(code: JobFailureCode, message: string, retryable: boolean);
+}
+export declare function executeJob(envelope: JobRequestEnvelope, deps: JobWorkerDeps): Promise<JobExecuteResponse>;
+export declare function readJobResultMaxBytes(value: string | undefined): number;
+//# sourceMappingURL=worker.d.ts.map

package/dist/jobs/worker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"worker.d.ts","sourceRoot":"","sources":["../../src/jobs/worker.ts"],"names":[],"mappings":"AAsBA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,0DAA0D,CAAC;AAChG,OAAO,KAAK,EACV,eAAe,EACf,mBAAmB,EACpB,MAAM,6CAA6C,CAAC;AAMrD,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,4CAA4C,CAAC;AACvF,OAAO,EAGL,KAAK,4BAA4B,EAClC,MAAM,6BAA6B,CAAC;AAKrC,OAAO,EAEL,KAAK,kBAAkB,EAExB,MAAM,sCAAsC,CAAC;AAC9C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,+CAA+C,CAAC;AACnF,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AACnC,OAAO,EAAkB,KAAK,OAAO,EAAE,KAAK,GAAG,EAAE,MAAM,MAAM,CAAC;AAE9D,OAAO,KAAK,EAAE,kBAAkB,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAmBrE,eAAO,MAAM,4BAA4B,QAAqB,CAAC;AAK/D,MAAM,WAAW,aAAa;IAC5B,WAAW,EAAE,OAAO,CAAC;IACrB,aAAa,EAAE,OAAO,CAAC;IACvB,eAAe,EAAE,GAAG,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,mBAAmB,CAAC;IAC7B,aAAa,EAAE,4BAA4B,CAAC;IAC5C,OAAO,EAAE,eAAe,CAAC;IACzB,KAAK,EAAE,aAAa,CAAC;IACrB,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,EAAE,eAAe,CAAC;IACjC,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,sEAAsE;IACtE,YAAY,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAChD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE;QACZ,eAAe,EAAE,MAAM,CAAC;QACxB,aAAa,EAAE,MAAM,CAAC;QACtB,WAAW,EAAE,MAAM,CAAC;QACpB,OAAO,EAAE,MAAM,CAAC;QAChB,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;KACtB,CAAC;CACH;AAED,qBAAa,UAAW,SAAQ,KAAK;aAEjB,IAAI,EAAE,cAAc;aAEpB,SAAS,EAAE,OAAO;gBAFlB,IAAI,EAAE,cAAc,EACpC,OAAO,EAAE,MAAM,EACC,SAAS,EAAE,OAAO;CAKrC;AAED,wBAAsB,UAAU,CAC9B,QAAQ,EAAE,kBAAkB,EAC5B,IAAI,EAAE,aAAa,GAClB,OAAO,CAAC,kBAAkB,CAAC,CAc7B;AA8RD,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,CAOvE"}