@sema-agent/core 7.15.0 → 7.17.0

package/dist/tools/artifact/local-stub.js

@@ -0,0 +1,296 @@
+import { randomBytes } from "node:crypto";
+import { closeSync, existsSync, fsyncSync, mkdirSync, openSync, readFileSync, readdirSync, realpathSync, renameSync, rmSync, writeFileSync } from "node:fs";
+import { dirname, join, resolve, sep } from "node:path";
+import {} from "../../core/artifact-host.js";
+import { acquireStoreDirLock } from "../../stores/file/fs-atomic.js";
+import { normalizePublishedPath } from "./artifact-tool.js";
+export const EVAL_STUB_URL_PREFIX = "eval-stub://artifact/";
+export const LOCAL_STUB_MAX_FILE_BYTES = 16 * 1024 * 1024;
+export const LOCAL_STUB_MAX_TOTAL_BYTES = 64 * 1024 * 1024;
+const LEDGER_FILE = "ledger.json";
+const TMP_DIR = ".tmp";
+const LOCK_FILE = "LOCK";
+const RESERVED_NAMES = new Set([LEDGER_FILE, TMP_DIR, LOCK_FILE].map((n) => n.toLowerCase()));
+const SLUG_RE = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/;
+const ATTEMPT_WINDOW = 512;
+const emptyLedger = () => ({ v: 1, artifacts: {}, attempts: {} });
+const isRow = (x, key) => {
+    if (x === null || typeof x !== "object")
+        return false;
+    const r = x;
+    if (typeof r.slug !== "string" || r.slug !== key || !SLUG_RE.test(r.slug) || r.slug.startsWith(".") || RESERVED_NAMES.has(r.slug.toLowerCase()))
+        return false;
+    if (typeof r.url !== "string" || !Number.isSafeInteger(r.version) || r.version < 1 || !Array.isArray(r.files))
+        return false;
+    return r.files.every((f) => f !== null && typeof f === "object" && typeof f.path === "string" && normalizePublishedPath(f.path).ok && Number.isSafeInteger(f.bytes));
+};
+function parseLedger(text, path) {
+    let parsed;
+    try {
+        parsed = JSON.parse(text);
+    }
+    catch {
+        throw new Error(`artifact stub ledger is corrupt (not JSON): ${path}`);
+    }
+    const l = parsed;
+    if (l === null || typeof l !== "object" || l.v !== 1 || l.artifacts === null || typeof l.artifacts !== "object" || l.attempts === null || typeof l.attempts !== "object") {
+        throw new Error(`artifact stub ledger is corrupt (unexpected shape): ${path}`);
+    }
+    for (const [key, row] of Object.entries(l.artifacts))
+        if (!isRow(row, key))
+            throw new Error(`artifact stub ledger is corrupt (bad artifact row): ${path}`);
+    return { v: 1, artifacts: { ...l.artifacts }, attempts: { ...l.attempts } };
+}
+function fsyncDirBestEffort(dir) {
+    try {
+        const fd = openSync(dir, "r");
+        try {
+            fsyncSync(fd);
+        }
+        finally {
+            closeSync(fd);
+        }
+    }
+    catch {
+    }
+}
+function fsyncTreeBestEffort(dir) {
+    for (const entry of readdirSync(dir, { withFileTypes: true }))
+        if (entry.isDirectory())
+            fsyncTreeBestEffort(join(dir, entry.name));
+    fsyncDirBestEffort(dir);
+}
+function writeFsync(path, bytes) {
+    const fd = openSync(path, "w", 0o600);
+    try {
+        writeFileSync(fd, bytes);
+        fsyncSync(fd);
+    }
+    finally {
+        closeSync(fd);
+    }
+}
+function slugBase(fileName) {
+    const dot = fileName.lastIndexOf(".");
+    const stem = (dot > 0 ? fileName.slice(0, dot) : fileName).toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 40);
+    return stem.length > 0 ? stem : "artifact";
+}
+export class LocalArtifactStub {
+    dir;
+    #mint = new Map();
+    #chain = Promise.resolve();
+    constructor(opts) {
+        if (typeof opts?.dir !== "string" || opts.dir.length === 0)
+            throw new Error("localArtifactStub: `dir` is required");
+        this.dir = resolve(opts.dir);
+    }
+    capabilities() {
+        return { plane: "local-stub", actions: ["publish"], defaultVisibility: "private", maxFileBytes: LOCAL_STUB_MAX_FILE_BYTES, maxTotalBytes: LOCAL_STUB_MAX_TOTAL_BYTES, multiFile: false, storageDir: this.dir };
+    }
+    onMount(runId) {
+        this.#mint.set(runId, new Set());
+    }
+    ownMinted(runId) {
+        return new Set(this.#mint.get(runId) ?? []);
+    }
+    publish(req, ctx) {
+        const run = this.#chain.then(() => this.#publishLocked(req, ctx));
+        this.#chain = run.then(() => undefined, () => undefined);
+        return run;
+    }
+    #ledgerPath() {
+        return join(this.dir, LEDGER_FILE);
+    }
+    #loadLedger() {
+        const p = this.#ledgerPath();
+        if (!existsSync(p))
+            return emptyLedger();
+        return parseLedger(readFileSync(p, "utf8"), p);
+    }
+    #versionDir(slug, version) {
+        return join(this.dir, slug, `v${version}`);
+    }
+    #snapshotOf(row) {
+        const base = this.#versionDir(row.slug, row.version);
+        const realBase = realpathSync(base);
+        const files = row.files.map((f) => {
+            const target = realpathSync(resolve(base, ...f.path.split("/")));
+            if (!target.startsWith(realBase + sep))
+                throw new Error(`artifact stub ledger is corrupt (a file resolves outside its version directory): ${this.#ledgerPath()}`);
+            return { path: f.path, content: new Uint8Array(readFileSync(target)), ...(f.mediaType !== undefined ? { mediaType: f.mediaType } : {}) };
+        });
+        return {
+            files,
+            ...(row.title !== undefined ? { title: row.title } : {}),
+            ...(row.description !== undefined ? { description: row.description } : {}),
+            ...(row.favicon !== undefined ? { favicon: row.favicon } : {}),
+            ...(row.label !== undefined ? { label: row.label } : {}),
+        };
+    }
+    #mintSlug(ledger, requested, firstFile) {
+        const fold = requested?.toLowerCase();
+        if (requested !== undefined && fold !== undefined && SLUG_RE.test(requested) && !RESERVED_NAMES.has(fold) && !requested.startsWith(".") && !Object.keys(ledger.artifacts).some((s) => s.toLowerCase() === fold))
+            return requested;
+        const base = slugBase(firstFile);
+        const taken = new Set(Object.keys(ledger.artifacts).map((s) => s.toLowerCase()));
+        for (;;) {
+            const slug = `${base}-${randomBytes(3).toString("hex")}`;
+            if (!taken.has(slug))
+                return slug;
+        }
+    }
+    #commitLedger(next) {
+        const ledgerTmp = join(this.dir, `${LEDGER_FILE}.${randomBytes(4).toString("hex")}.tmp`);
+        try {
+            const text = JSON.stringify(next, null, 2);
+            writeFsync(ledgerTmp, text);
+            const verified = parseLedger(readFileSync(ledgerTmp, "utf8"), ledgerTmp);
+            if (JSON.stringify(verified) !== JSON.stringify(next))
+                throw new Error("artifact stub ledger did not read back as written");
+            renameSync(ledgerTmp, this.#ledgerPath());
+        }
+        catch (err) {
+            try {
+                rmSync(ledgerTmp, { force: true });
+            }
+            catch {
+            }
+            throw err;
+        }
+        fsyncDirBestEffort(this.dir);
+    }
+    #withAttempt(ledger, attemptId, attempt) {
+        const entries = Object.entries(ledger.attempts).filter(([k]) => k !== attemptId);
+        while (entries.length >= ATTEMPT_WINDOW)
+            entries.shift();
+        entries.push([attemptId, attempt]);
+        return Object.fromEntries(entries);
+    }
+    #replayOf(ledger, attempt) {
+        if ("ref" in attempt)
+            return { ok: true, ref: attempt.ref };
+        if (attempt.code === "conflict") {
+            const row = ledger.artifacts[attempt.slug];
+            return { ok: false, code: "conflict", currentVersion: row !== undefined ? String(row.version) : attempt.currentVersion, current: row !== undefined ? this.#snapshotOf(row) : "reread" };
+        }
+        return { ok: false, code: attempt.code, message: attempt.message };
+    }
+    async #publishLocked(req, ctx) {
+        mkdirSync(this.dir, { recursive: true, mode: 0o700 });
+        const lease = acquireStoreDirLock(this.dir, { label: "artifact stub", fileName: LOCK_FILE });
+        try {
+            const ledger = this.#loadLedger();
+            const replay = ledger.attempts[ctx.attemptId];
+            if (replay !== undefined)
+                return this.#replayOf(ledger, replay);
+            const decided = this.#decide(ledger, req, ctx);
+            if (!decided.ok) {
+                const attempt = decided.outcome.code === "conflict" ? { code: "conflict", slug: decided.slug ?? "", currentVersion: decided.outcome.currentVersion } : { code: decided.outcome.code, message: decided.outcome.message };
+                this.#commitLedger({ ...ledger, attempts: this.#withAttempt(ledger, ctx.attemptId, attempt) });
+                return decided.outcome;
+            }
+            const { slug, version, existing, files, mint } = decided;
+            const versionDir = this.#versionDir(slug, version);
+            const tmpDir = join(this.dir, TMP_DIR, `${slug}-v${version}-${randomBytes(4).toString("hex")}`);
+            let staged = false;
+            try {
+                if (existsSync(versionDir))
+                    rmSync(versionDir, { recursive: true, force: true });
+                mkdirSync(tmpDir, { recursive: true, mode: 0o700 });
+                for (const f of files) {
+                    const target = join(tmpDir, ...f.path.split("/"));
+                    mkdirSync(dirname(target), { recursive: true, mode: 0o700 });
+                    writeFsync(target, f.content);
+                }
+                fsyncTreeBestEffort(tmpDir);
+                mkdirSync(dirname(versionDir), { recursive: true, mode: 0o700 });
+                renameSync(tmpDir, versionDir);
+                staged = true;
+                fsyncDirBestEffort(dirname(versionDir));
+                const now = new Date().toISOString();
+                const url = `${EVAL_STUB_URL_PREFIX}${slug}`;
+                const row = {
+                    slug,
+                    url,
+                    version,
+                    ...(req.title !== undefined ? { title: req.title } : existing?.title !== undefined ? { title: existing.title } : {}),
+                    ...(req.description !== undefined ? { description: req.description } : existing?.description !== undefined ? { description: existing.description } : {}),
+                    ...(req.favicon !== undefined ? { favicon: req.favicon } : existing?.favicon !== undefined ? { favicon: existing.favicon } : {}),
+                    ...(req.label !== undefined ? { label: req.label } : {}),
+                    files: files.map((f) => ({ path: f.path, ...(f.mediaType !== undefined ? { mediaType: f.mediaType } : {}), bytes: f.content.byteLength })),
+                    createdAt: existing?.createdAt ?? now,
+                    updatedAt: now,
+                };
+                const ref = { url, slug, version: String(version), ...(row.title !== undefined ? { title: row.title } : {}) };
+                this.#commitLedger({ v: 1, artifacts: { ...ledger.artifacts, [slug]: row }, attempts: this.#withAttempt(ledger, ctx.attemptId, { ref }) });
+                mint.add(slug);
+                return { ok: true, ref };
+            }
+            catch (err) {
+                for (const orphan of [tmpDir, ...(staged ? [versionDir] : [])]) {
+                    try {
+                        rmSync(orphan, { recursive: true, force: true });
+                    }
+                    catch {
+                    }
+                }
+                throw err;
+            }
+        }
+        finally {
+            lease.release();
+        }
+    }
+    #decide(ledger, req, ctx) {
+        const refuse = (code, message, slug) => ({ ok: false, outcome: { ok: false, code, message }, ...(slug !== undefined ? { slug } : {}) });
+        if (req.removeFiles !== undefined && req.removeFiles.length > 0)
+            return refuse("refused", "the local stub does not apply removeFiles");
+        if (req.files.length === 0)
+            return refuse("refused", "a publish needs at least one file");
+        const seen = new Set();
+        const files = [];
+        let total = 0;
+        for (const f of req.files) {
+            const norm = normalizePublishedPath(f.path);
+            if (!norm.ok)
+                return refuse("refused", `published path ${JSON.stringify(f.path).slice(0, 120)} ${norm.reason}`);
+            const fold = norm.path.toLowerCase();
+            if (seen.has(fold))
+                return refuse("refused", `published path ${JSON.stringify(norm.path).slice(0, 120)} collides with another entry`);
+            seen.add(fold);
+            total += f.content.byteLength;
+            if (f.content.byteLength > LOCAL_STUB_MAX_FILE_BYTES)
+                return refuse("payload_too_large", `${norm.path.slice(0, 120)} is ${f.content.byteLength} bytes; the limit is ${LOCAL_STUB_MAX_FILE_BYTES} bytes per file`);
+            files.push({ path: norm.path, content: f.content, ...(f.mediaType !== undefined ? { mediaType: f.mediaType } : {}) });
+        }
+        if (total > LOCAL_STUB_MAX_TOTAL_BYTES)
+            return refuse("payload_too_large", `the files total ${total} bytes; the limit is ${LOCAL_STUB_MAX_TOTAL_BYTES} bytes per version`);
+        const mint = this.#mint.get(ctx.runId) ?? new Set();
+        this.#mint.set(ctx.runId, mint);
+        if (req.url !== undefined) {
+            if (!req.url.startsWith(EVAL_STUB_URL_PREFIX))
+                return refuse("url_foreign", "not an eval-stub:// link this run published");
+            const slug = req.url.slice(EVAL_STUB_URL_PREFIX.length);
+            if (!mint.has(slug))
+                return refuse("url_foreign", "this run did not publish that artifact", slug);
+            const existing = ledger.artifacts[slug];
+            if (existing === undefined)
+                return refuse("not_found", "the artifact is no longer in the ledger", slug);
+            if (req.baseVersion !== undefined && req.baseVersion !== String(existing.version) && req.force !== true) {
+                return { ok: false, outcome: { ok: false, code: "conflict", currentVersion: String(existing.version), current: this.#snapshotOf(existing) }, slug };
+            }
+            return { ok: true, slug, version: existing.version + 1, existing, files, mint };
+        }
+        return { ok: true, slug: this.#mintSlug(ledger, req.slug, files[0].path), version: 1, existing: undefined, files, mint };
+    }
+    rows() {
+        return Object.values(this.#loadLedger().artifacts).map((r) => ({ url: r.url, slug: r.slug, version: String(r.version), ...(r.title !== undefined ? { title: r.title } : {}) }));
+    }
+    versionFiles(slug, version) {
+        const dir = this.#versionDir(slug, version);
+        return existsSync(dir) ? readdirSync(dir, { recursive: true }).map(String).sort() : [];
+    }
+}
+export function localArtifactStub(opts) {
+    return new LocalArtifactStub(opts);
+}