hammerkit 1.25.0 → 1.27.0-next.1

package/dist/server/sandboxes.js

@@ -0,0 +1,519 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SandboxRegistry = exports.SandboxError = void 0;
+const crypto_1 = require("crypto");
+const promises_1 = require("fs/promises");
+const path_1 = require("path");
+const units_1 = require("../utils/units");
+const git_checkout_1 = require("./git-checkout");
+const runs_1 = require("./runs");
+const repo_1 = require("./repo");
+const sandbox_storage_1 = require("./sandbox-storage");
+// How long creating a sandbox or moving it to another commit may take
+const FETCH_TIMEOUT_MS = 10 * 60 * 1000;
+// how old storage without a record has to be before a start removes it
+const ORPHAN_GRACE_MS = 60 * 60 * 1000;
+class SandboxError extends Error {
+    constructor(status, message) {
+        super(message);
+        this.status = status;
+    }
+}
+exports.SandboxError = SandboxError;
+/**
+ * The sandboxes of a server: who owns which, the limits, and one change at a time per
+ * sandbox. Everything is scoped to the owner: somebody else's sandbox is "not found".
+ */
+class SandboxRegistry {
+    constructor(options) {
+        this.options = options;
+        this.sandboxes = new Map();
+        // creates that are under way, so they count against the limit of their owner, with the
+        // bytes they take once they were measured
+        this.creating = new Map();
+        this.deleting = new Set();
+        // the measures of runs that are over, which a run's result does not wait for
+        this.measuring = new Map();
+        this.locks = new Map();
+        this.runs = { holder: () => undefined, stop: async () => undefined };
+        this.now = options.now ?? Date.now;
+    }
+    // The runs are made after the executor, which needs the sandboxes.
+    useRuns(runs) {
+        this.runs = runs;
+    }
+    get limits() {
+        return this.options.limits;
+    }
+    /**
+     * Takes back the sandboxes of a previous start. Storage without a record is what a
+     * create left behind when the server stopped, and a record without storage is of no use.
+     */
+    async load() {
+        const { storage, records } = this.options;
+        // no records at all only on a first start: a directory that can't be read is an error,
+        // not a reason to remove every sandbox
+        const names = await (0, promises_1.readdir)(records).catch((e) => {
+            if (e.code === 'ENOENT') {
+                return [];
+            }
+            throw e;
+        });
+        // a record that can't be read keeps its files, for the operator to look at
+        const recorded = new Set(names.filter((name) => name.endsWith('.json')).map((name) => name.slice(0, -5)));
+        const stored = await storage.list();
+        const ids = new Set(stored.map((entry) => entry.id));
+        for (const name of names) {
+            if (!name.endsWith('.json')) {
+                await (0, promises_1.rm)((0, path_1.join)(records, name), { force: true });
+                continue;
+            }
+            try {
+                const sandbox = JSON.parse(await (0, promises_1.readFile)((0, path_1.join)(records, name), 'utf8'));
+                if (!runs_1.SANDBOX_ID_PATTERN.test(sandbox.id) || `${sandbox.id}.json` !== name) {
+                    throw new Error('not a sandbox record');
+                }
+                if (!ids.has(sandbox.id)) {
+                    this.options.log?.(`drop sandbox ${sandbox.id}: its files are gone`);
+                    await (0, promises_1.rm)((0, path_1.join)(records, name), { force: true });
+                    continue;
+                }
+                this.sandboxes.set(sandbox.id, sandbox);
+            }
+            catch {
+                this.options.log?.(`skip sandbox record ${name}: it can not be read`);
+            }
+        }
+        const now = Date.now();
+        for (const entry of stored) {
+            if (recorded.has(entry.id) || !entry.remove) {
+                continue;
+            }
+            // a sandbox being created has storage before its record
+            if (entry.createdAt && now - entry.createdAt.getTime() < ORPHAN_GRACE_MS) {
+                continue;
+            }
+            this.options.log?.(`remove the storage of sandbox ${entry.id}: it has no record`);
+            await entry.remove();
+        }
+    }
+    // oldest first
+    list(owner) {
+        return [...this.sandboxes.values()]
+            .filter((sandbox) => owns(owner, sandbox))
+            .sort((a, b) => a.createdAt.localeCompare(b.createdAt))
+            .map(copy);
+    }
+    get(owner, id) {
+        const sandbox = this.sandboxes.get(id);
+        return sandbox && owns(owner, sandbox) && !this.deleting.has(id) ? copy(sandbox) : undefined;
+    }
+    holder(id) {
+        return this.runs.holder(id);
+    }
+    usage(owner) {
+        const mine = this.list(owner);
+        const creating = [...this.creating.values()].filter((create) => owns(owner, create.owner));
+        return {
+            count: mine.length + creating.length,
+            bytes: [...mine, ...creating].reduce((sum, sandbox) => sum + sandbox.bytes, 0),
+        };
+    }
+    // Waits for the measures of the owner's runs that are over, so their limits hold.
+    async measured(owner) {
+        const pending = [...this.measuring.entries()].filter(([id]) => {
+            const sandbox = this.sandboxes.get(id);
+            return sandbox && owns(owner, sandbox);
+        });
+        await Promise.all(pending.map(([, measure]) => measure));
+    }
+    // Whether the owner is over their bytes, so no more runs grow their sandboxes.
+    overLimit(owner) {
+        const { bytes } = this.usage(owner);
+        return bytes > this.limits.bytes
+            ? `your sandboxes take ${(0, units_1.formatSize)(bytes)}, more than your limit of ${(0, units_1.formatSize)(this.limits.bytes)}, delete a sandbox`
+            : undefined;
+    }
+    /**
+     * Fetches a branch of a repository into a new sandbox of the owner. Whether they may is the
+     * caller's to decide (`decideSandbox`); this checks the limits.
+     */
+    async create(owner, repo, ref) {
+        const { limits, storage, repos } = this.options;
+        await this.measured(owner);
+        const usage = this.usage(owner);
+        if (usage.count >= limits.count) {
+            throw new SandboxError(413, `you have ${usage.count} sandboxes, the limit is ${limits.count}: delete one`);
+        }
+        if (usage.bytes >= limits.bytes) {
+            throw new SandboxError(413, `your sandboxes take ${(0, units_1.formatSize)(usage.bytes)}, your limit is ${(0, units_1.formatSize)(limits.bytes)}: delete one`);
+        }
+        if (this.total() >= limits.total) {
+            throw new SandboxError(507, 'the server has no room for another sandbox');
+        }
+        const id = `sbx_${(0, crypto_1.randomBytes)(12).toString('base64url')}`;
+        const key = { id, owner: { iss: owner.iss, sub: owner.sub } };
+        this.creating.set(id, { owner: { iss: owner.iss, sub: owner.sub }, bytes: 0 });
+        try {
+            const signal = AbortSignal.timeout(FETCH_TIMEOUT_MS);
+            await this.resolve(repo, ref, signal);
+            const credentials = await repos.credentials(repo);
+            let commit;
+            try {
+                commit = await storage.create(key, { url: repos.url(repo), ref, env: credentials.env, signal });
+            }
+            catch (e) {
+                // the storage's own limits (a quota on a cluster) are said as they are
+                throw e instanceof SandboxError ? e : new SandboxError(502, repos.scrub(e, credentials.values).message);
+            }
+            const bytes = await storage.usage(key);
+            // checked against what is there now, other creates of the owner included, and taken
+            // before anything else can check: no await between the two
+            const used = this.usage(owner).bytes;
+            const problem = used + bytes > limits.bytes
+                ? new SandboxError(413, `${repo} at ${ref} takes ${(0, units_1.formatSize)(bytes)}, more than is left of your limit of ${(0, units_1.formatSize)(limits.bytes)}`)
+                : this.total() + bytes > limits.total
+                    ? new SandboxError(507, 'the server has no room for another sandbox')
+                    : undefined;
+            if (problem) {
+                await storage.remove(key);
+                throw problem;
+            }
+            this.creating.set(id, { owner: { iss: owner.iss, sub: owner.sub }, bytes });
+            const at = this.timestamp();
+            const sandbox = {
+                id,
+                owner: { iss: owner.iss, sub: owner.sub },
+                repo,
+                ref,
+                commit,
+                bytes,
+                createdAt: at,
+                lastUsedAt: at,
+            };
+            await this.save(sandbox);
+            this.sandboxes.set(id, sandbox);
+            return copy(sandbox);
+        }
+        finally {
+            this.creating.delete(id);
+        }
+    }
+    /**
+     * Moves a sandbox to the commit a ref points to now. Only a sandbox without changes is
+     * moved, and none that a run holds: the files of a run are not changed under it.
+     */
+    async fetch(owner, id, ref) {
+        return this.withLock(id, async () => {
+            const sandbox = this.mine(owner, id);
+            this.mustBeFree(id);
+            const state = await this.options.storage.state(keyOf(sandbox));
+            if (!state.clean) {
+                throw new SandboxError(409, `the sandbox ${id} has changes, a fetch would mix them with another commit`);
+            }
+            const { storage, repos } = this.options;
+            const signal = AbortSignal.timeout(FETCH_TIMEOUT_MS);
+            await this.resolve(sandbox.repo, ref, signal);
+            const credentials = await repos.credentials(sandbox.repo);
+            try {
+                sandbox.commit = await storage.fetch(keyOf(sandbox), {
+                    url: repos.url(sandbox.repo),
+                    ref,
+                    env: credentials.env,
+                    signal,
+                });
+            }
+            catch (e) {
+                throw new SandboxError(502, repos.scrub(e, credentials.values).message);
+            }
+            sandbox.ref = ref;
+            sandbox.bytes = await storage.usage(keyOf(sandbox));
+            sandbox.lastUsedAt = this.timestamp();
+            await this.save(sandbox);
+            return copy(sandbox);
+        });
+    }
+    /**
+     * Commits the changes of a sandbox as its owner and pushes them to a branch the operator
+     * allows (`sources[].push.branches`). The branch must not exist yet, or be one this
+     * sandbox pushed and still at what it pushed: a sandbox never writes over somebody
+     * else's work. The sandbox is at the pushed commit afterwards, so its changes are what
+     * is not pushed yet and the next push to the branch adds a commit to it.
+     */
+    // `allowed` are the branches the owner's entitlements let them push to from a sandbox of
+    // its repository
+    async push(owner, id, branch, author, allowed) {
+        return this.withLock(id, async () => {
+            const sandbox = this.mine(owner, id);
+            const { storage, repos } = this.options;
+            if (!repos.source(sandbox.repo)?.secret) {
+                throw new SandboxError(403, `this server does not push to ${sandbox.repo}`);
+            }
+            if (allowed.length === 0) {
+                throw new SandboxError(403, `you may not push from sandboxes of ${sandbox.repo}`);
+            }
+            if (!(0, git_checkout_1.isValidBranch)(branch)) {
+                throw new SandboxError(400, `${JSON.stringify(branch.slice(0, 80))} is not a branch name`);
+            }
+            if (!(0, repo_1.matchesRef)(allowed, branch)) {
+                throw new SandboxError(403, `you may only push to ${allowed.join(', ')} of ${sandbox.repo}`);
+            }
+            this.mustBeFree(id);
+            if ((await storage.state(keyOf(sandbox))).clean) {
+                throw new SandboxError(409, `the sandbox ${id} has no changes to push`);
+            }
+            const signal = AbortSignal.timeout(FETCH_TIMEOUT_MS);
+            const url = repos.url(sandbox.repo);
+            const read = await repos.credentials(sandbox.repo);
+            let current;
+            try {
+                current = await (0, git_checkout_1.remoteBranch)(url, branch, read.env, signal);
+            }
+            catch (e) {
+                throw new SandboxError(502, repos.scrub(e, read.values).message);
+            }
+            const pushed = sandbox.pushed?.[branch];
+            if (current && current !== pushed) {
+                throw new SandboxError(409, pushed
+                    ? `${branch} moved since this sandbox pushed it, push to a new branch`
+                    : `${branch} exists and this sandbox did not push it, push to a new branch`);
+            }
+            let write;
+            try {
+                write = await repos.pushCredentials(sandbox.repo);
+            }
+            catch (e) {
+                // what the provider said is the operator's: it can name accounts, projects and paths
+                this.options.log?.(`sandbox push ${id}: ${e instanceof Error ? e.message : e}`);
+                throw new SandboxError(502, 'the server could not read its credential to push');
+            }
+            let commit;
+            try {
+                commit = await storage.push(keyOf(sandbox), {
+                    url,
+                    branch,
+                    expect: current ?? null,
+                    author,
+                    env: write.env,
+                    signal,
+                });
+            }
+            catch (e) {
+                if (e instanceof sandbox_storage_1.SandboxPushConflict) {
+                    throw new SandboxError(409, e.message);
+                }
+                // the lease refuses a branch that somebody created or moved since it was read; and a
+                // push that failed late (a timeout after the remote answered) may have gone through
+                const now = await (0, git_checkout_1.remoteBranch)(url, branch, read.env, AbortSignal.timeout(30 * 1000)).catch(() => current);
+                if (e instanceof sandbox_storage_1.SandboxPushFailed && now === e.commit) {
+                    commit = e.commit;
+                }
+                else if (now !== current) {
+                    throw new SandboxError(409, `${branch} changed while the sandbox pushed to it, push to a new branch`);
+                }
+                else {
+                    throw new SandboxError(502, repos.scrub(e, [...write.values, ...read.values]).message);
+                }
+            }
+            if (!commit) {
+                // only files the API never reads changed: the deny list stays on the server
+                throw new SandboxError(409, `the sandbox ${id} has no changes to push`);
+            }
+            // the branch is this sandbox's from now on, whatever happens next
+            sandbox.pushed = { ...sandbox.pushed, [branch]: commit };
+            sandbox.lastUsedAt = this.timestamp();
+            await this.save(sandbox);
+            try {
+                await storage.moveTo(keyOf(sandbox), commit);
+            }
+            catch (e) {
+                throw new SandboxError(502, `pushed ${commit} to ${branch}, but could not move the sandbox to it: ${e instanceof Error ? e.message : e}`);
+            }
+            sandbox.commit = commit;
+            sandbox.ref = branch;
+            sandbox.bytes = await storage.usage(keyOf(sandbox));
+            await this.save(sandbox);
+            return copy(sandbox);
+        });
+    }
+    // Cancels the runs on the sandbox, waits for them to stop and removes its files.
+    async remove(owner, id) {
+        const sandbox = this.get(owner, id);
+        if (!sandbox) {
+            return false;
+        }
+        this.deleting.add(id);
+        try {
+            await this.runs.stop(id);
+            await this.withLock(id, async () => {
+                await this.options.storage.remove(keyOf(sandbox));
+                await (0, promises_1.rm)((0, path_1.join)(this.options.records, `${id}.json`), { force: true });
+                this.sandboxes.delete(id);
+            });
+            return true;
+        }
+        finally {
+            this.deleting.delete(id);
+        }
+    }
+    /**
+     * Runs `fn` while nothing else changes the sandbox: a fetch, a delete, or another run
+     * being submitted. The sandbox has to be the owner's and not on its way out.
+     */
+    async withSandbox(owner, id, fn) {
+        return this.withLock(id, async () => fn(copy(this.mine(owner, id))));
+    }
+    // The files of one of the owner's sandboxes, to read one: while a run holds it too.
+    files(owner, id) {
+        const sandbox = this.mine(owner, id);
+        return this.options.storage.files(keyOf(sandbox));
+    }
+    /**
+     * A read that walks the files with other programs (ripgrep, git): between two batches of
+     * edits, and not while a run holds the sandbox. Those programs follow a directory that a
+     * task swaps for a link while they read; a read of one file checks what it opened.
+     */
+    async read(owner, id, fn) {
+        return this.withLock(id, async () => {
+            const sandbox = this.mine(owner, id);
+            this.mustBeFree(id);
+            return fn(this.options.storage.files(keyOf(sandbox)));
+        });
+    }
+    /**
+     * Applies a batch of edits to the files of a sandbox, all of them or none. Not while a run
+     * holds the sandbox, and not past the owner's bytes.
+     */
+    async edit(owner, id, ops, dryRun = false) {
+        return this.withLock(id, async () => {
+            const sandbox = this.mine(owner, id);
+            this.mustBeFree(id);
+            const plan = await this.options.storage.files(keyOf(sandbox)).plan(ops);
+            if (dryRun || plan.files.length === 0) {
+                return plan.files;
+            }
+            if (plan.bytes > 0) {
+                const { limits } = this.options;
+                const used = this.usage(owner).bytes;
+                if (used + plan.bytes > limits.bytes) {
+                    throw new SandboxError(413, `the edits take ${(0, units_1.formatSize)(plan.bytes)} more, your sandboxes take ${(0, units_1.formatSize)(used)} of your limit of ${(0, units_1.formatSize)(limits.bytes)}: delete a sandbox`);
+                }
+                if (this.total() + plan.bytes > limits.total) {
+                    throw new SandboxError(507, 'the server has no room for these edits');
+                }
+            }
+            await plan.commit();
+            sandbox.bytes = Math.max(0, sandbox.bytes + plan.bytes);
+            sandbox.lastUsedAt = this.timestamp();
+            await this.save(sandbox);
+            return plan.files;
+        });
+    }
+    async state(id, signal) {
+        return this.options.storage.state(keyOf(this.existing(id)), signal);
+    }
+    // What a run works on, `into` a directory of the run when the storage copies the files;
+    // the run is the sandbox's use.
+    async attach(id, into) {
+        const sandbox = this.existing(id);
+        sandbox.lastUsedAt = this.timestamp();
+        await this.save(sandbox);
+        return this.options.storage.attach(keyOf(sandbox), into);
+    }
+    // What a sandbox takes after a run wrote to it. The run's result is not held up by it: the
+    // owner's next run or create waits for it (`measured`).
+    measure(id) {
+        const previous = this.measuring.get(id) ?? Promise.resolve();
+        const measure = previous.then(async () => {
+            const sandbox = this.sandboxes.get(id);
+            if (!sandbox || this.deleting.has(id)) {
+                return;
+            }
+            sandbox.bytes = await this.options.storage.usage(keyOf(sandbox));
+            await this.save(sandbox);
+        });
+        const settled = measure.catch(() => undefined);
+        this.measuring.set(id, settled);
+        settled.then(() => {
+            if (this.measuring.get(id) === settled) {
+                this.measuring.delete(id);
+            }
+        });
+        return measure;
+    }
+    existing(id) {
+        const sandbox = this.sandboxes.get(id);
+        if (!sandbox || this.deleting.has(id)) {
+            throw new Error(`the sandbox ${id} was deleted`);
+        }
+        return sandbox;
+    }
+    mine(owner, id) {
+        const sandbox = this.sandboxes.get(id);
+        if (!sandbox || !owns(owner, sandbox) || this.deleting.has(id)) {
+            throw new SandboxError(404, 'not found');
+        }
+        return sandbox;
+    }
+    mustBeFree(id) {
+        const run = this.runs.holder(id);
+        if (run) {
+            throw new SandboxError(409, `the run ${run} holds the sandbox ${id}, wait for it to finish or cancel it`);
+        }
+    }
+    // a sandbox starts from a branch, never a tag or a commit
+    async resolve(repo, ref, signal) {
+        try {
+            (0, git_checkout_1.assertBranchName)(ref);
+            return await this.options.repos.resolve(repo, ref, signal, 'branch');
+        }
+        catch (e) {
+            const message = e instanceof Error ? e.message : String(e);
+            throw e instanceof git_checkout_1.RefError
+                ? new SandboxError(400, message)
+                : new SandboxError(502, `could not resolve ${ref}: ${message}`);
+        }
+    }
+    total() {
+        return [...this.sandboxes.values(), ...this.creating.values()].reduce((sum, sandbox) => sum + sandbox.bytes, 0);
+    }
+    withLock(id, fn) {
+        const previous = this.locks.get(id) ?? Promise.resolve();
+        const next = previous.then(fn, fn);
+        const settled = next.then(() => undefined, () => undefined);
+        this.locks.set(id, settled);
+        void settled.then(() => {
+            if (this.locks.get(id) === settled) {
+                this.locks.delete(id);
+            }
+        });
+        return next;
+    }
+    async save(sandbox) {
+        const directory = this.options.records;
+        await (0, promises_1.mkdir)(directory, { recursive: true, mode: 0o700 });
+        const file = (0, path_1.join)(directory, `${sandbox.id}.json`);
+        const temporary = `${file}.${(0, crypto_1.randomBytes)(4).toString('hex')}.tmp`;
+        await (0, promises_1.writeFile)(temporary, JSON.stringify(sandbox), { mode: 0o600 });
+        await (0, promises_1.rename)(temporary, file);
+    }
+    timestamp() {
+        return new Date(this.now()).toISOString();
+    }
+}
+exports.SandboxRegistry = SandboxRegistry;
+function owns(owner, sandbox) {
+    const of = 'owner' in sandbox ? sandbox.owner : sandbox;
+    return of.iss === owner.iss && of.sub === owner.sub;
+}
+function keyOf(sandbox) {
+    return { id: sandbox.id, owner: sandbox.owner };
+}
+function copy(sandbox) {
+    return {
+        ...sandbox,
+        owner: { ...sandbox.owner },
+        ...(sandbox.pushed ? { pushed: { ...sandbox.pushed } } : {}),
+    };
+}
+//# sourceMappingURL=sandboxes.js.map