hammerkit 1.24.0 → 1.25.0

package/dist/server/proposals.js

@@ -1,9 +1,32 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ProposalRegistry = exports.ProposalError = void 0;
+exports.ProposalRegistry = exports.StaleProposalError = exports.CooldownError = exports.ProposalError = void 0;
+const units_1 = require("../utils/units");
 class ProposalError extends Error {
+    constructor() {
+        super(...arguments);
+        // what the caller is told
+        this.status = 503;
+    }
 }
 exports.ProposalError = ProposalError;
+// The caller proposes too soon after a rejection
+class CooldownError extends ProposalError {
+    constructor() {
+        super(...arguments);
+        this.status = 429;
+    }
+}
+exports.CooldownError = CooldownError;
+// The build file moved on since the proposal was made
+class StaleProposalError extends Error {
+    constructor(base, latest) {
+        super(`the build file changed since this was proposed (it follows ${base ?? 'nothing'}, the latest is ${latest ?? 'nothing'}), propose it again on top of the latest`);
+        this.base = base;
+        this.latest = latest;
+    }
+}
+exports.StaleProposalError = StaleProposalError;
 // The proposals of the server, in memory: a restart forgets what waits, and
 // whoever proposed it proposes it again. Nothing in here is runnable.
 class ProposalRegistry {
@@ -13,10 +36,11 @@ class ProposalRegistry {
         this.maxWaiting = options.maxWaiting ?? 100;
         this.timeout = options.timeout ?? 60 * 60 * 1000;
         this.keep = options.keep ?? 100;
+        this.cooldown = options.cooldown ?? 0;
         this.now = options.now ?? Date.now;
     }
     // Proposing what already waits is not an error and changes nothing.
-    propose(version, bundle, proposer) {
+    propose(version, bundle, proposer, missing = [], base = null) {
         if (this.closed) {
             throw new ProposalError('the server is shutting down');
         }
@@ -25,6 +49,7 @@ class ProposalRegistry {
         if (existing && existing.proposal.state === 'waiting') {
             return { proposal: { ...existing.proposal }, created: false };
         }
+        this.checkCooldown(version, proposer);
         if (this.waiting().length >= this.maxWaiting) {
             throw new ProposalError('too many build file versions are waiting for an approval');
         }
@@ -34,6 +59,10 @@ class ProposalRegistry {
             entry: version.entry,
             tasks: version.tasks,
             services: version.services,
+            accounts: version.accounts ?? [],
+            missing,
+            plan: version.plan ?? {},
+            base,
             proposedBy: { iss: proposer.iss, sub: proposer.sub },
             createdAt: new Date(this.now()).toISOString(),
             expiresAt: new Date(this.now() + this.timeout).toISOString(),
@@ -67,7 +96,7 @@ class ProposalRegistry {
     // decides on their own proposal.
     async decide(decider, name, hash, outcome, mayDecide, publish, reason) {
         const entry = this.entries.get(this.key(name, hash));
-        if (!entry || !mayDecide(name)) {
+        if (!entry || !mayDecide(entry.proposal)) {
             return { status: 'not-found' };
         }
         if (entry.proposal.proposedBy.iss === decider.iss && entry.proposal.proposedBy.sub === decider.sub) {
@@ -112,6 +141,24 @@ class ProposalRegistry {
         }
         entry.bundle = undefined;
     }
+    // A rejection is a no; a proposal that only ran out of time is not.
+    checkCooldown(version, proposer) {
+        if (this.cooldown <= 0) {
+            return;
+        }
+        for (const { proposal } of this.entries.values()) {
+            if (proposal.state !== 'rejected' || proposal.name !== version.name) {
+                continue;
+            }
+            const left = Date.parse(proposal.decidedAt ?? '') + this.cooldown - this.now();
+            const sameVersion = proposal.hash === version.hash;
+            const sameProposer = proposal.proposedBy.iss === proposer.iss && proposal.proposedBy.sub === proposer.sub;
+            if (left > 0 && (sameVersion || sameProposer)) {
+                throw new CooldownError(`${sameVersion ? 'this version was' : 'a version of this build file by you was'} rejected, ` +
+                    `propose again in ${(0, units_1.formatDuration)(Math.ceil(left / 1000) * 1000)}`);
+            }
+        }
+    }
     evict() {
         const decided = [...this.entries.entries()].filter(([, e]) => e.proposal.state !== 'waiting');
         for (const [key] of decided.slice(0, Math.max(0, decided.length - this.keep))) {

package/dist/server/repo-access.js

@@ -0,0 +1,127 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RepoAccess = void 0;
+const redact_1 = require("../utils/redact");
+const provider_values_1 = require("../secrets/provider-values");
+const git_checkout_1 = require("./git-checkout");
+const repo_1 = require("./repo");
+// A token is read again after this long, so a short-lived one (a GitHub App
+// installation token minted by the provider command) is not used up and a revoked one
+// stops working soon.
+const TOKEN_TTL_MS = 60 * 1000;
+// A run waits for the repository at submit, not forever
+const RESOLVE_TIMEOUT_MS = 30 * 1000;
+/**
+ * The repositories a server can run code from. What may be read is the operator's
+ * `sources` list; who may name which of them is the entitlement's `repos`. The read
+ * credential is a secret of a provider, read as a service account that nobody can use
+ * in a task. It is handed to git for one fetch through the environment of that git
+ * process (never an argument, never `.git/config`, never the environment of a task)
+ * and masked in whatever the run logs.
+ */
+class RepoAccess {
+    constructor(config, 
+    // the base environment of the server's runs, with `hosted` set
+    environment, options = {}) {
+        this.config = config;
+        this.environment = environment;
+        this.options = options;
+        this.tokens = new Map();
+        this.now = options.now ?? Date.now;
+    }
+    // The first source that covers the repository: patterns are listed from the most
+    // specific to the least.
+    source(repo) {
+        return this.config.sources.find((source) => (0, repo_1.matchesAnyRepo)([source.match], repo));
+    }
+    readable(repo) {
+        return this.source(repo) !== undefined;
+    }
+    url(repo) {
+        return this.options.url ? this.options.url(repo) : `https://${repo}.git`;
+    }
+    // The commit a ref points to now, or an error that says what is wrong with the ref.
+    async resolve(repo, ref, signal) {
+        const timeout = AbortSignal.timeout(RESOLVE_TIMEOUT_MS);
+        const abort = signal ? AbortSignal.any([signal, timeout]) : timeout;
+        if (this.options.resolve) {
+            return this.options.resolve(repo, ref, abort);
+        }
+        const credentials = await this.credentials(repo);
+        try {
+            return await (0, git_checkout_1.resolveRef)(this.url(repo), ref, credentials.env, abort);
+        }
+        catch (e) {
+            throw this.scrub(e, credentials.values);
+        }
+    }
+    /**
+     * The environment a git process needs to read the repository: the allowlisted
+     * variables of the server, and the token as a header of requests to that host. The
+     * values are what the run has to mask.
+     */
+    async credentials(repo) {
+        const env = {};
+        for (const [key, value] of Object.entries(this.environment.processEnvs)) {
+            if (value !== undefined) {
+                env[key] = value;
+            }
+        }
+        const secret = this.source(repo)?.secret;
+        if (!secret) {
+            return { env, values: [] };
+        }
+        const token = await this.token(secret);
+        const host = repo.split('/')[0];
+        const header = `Authorization: Basic ${Buffer.from(`${secret.username}:${token}`).toString('base64')}`;
+        env.GIT_CONFIG_COUNT = '1';
+        env.GIT_CONFIG_KEY_0 = `http.https://${host}/.extraHeader`;
+        env.GIT_CONFIG_VALUE_0 = header;
+        return { env, values: [token, header] };
+    }
+    async token(secret) {
+        const key = `${secret.account}\0${secret.from}`;
+        const known = this.tokens.get(key);
+        if (known && this.now() - known.at < TOKEN_TTL_MS) {
+            return known.value;
+        }
+        const catalog = this.environment.hosted?.catalog;
+        const kind = secret.from.substring(0, secret.from.indexOf(':'));
+        const ref = secret.from.substring(secret.from.indexOf(':') + 1);
+        const provider = catalog.providers[kind];
+        const account = catalog.accounts[secret.account];
+        const credentials = account?.providers[kind];
+        if (!provider || !credentials) {
+            throw new Error(`the source credential ${secret.from} as ${secret.account} is not defined`);
+        }
+        // a fresh environment per read: values are kept here and for a short time only
+        const registry = (0, redact_1.createSecretRegistry)();
+        const environment = {
+            ...this.environment,
+            secrets: registry,
+            hosted: { catalog, accounts: [secret.account] },
+        };
+        let value;
+        try {
+            value = await (0, provider_values_1.fetchProviderValue)({ definition: provider, account: { name: secret.account, env: credentials.env } }, ref, environment);
+        }
+        catch (e) {
+            throw new Error(`could not read the source credential ${secret.from}: ${e instanceof Error ? e.message : e}`);
+        }
+        if (value.length === 0) {
+            throw new Error(`the source credential ${secret.from} is empty`);
+        }
+        this.tokens.set(key, { value, at: this.now() });
+        return value;
+    }
+    // An error as the caller may read it: without the token.
+    scrub(error, values) {
+        const registry = (0, redact_1.createSecretRegistry)();
+        values.forEach((value) => registry.register(value));
+        const message = error instanceof Error ? error.message : String(error);
+        const scrubbed = new (error instanceof Error ? error.constructor : Error)(registry.redact(message));
+        return scrubbed;
+    }
+}
+exports.RepoAccess = RepoAccess;
+//# sourceMappingURL=repo-access.js.map

package/dist/server/repo.js

@@ -0,0 +1,94 @@
+"use strict";
+// Repositories are named `host/path`, such as `github.com/corp/payments` or
+// `gitlab.corp.com/platform/team/tool`, so one pattern can cover a GitHub
+// organisation or a GitLab group with its subgroups. Names are compared in
+// lower case: both hosts treat them that way.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.matchesRef = exports.matchesAnyRepo = exports.matchesRepo = exports.normalizeRepoPattern = exports.isRepoPattern = exports.normalizeRepo = void 0;
+const SEGMENT = /^[a-z0-9._-]+$/;
+const PATTERN_SEGMENT = /^[a-z0-9._*-]+$/;
+// `host/path` of what a caller or an operator wrote, or null when it is not a
+// plain repository address: no credentials, port, query, option-like names or
+// `..`. A scheme (`https://`) and a trailing `.git` are accepted and dropped.
+function normalizeRepo(input) {
+    let value = input.trim().toLowerCase();
+    value = value.replace(/^https:\/\//, '');
+    value = value.replace(/\.git$/, '').replace(/\/+$/, '');
+    const parts = value.split('/');
+    if (parts.length < 2 || parts.length > 12) {
+        return null;
+    }
+    const [host, ...path] = parts;
+    if (!/^[a-z0-9]([a-z0-9.-]*[a-z0-9])?$/.test(host) || host.includes('..')) {
+        return null;
+    }
+    if (path.some((part) => !SEGMENT.test(part) || part === '.' || part === '..' || part.startsWith('-'))) {
+        return null;
+    }
+    return [host, ...path].join('/');
+}
+exports.normalizeRepo = normalizeRepo;
+// A pattern is a repository where a segment may hold `*` (any characters but `/`),
+// and the last one may be `**`: everything below, subgroups included.
+function isRepoPattern(input) {
+    const value = input
+        .trim()
+        .toLowerCase()
+        .replace(/^https:\/\//, '')
+        .replace(/\/+$/, '');
+    const parts = value.split('/');
+    if (parts.length < 2 || !/^[a-z0-9*]([a-z0-9.*-]*[a-z0-9*])?$/.test(parts[0])) {
+        return false;
+    }
+    return parts.slice(1).every((part, index, rest) => {
+        if (part === '**') {
+            return index === rest.length - 1;
+        }
+        return PATTERN_SEGMENT.test(part) && part !== '.' && part !== '..' && !part.startsWith('-') && !part.includes('**');
+    });
+}
+exports.isRepoPattern = isRepoPattern;
+function normalizeRepoPattern(input) {
+    return input
+        .trim()
+        .toLowerCase()
+        .replace(/^https:\/\//, '')
+        .replace(/\/+$/, '');
+}
+exports.normalizeRepoPattern = normalizeRepoPattern;
+function matchesRepo(pattern, repo) {
+    const parts = normalizeRepoPattern(pattern).split('/');
+    const target = repo.split('/');
+    const last = parts[parts.length - 1];
+    if (last === '**') {
+        const fixed = parts.slice(0, -1);
+        return target.length > fixed.length && fixed.every((part, index) => segmentMatches(part, target[index]));
+    }
+    return parts.length === target.length && parts.every((part, index) => segmentMatches(part, target[index]));
+}
+exports.matchesRepo = matchesRepo;
+function segmentMatches(pattern, segment) {
+    const regex = pattern
+        .split('*')
+        .map((part) => part.replace(/[.+?^${}()|[\]\\]/g, '\\$&'))
+        .join('[^/]*');
+    return new RegExp(`^${regex}$`).test(segment);
+}
+function matchesAnyRepo(patterns, repo) {
+    return patterns.some((pattern) => matchesRepo(pattern, repo));
+}
+exports.matchesAnyRepo = matchesAnyRepo;
+// A ref is matched as written: `main`, `release/*` (one level) or `release/**`.
+function matchesRef(patterns, ref) {
+    return patterns.some((pattern) => {
+        const parts = pattern.split('/');
+        const target = ref.split('/');
+        if (parts[parts.length - 1] === '**') {
+            const fixed = parts.slice(0, -1);
+            return target.length > fixed.length && fixed.every((part, index) => segmentMatches(part, target[index]));
+        }
+        return parts.length === target.length && parts.every((part, index) => segmentMatches(part, target[index]));
+    });
+}
+exports.matchesRef = matchesRef;
+//# sourceMappingURL=repo.js.map

package/dist/server/run-environment.js

@@ -0,0 +1,20 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runEnvironment = void 0;
+// What a run sees of the server's environment: the variables the operator listed
+// in `runs.env.allow`, and the fixed ones of `runs.env.set`. The credentials of a
+// run are not among them, they exist only as the secrets of an account.
+function runEnvironment(config, processEnvs) {
+    const passed = {};
+    for (const name of config.runs.env.allow) {
+        // Windows names are not case sensitive
+        const windows = process.platform === 'win32';
+        const key = Object.keys(processEnvs).find((k) => k === name || (windows && k.toLowerCase() === name.toLowerCase()));
+        if (key !== undefined && processEnvs[key] !== undefined) {
+            passed[name] = processEnvs[key];
+        }
+    }
+    return { ...passed, ...config.runs.env.set };
+}
+exports.runEnvironment = runEnvironment;
+//# sourceMappingURL=run-environment.js.map

package/dist/server/run-plan.js

@@ -0,0 +1,44 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.planRun = exports.getRunScopes = void 0;
+const parse_label_arguments_1 = require("../parser/parse-label-arguments");
+const plan_1 = require("./plan");
+function labels(values) {
+    return values && values.length > 0 ? (0, parse_label_arguments_1.parseLabelArguments)(values) : undefined;
+}
+// The tasks in the order they were named, one scope each; by labels when none is.
+function getRunScopes(tasks, options) {
+    return tasks.length > 0
+        ? tasks.map((task) => ({
+            name: task,
+            scope: { taskName: task, environmentName: options.env ?? null },
+        }))
+        : [
+            {
+                name: 'labels',
+                scope: {
+                    filterLabels: labels(options.filter),
+                    excludeLabels: labels(options.exclude),
+                    environmentName: options.env ?? null,
+                },
+            },
+        ];
+}
+exports.getRunScopes = getRunScopes;
+/**
+ * What a run will do, read with the engine from the files of the version it is
+ * pinned to and with the environment a run has: the same function answers at
+ * submit, and again in the executor, so what was approved is what runs.
+ */
+async function planRun(bundle, run, environment, policy) {
+    const plan = {};
+    for (const { scope } of getRunScopes(run.tasks, run.options)) {
+        Object.assign(plan, (await (0, plan_1.planBundle)(bundle, environment, { scope, policy })).plan);
+    }
+    return {
+        plan,
+        accounts: [...new Set(Object.values(plan).flatMap((entry) => entry.accounts))].sort(),
+    };
+}
+exports.planRun = planRun;
+//# sourceMappingURL=run-plan.js.map

package/dist/server/run-request.js

@@ -5,9 +5,11 @@ const zod_1 = require("zod");
 const units_1 = require("../utils/units");
 const bundle_1 = require("../remote/bundle");
 const server_config_1 = require("./server-config");
+const repo_1 = require("./repo");
 const name = zod_1.z.string().min(1).max(200);
-// What `POST /v1/runs` takes. Who is asking, which accounts they may use and
-// where code may come from is decided from the token, never from this body.
+// What `POST /v1/runs` takes. Who is asking and which accounts they may use is
+// decided from the token, never from this body; the accounts a run uses are those
+// its secrets are read as, which the server derives from the published build file.
 const duration = zod_1.z.string().refine((value) => {
     try {
         (0, units_1.parseDuration)(value);
@@ -50,10 +52,26 @@ exports.runRequestSchema = zod_1.z
     // tasks by their name in the build file, as on the command line; labels in
     // the options can select tasks instead
     tasks: zod_1.z.array(name).max(100).default([]),
-    // the service account to use; the caller's default when left out
-    account: name.optional(),
     source: zod_1.z.discriminatedUnion('type', [
-        zod_1.z.object({ type: zod_1.z.literal('ref'), ref: name }).strict(),
+        zod_1.z
+            .object({
+            type: zod_1.z.literal('ref'),
+            // `host/path`, such as github.com/corp/payments; the server reads it from its own
+            // sources, a url is never taken from a request
+            repo: zod_1.z.string().transform((value, ctx) => {
+                const repo = (0, repo_1.normalizeRepo)(value);
+                if (!repo) {
+                    ctx.addIssue({
+                        code: zod_1.z.ZodIssueCode.custom,
+                        message: 'expected a repository like github.com/corp/payments',
+                    });
+                    return zod_1.z.NEVER;
+                }
+                return repo;
+            }),
+            ref: name,
+        })
+            .strict(),
         // the name of a tree announced with POST /v1/sources
         zod_1.z.object({ type: zod_1.z.literal('upload'), snapshot: zod_1.z.string().regex(bundle_1.HASH_PATTERN) }).strict(),
     ]),

package/dist/server/run-store.js

@@ -0,0 +1,61 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.FileRunStore = void 0;
+const promises_1 = require("fs/promises");
+const path_1 = require("path");
+const runs_1 = require("./runs");
+/**
+ * One file per run, written to a temporary file and renamed, so a crash leaves the
+ * old record and not half a new one. A record holds what the server already shows
+ * the caller: the state, the plan, the logs as they were masked. Only the user of the
+ * server can read the files.
+ */
+class FileRunStore {
+    constructor(directory) {
+        this.directory = directory;
+    }
+    async save(record) {
+        await (0, promises_1.mkdir)(this.directory, { recursive: true, mode: 0o700 });
+        const file = (0, path_1.join)(this.directory, `${record.run.id}.json`);
+        const temporary = `${file}.${process.pid}.tmp`;
+        await (0, promises_1.writeFile)(temporary, JSON.stringify(record), { mode: 0o600 });
+        await (0, promises_1.rename)(temporary, file);
+    }
+    async remove(id) {
+        await (0, promises_1.rm)((0, path_1.join)(this.directory, `${id}.json`), { force: true });
+    }
+    async load() {
+        const records = [];
+        const skipped = [];
+        let names;
+        try {
+            names = await (0, promises_1.readdir)(this.directory);
+        }
+        catch {
+            return { records, skipped };
+        }
+        for (const name of names) {
+            if (!name.endsWith('.json')) {
+                // a record that was being written when the server stopped
+                if (name.endsWith('.tmp')) {
+                    await (0, promises_1.rm)((0, path_1.join)(this.directory, name), { force: true });
+                }
+                continue;
+            }
+            try {
+                const record = JSON.parse(await (0, promises_1.readFile)((0, path_1.join)(this.directory, name), 'utf8'));
+                if (!runs_1.RUN_ID_PATTERN.test(record.run.id) || `${record.run.id}.json` !== name || !Array.isArray(record.events)) {
+                    throw new Error('not a run record');
+                }
+                records.push(record);
+            }
+            catch {
+                skipped.push(name);
+            }
+        }
+        records.sort((a, b) => a.run.createdAt.localeCompare(b.run.createdAt));
+        return { records, skipped };
+    }
+}
+exports.FileRunStore = FileRunStore;
+//# sourceMappingURL=run-store.js.map

package/dist/server/run-worker-command.js

@@ -0,0 +1,17 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.registerRunWorkerCommand = void 0;
+const run_worker_1 = require("./run-worker");
+// `hammerkit server-run`: what the server starts for each run. It reads the run on its
+// standard input, executes it in the current directory and reports on its standard output.
+// Not for people: the input comes from the server and says which accounts it may read.
+function registerRunWorkerCommand(program, environment) {
+    program
+        .command('server-run', { hidden: true })
+        .description('execute one run for a hammerkit server (internal)')
+        .action(async () => {
+        await (0, run_worker_1.workerMain)(process.stdin, environment.stdout, environment.processEnvs, environment.cwd, environment.abortCtrl);
+    });
+}
+exports.registerRunWorkerCommand = registerRunWorkerCommand;
+//# sourceMappingURL=run-worker-command.js.map

package/dist/server/run-worker-main.js

@@ -0,0 +1,7 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const run_worker_1 = require("./run-worker");
+// The process of a run without the command line, for tests that run from source
+// (`vite-node src/server/run-worker-main.ts`). A build uses `hammerkit server-run`.
+(0, run_worker_1.workerMain)(process.stdin, process.stdout, process.env, process.cwd(), new AbortController()).then(() => process.exit(0), () => process.exit(1));
+//# sourceMappingURL=run-worker-main.js.map