hammerkit 1.22.0 → 1.24.0

package/dist/server/sources.js

@@ -0,0 +1,258 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SourceStore = exports.subjectKey = exports.DEFAULT_SOURCE_LIMITS = exports.SourceError = void 0;
+const crypto_1 = require("crypto");
+const fs_1 = require("fs");
+const promises_1 = require("fs/promises");
+const path_1 = require("path");
+const crypto_2 = require("crypto");
+const bundle_1 = require("../remote/bundle");
+const source_manifest_1 = require("../remote/source-manifest");
+class SourceError extends Error {
+    constructor(status, message) {
+        super(message);
+        this.status = status;
+    }
+}
+exports.SourceError = SourceError;
+exports.DEFAULT_SOURCE_LIMITS = { subjectBytes: 4 * 1024 * 1024 * 1024 };
+// The directory name of a caller: the issuer and subject, hashed.
+function subjectKey(principal) {
+    return (0, crypto_1.createHash)('sha256')
+        .update(JSON.stringify([principal.iss, principal.sub]))
+        .digest('hex')
+        .slice(0, 32);
+}
+exports.subjectKey = subjectKey;
+/**
+ * The project files that runs on this server work on, kept per caller:
+ *
+ *   <directory>/<caller>/blobs/<sha-256>            content, once per digest
+ *   <directory>/<caller>/manifests/<hash>.json      the trees that were announced
+ *
+ * Content is shared between the manifests of one caller and never between
+ * callers: a server that answers "I have this file" to anybody would tell them
+ * what other people work on.
+ */
+class SourceStore {
+    constructor(directory, limits = exports.DEFAULT_SOURCE_LIMITS, now = Date.now) {
+        this.directory = directory;
+        this.limits = limits;
+        this.now = now;
+        this.subjects = new Map();
+    }
+    // Announces a tree. Answers what has to be sent to complete it.
+    async announce(principal, manifest) {
+        const denied = manifest.files
+            .map((file) => ({ path: file.path, reason: (0, source_manifest_1.deniedSourcePath)(file.path) }))
+            .filter((entry) => entry.reason !== null);
+        if (denied.length > 0) {
+            const shown = denied
+                .slice(0, 5)
+                .map((entry) => `${entry.path} (${entry.reason})`)
+                .join(', ');
+            throw new SourceError(400, `${denied.length} file${denied.length === 1 ? '' : 's'} may not be sent: ${shown}${denied.length > 5 ? ', ...' : ''}`);
+        }
+        const key = subjectKey(principal);
+        const subject = await this.subject(key);
+        const snapshot = (0, source_manifest_1.hashSource)(manifest);
+        const existing = subject.manifests.get(snapshot);
+        if (existing) {
+            return { status: await this.statusOf(key, snapshot, existing), created: false };
+        }
+        // one digest has one size
+        const sizes = new Map();
+        for (const file of manifest.files) {
+            const known = sizes.get(file.sha256) ?? subject.wanted.get(file.sha256);
+            if (known !== undefined && known !== file.size) {
+                throw new SourceError(400, `${file.path}: ${file.sha256} is announced with two sizes`);
+            }
+            sizes.set(file.sha256, file.size);
+        }
+        const stored = { manifest, createdAt: new Date(this.now()).toISOString() };
+        await (0, promises_1.mkdir)((0, path_1.join)(this.directory, key, 'manifests'), { recursive: true });
+        await this.write((0, path_1.join)(this.directory, key, 'manifests', `${snapshot.slice('sha256:'.length)}.json`), stored);
+        subject.manifests.set(snapshot, stored);
+        for (const [digest, size] of sizes) {
+            subject.wanted.set(digest, size);
+        }
+        return { status: await this.statusOf(key, snapshot, stored), created: true };
+    }
+    // Keeps the content of one file, once the digest of what arrives is the one
+    // that was announced. Only content of an announced tree is taken.
+    async put(principal, digest, body) {
+        if (!source_manifest_1.DIGEST_PATTERN.test(digest)) {
+            throw new SourceError(404, 'not found');
+        }
+        const key = subjectKey(principal);
+        const subject = await this.subject(key);
+        const size = subject.wanted.get(digest);
+        if (size === undefined) {
+            throw new SourceError(400, 'this file is not part of a tree you announced');
+        }
+        const target = this.blob(key, digest);
+        if (await exists(target)) {
+            body.resume();
+            return 'exists';
+        }
+        if (subject.stored + size > this.limits.subjectBytes) {
+            body.resume();
+            throw new SourceError(413, `the server keeps at most ${this.limits.subjectBytes} bytes of files for you`);
+        }
+        await (0, promises_1.mkdir)((0, path_1.join)(this.directory, key, 'blobs'), { recursive: true });
+        const temporary = `${target}.${(0, crypto_2.randomBytes)(8).toString('hex')}.tmp`;
+        const hash = (0, crypto_1.createHash)('sha256');
+        let received = 0;
+        const out = (0, fs_1.createWriteStream)(temporary, { flags: 'wx' });
+        try {
+            for await (const chunk of body) {
+                received += chunk.length;
+                if (received > size) {
+                    throw new SourceError(400, `the file is larger than the ${size} bytes that were announced`);
+                }
+                hash.update(chunk);
+                if (!out.write(chunk)) {
+                    await new Promise((resolve) => out.once('drain', () => resolve()));
+                }
+            }
+            await new Promise((resolve, reject) => {
+                out.once('error', reject);
+                out.end(resolve);
+            });
+            if (received !== size) {
+                throw new SourceError(400, `the file has ${received} bytes, ${size} were announced`);
+            }
+            if (hash.digest('hex') !== digest) {
+                throw new SourceError(400, 'the content does not have the digest it was sent for');
+            }
+            await (0, promises_1.rename)(temporary, target);
+        }
+        catch (e) {
+            out.destroy();
+            await (0, promises_1.rm)(temporary, { force: true });
+            throw e;
+        }
+        subject.stored += size;
+        return 'created';
+    }
+    async get(principal, snapshot) {
+        if (!bundle_1.HASH_PATTERN.test(snapshot)) {
+            return undefined;
+        }
+        const key = subjectKey(principal);
+        const subject = await this.subject(key);
+        const stored = subject.manifests.get(snapshot);
+        return stored ? this.statusOf(key, snapshot, stored) : undefined;
+    }
+    async list(principal) {
+        const subject = await this.subject(subjectKey(principal));
+        const snapshots = [...subject.manifests.entries()]
+            .map(([snapshot, stored]) => ({
+            snapshot,
+            files: stored.manifest.files.length,
+            bytes: stored.manifest.files.reduce((sum, file) => sum + file.size, 0),
+            createdAt: stored.createdAt,
+        }))
+            .sort((a, b) => b.createdAt.localeCompare(a.createdAt));
+        return { snapshots, bytes: subject.stored, limit: this.limits.subjectBytes };
+    }
+    // Forgets a tree and the content nothing else of the caller's needs.
+    async remove(principal, snapshot) {
+        if (!bundle_1.HASH_PATTERN.test(snapshot)) {
+            return false;
+        }
+        const key = subjectKey(principal);
+        const subject = await this.subject(key);
+        if (!subject.manifests.delete(snapshot)) {
+            return false;
+        }
+        await (0, promises_1.rm)((0, path_1.join)(this.directory, key, 'manifests', `${snapshot.slice('sha256:'.length)}.json`), { force: true });
+        const kept = new Map();
+        for (const stored of subject.manifests.values()) {
+            for (const file of stored.manifest.files) {
+                kept.set(file.sha256, file.size);
+            }
+        }
+        for (const [digest, size] of subject.wanted) {
+            if (!kept.has(digest)) {
+                subject.wanted.delete(digest);
+                if (await exists(this.blob(key, digest))) {
+                    await (0, promises_1.rm)(this.blob(key, digest), { force: true });
+                    subject.stored -= size;
+                }
+            }
+        }
+        return true;
+    }
+    async statusOf(key, snapshot, stored) {
+        const missing = [];
+        for (const digest of new Set(stored.manifest.files.map((file) => file.sha256))) {
+            if (!(await exists(this.blob(key, digest)))) {
+                missing.push(digest);
+            }
+        }
+        return {
+            snapshot,
+            files: stored.manifest.files.length,
+            bytes: stored.manifest.files.reduce((sum, file) => sum + file.size, 0),
+            createdAt: stored.createdAt,
+            missing,
+        };
+    }
+    blob(key, digest) {
+        return (0, path_1.join)(this.directory, key, 'blobs', digest);
+    }
+    async write(file, value) {
+        const temporary = `${file}.${(0, crypto_2.randomBytes)(8).toString('hex')}.tmp`;
+        await (0, promises_1.writeFile)(temporary, JSON.stringify(value), { flag: 'wx' });
+        await (0, promises_1.rename)(temporary, file);
+    }
+    // Reads what is kept for one caller the first time they are seen.
+    subject(key) {
+        let loaded = this.subjects.get(key);
+        if (!loaded) {
+            loaded = this.load(key);
+            this.subjects.set(key, loaded);
+        }
+        return loaded;
+    }
+    async load(key) {
+        const subject = { wanted: new Map(), stored: 0, manifests: new Map() };
+        const manifests = (0, path_1.join)(this.directory, key, 'manifests');
+        let names = [];
+        try {
+            names = await (0, promises_1.readdir)(manifests);
+        }
+        catch {
+            return subject;
+        }
+        for (const name of names.filter((n) => n.endsWith('.json'))) {
+            const stored = JSON.parse(await (0, promises_1.readFile)((0, path_1.join)(manifests, name), 'utf8'));
+            subject.manifests.set((0, source_manifest_1.hashSource)(stored.manifest), stored);
+            for (const file of stored.manifest.files) {
+                subject.wanted.set(file.sha256, file.size);
+            }
+        }
+        for (const [digest, size] of subject.wanted) {
+            try {
+                await (0, promises_1.stat)(this.blob(key, digest));
+                subject.stored += size;
+            }
+            catch {
+                // announced, not sent yet
+            }
+        }
+        return subject;
+    }
+}
+exports.SourceStore = SourceStore;
+async function exists(file) {
+    try {
+        await (0, promises_1.access)(file);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+//# sourceMappingURL=sources.js.map

package/dist/utils/redact.js

@@ -1,29 +1,85 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.redact = exports.registerSecretValue = void 0;
-// Secret values hammerkit has read, masked in everything it logs, including the
-// output it streams from a task or service. Logs are per line, so each line of
-// a multi-line value (a key file) is masked on its own. Values shorter than
-// MIN_LENGTH are left alone: masking them would mangle unrelated output.
+exports.getEncodedForms = exports.createSecretRegistry = void 0;
+// Secret values a run has read, masked in everything it logs, including the
+// output it streams from a task or service. The registry belongs to one run
+// (its Environment): a process that serves several runs must not mask one
+// run's values in another's output, and must not keep them after the run.
+//
+// Logs are per line, so each line of a multi-line value (a key file) is masked
+// on its own. Values shorter than MIN_LENGTH are left alone: masking them would
+// mangle unrelated output. The usual encodings of a value (base64, hex, URL,
+// JSON) are masked too, since a task or tool often prints those instead; an
+// encoding must be MIN_ENCODED_LENGTH long to be matched, for the same reason.
 const MIN_LENGTH = 4;
+const MIN_ENCODED_LENGTH = 8;
 const MASK = '***';
-const values = new Set();
-function registerSecretValue(value) {
-    for (const candidate of [value, ...value.split(/\r?\n/)]) {
-        const trimmed = candidate.trim();
-        if (trimmed.length >= MIN_LENGTH) {
-            values.add(trimmed);
+function createSecretRegistry() {
+    const values = new Set();
+    // longest first, so a value containing another is masked as a whole
+    let sorted = null;
+    function add(candidate, minLength) {
+        if (candidate.length >= minLength && !values.has(candidate)) {
+            values.add(candidate);
+            sorted = null;
         }
     }
+    return {
+        register(value) {
+            // the whole value, then each line of it; encodings are taken of each as it was written
+            for (const base of new Set([value, ...value.split(/\r?\n/)])) {
+                if (base.trim().length < MIN_LENGTH) {
+                    continue;
+                }
+                add(base.trim(), MIN_LENGTH);
+                for (const form of getEncodedForms(base)) {
+                    add(form.value, form.minLength);
+                }
+            }
+        },
+        redact(message) {
+            if (values.size === 0) {
+                return message;
+            }
+            sorted = sorted ?? [...values].sort((a, b) => b.length - a.length);
+            return sorted.reduce((result, value) => result.split(value).join(MASK), message);
+        },
+    };
 }
-exports.registerSecretValue = registerSecretValue;
-function redact(message) {
-    if (values.size === 0) {
-        return message;
+exports.createSecretRegistry = createSecretRegistry;
+// The encodings of a value that are likely to show up in output. Base64 is
+// matched wherever it starts inside a longer base64 string: the value begins
+// at byte offset 0, 1 or 2 of a 3-byte group, and the characters that mix in
+// bytes before or after it are dropped, since they depend on unknown data.
+function getEncodedForms(value) {
+    const bytes = Buffer.from(value, 'utf8');
+    const forms = [];
+    // the value on its own, as a task or tool usually prints it
+    const whole = bytes.toString('base64');
+    forms.push({ value: whole, minLength: MIN_ENCODED_LENGTH });
+    forms.push({ value: whole.replace(/\+/g, '-').replace(/\//g, '_'), minLength: MIN_ENCODED_LENGTH });
+    for (let offset = 0; offset < 3; offset++) {
+        const encoded = Buffer.concat([Buffer.alloc(offset), bytes])
+            .toString('base64')
+            .replace(/=+$/, '');
+        const leading = [0, 2, 3][offset];
+        const trailing = (bytes.length + offset) % 3 === 0 ? 0 : 1;
+        const form = encoded.substring(leading, encoded.length - trailing);
+        forms.push({ value: form, minLength: MIN_ENCODED_LENGTH });
+        forms.push({ value: form.replace(/\+/g, '-').replace(/\//g, '_'), minLength: MIN_ENCODED_LENGTH });
     }
-    // longest first, so a value containing another is masked as a whole
-    const sorted = [...values].sort((a, b) => b.length - a.length);
-    return sorted.reduce((result, value) => result.split(value).join(MASK), message);
+    const hex = bytes.toString('hex');
+    forms.push({ value: hex, minLength: MIN_ENCODED_LENGTH });
+    forms.push({ value: hex.toUpperCase(), minLength: MIN_ENCODED_LENGTH });
+    const url = encodeURIComponent(value);
+    if (url !== value) {
+        forms.push({ value: url, minLength: MIN_LENGTH });
+    }
+    const json = JSON.stringify(value).slice(1, -1);
+    if (json !== value) {
+        forms.push({ value: json, minLength: MIN_LENGTH });
+    }
+    return forms;
 }
-exports.redact = redact;
+exports.getEncodedForms = getEncodedForms;
 //# sourceMappingURL=redact.js.map

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "hammerkit",
   "description": "Containerized build tool with incremental caching that runs the same build file locally, in Docker and on Kubernetes.",
-  "version": "1.22.0",
+  "version": "1.24.0",
   "keywords": [
     "build",
     "build-tool",