hammerkit 1.23.0 → 1.25.0

package/dist/server/sources.js

@@ -0,0 +1,258 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SourceStore = exports.subjectKey = exports.DEFAULT_SOURCE_LIMITS = exports.SourceError = void 0;
+const crypto_1 = require("crypto");
+const fs_1 = require("fs");
+const promises_1 = require("fs/promises");
+const path_1 = require("path");
+const crypto_2 = require("crypto");
+const bundle_1 = require("../remote/bundle");
+const source_manifest_1 = require("../remote/source-manifest");
+class SourceError extends Error {
+    constructor(status, message) {
+        super(message);
+        this.status = status;
+    }
+}
+exports.SourceError = SourceError;
+exports.DEFAULT_SOURCE_LIMITS = { subjectBytes: 4 * 1024 * 1024 * 1024 };
+// The directory name of a caller: the issuer and subject, hashed.
+function subjectKey(principal) {
+    return (0, crypto_1.createHash)('sha256')
+        .update(JSON.stringify([principal.iss, principal.sub]))
+        .digest('hex')
+        .slice(0, 32);
+}
+exports.subjectKey = subjectKey;
+/**
+ * The project files that runs on this server work on, kept per caller:
+ *
+ *   <directory>/<caller>/blobs/<sha-256>            content, once per digest
+ *   <directory>/<caller>/manifests/<hash>.json      the trees that were announced
+ *
+ * Content is shared between the manifests of one caller and never between
+ * callers: a server that answers "I have this file" to anybody would tell them
+ * what other people work on.
+ */
+class SourceStore {
+    constructor(directory, limits = exports.DEFAULT_SOURCE_LIMITS, now = Date.now) {
+        this.directory = directory;
+        this.limits = limits;
+        this.now = now;
+        this.subjects = new Map();
+    }
+    // Announces a tree. Answers what has to be sent to complete it.
+    async announce(principal, manifest) {
+        const denied = manifest.files
+            .map((file) => ({ path: file.path, reason: (0, source_manifest_1.deniedSourcePath)(file.path) }))
+            .filter((entry) => entry.reason !== null);
+        if (denied.length > 0) {
+            const shown = denied
+                .slice(0, 5)
+                .map((entry) => `${entry.path} (${entry.reason})`)
+                .join(', ');
+            throw new SourceError(400, `${denied.length} file${denied.length === 1 ? '' : 's'} may not be sent: ${shown}${denied.length > 5 ? ', ...' : ''}`);
+        }
+        const key = subjectKey(principal);
+        const subject = await this.subject(key);
+        const snapshot = (0, source_manifest_1.hashSource)(manifest);
+        const existing = subject.manifests.get(snapshot);
+        if (existing) {
+            return { status: await this.statusOf(key, snapshot, existing), created: false };
+        }
+        // one digest has one size
+        const sizes = new Map();
+        for (const file of manifest.files) {
+            const known = sizes.get(file.sha256) ?? subject.wanted.get(file.sha256);
+            if (known !== undefined && known !== file.size) {
+                throw new SourceError(400, `${file.path}: ${file.sha256} is announced with two sizes`);
+            }
+            sizes.set(file.sha256, file.size);
+        }
+        const stored = { manifest, createdAt: new Date(this.now()).toISOString() };
+        await (0, promises_1.mkdir)((0, path_1.join)(this.directory, key, 'manifests'), { recursive: true });
+        await this.write((0, path_1.join)(this.directory, key, 'manifests', `${snapshot.slice('sha256:'.length)}.json`), stored);
+        subject.manifests.set(snapshot, stored);
+        for (const [digest, size] of sizes) {
+            subject.wanted.set(digest, size);
+        }
+        return { status: await this.statusOf(key, snapshot, stored), created: true };
+    }
+    // Keeps the content of one file, once the digest of what arrives is the one
+    // that was announced. Only content of an announced tree is taken.
+    async put(principal, digest, body) {
+        if (!source_manifest_1.DIGEST_PATTERN.test(digest)) {
+            throw new SourceError(404, 'not found');
+        }
+        const key = subjectKey(principal);
+        const subject = await this.subject(key);
+        const size = subject.wanted.get(digest);
+        if (size === undefined) {
+            throw new SourceError(400, 'this file is not part of a tree you announced');
+        }
+        const target = this.blob(key, digest);
+        if (await exists(target)) {
+            body.resume();
+            return 'exists';
+        }
+        if (subject.stored + size > this.limits.subjectBytes) {
+            body.resume();
+            throw new SourceError(413, `the server keeps at most ${this.limits.subjectBytes} bytes of files for you`);
+        }
+        await (0, promises_1.mkdir)((0, path_1.join)(this.directory, key, 'blobs'), { recursive: true });
+        const temporary = `${target}.${(0, crypto_2.randomBytes)(8).toString('hex')}.tmp`;
+        const hash = (0, crypto_1.createHash)('sha256');
+        let received = 0;
+        const out = (0, fs_1.createWriteStream)(temporary, { flags: 'wx' });
+        try {
+            for await (const chunk of body) {
+                received += chunk.length;
+                if (received > size) {
+                    throw new SourceError(400, `the file is larger than the ${size} bytes that were announced`);
+                }
+                hash.update(chunk);
+                if (!out.write(chunk)) {
+                    await new Promise((resolve) => out.once('drain', () => resolve()));
+                }
+            }
+            await new Promise((resolve, reject) => {
+                out.once('error', reject);
+                out.end(resolve);
+            });
+            if (received !== size) {
+                throw new SourceError(400, `the file has ${received} bytes, ${size} were announced`);
+            }
+            if (hash.digest('hex') !== digest) {
+                throw new SourceError(400, 'the content does not have the digest it was sent for');
+            }
+            await (0, promises_1.rename)(temporary, target);
+        }
+        catch (e) {
+            out.destroy();
+            await (0, promises_1.rm)(temporary, { force: true });
+            throw e;
+        }
+        subject.stored += size;
+        return 'created';
+    }
+    async get(principal, snapshot) {
+        if (!bundle_1.HASH_PATTERN.test(snapshot)) {
+            return undefined;
+        }
+        const key = subjectKey(principal);
+        const subject = await this.subject(key);
+        const stored = subject.manifests.get(snapshot);
+        return stored ? this.statusOf(key, snapshot, stored) : undefined;
+    }
+    async list(principal) {
+        const subject = await this.subject(subjectKey(principal));
+        const snapshots = [...subject.manifests.entries()]
+            .map(([snapshot, stored]) => ({
+            snapshot,
+            files: stored.manifest.files.length,
+            bytes: stored.manifest.files.reduce((sum, file) => sum + file.size, 0),
+            createdAt: stored.createdAt,
+        }))
+            .sort((a, b) => b.createdAt.localeCompare(a.createdAt));
+        return { snapshots, bytes: subject.stored, limit: this.limits.subjectBytes };
+    }
+    // Forgets a tree and the content nothing else of the caller's needs.
+    async remove(principal, snapshot) {
+        if (!bundle_1.HASH_PATTERN.test(snapshot)) {
+            return false;
+        }
+        const key = subjectKey(principal);
+        const subject = await this.subject(key);
+        if (!subject.manifests.delete(snapshot)) {
+            return false;
+        }
+        await (0, promises_1.rm)((0, path_1.join)(this.directory, key, 'manifests', `${snapshot.slice('sha256:'.length)}.json`), { force: true });
+        const kept = new Map();
+        for (const stored of subject.manifests.values()) {
+            for (const file of stored.manifest.files) {
+                kept.set(file.sha256, file.size);
+            }
+        }
+        for (const [digest, size] of subject.wanted) {
+            if (!kept.has(digest)) {
+                subject.wanted.delete(digest);
+                if (await exists(this.blob(key, digest))) {
+                    await (0, promises_1.rm)(this.blob(key, digest), { force: true });
+                    subject.stored -= size;
+                }
+            }
+        }
+        return true;
+    }
+    async statusOf(key, snapshot, stored) {
+        const missing = [];
+        for (const digest of new Set(stored.manifest.files.map((file) => file.sha256))) {
+            if (!(await exists(this.blob(key, digest)))) {
+                missing.push(digest);
+            }
+        }
+        return {
+            snapshot,
+            files: stored.manifest.files.length,
+            bytes: stored.manifest.files.reduce((sum, file) => sum + file.size, 0),
+            createdAt: stored.createdAt,
+            missing,
+        };
+    }
+    blob(key, digest) {
+        return (0, path_1.join)(this.directory, key, 'blobs', digest);
+    }
+    async write(file, value) {
+        const temporary = `${file}.${(0, crypto_2.randomBytes)(8).toString('hex')}.tmp`;
+        await (0, promises_1.writeFile)(temporary, JSON.stringify(value), { flag: 'wx' });
+        await (0, promises_1.rename)(temporary, file);
+    }
+    // Reads what is kept for one caller the first time they are seen.
+    subject(key) {
+        let loaded = this.subjects.get(key);
+        if (!loaded) {
+            loaded = this.load(key);
+            this.subjects.set(key, loaded);
+        }
+        return loaded;
+    }
+    async load(key) {
+        const subject = { wanted: new Map(), stored: 0, manifests: new Map() };
+        const manifests = (0, path_1.join)(this.directory, key, 'manifests');
+        let names = [];
+        try {
+            names = await (0, promises_1.readdir)(manifests);
+        }
+        catch {
+            return subject;
+        }
+        for (const name of names.filter((n) => n.endsWith('.json'))) {
+            const stored = JSON.parse(await (0, promises_1.readFile)((0, path_1.join)(manifests, name), 'utf8'));
+            subject.manifests.set((0, source_manifest_1.hashSource)(stored.manifest), stored);
+            for (const file of stored.manifest.files) {
+                subject.wanted.set(file.sha256, file.size);
+            }
+        }
+        for (const [digest, size] of subject.wanted) {
+            try {
+                await (0, promises_1.stat)(this.blob(key, digest));
+                subject.stored += size;
+            }
+            catch {
+                // announced, not sent yet
+            }
+        }
+        return subject;
+    }
+}
+exports.SourceStore = SourceStore;
+async function exists(file) {
+    try {
+        await (0, promises_1.access)(file);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+//# sourceMappingURL=sources.js.map

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "hammerkit",
   "description": "Containerized build tool with incremental caching that runs the same build file locally, in Docker and on Kubernetes.",
-  "version": "1.23.0",
+  "version": "1.25.0",
   "keywords": [
     "build",
     "build-tool",