hammerkit 1.13.1 → 1.14.0

package/build.schema.json

@@ -501,13 +501,44 @@
         "references": {
           "type": "object",
           "additionalProperties": {
-            "type": "string"
+            "anyOf": [
+              {
+                "type": "string"
+              },
+              {
+                "type": "object",
+                "properties": {
+                  "git": {
+                    "type": "string",
+                    "minLength": 1,
+                    "description": "Git repository URL or path"
+                  },
+                  "ref": {
+                    "type": "string",
+                    "minLength": 1,
+                    "description": "Branch, tag or commit SHA, the repository HEAD when omitted"
+                  },
+                  "path": {
+                    "type": "string",
+                    "description": "Build file or directory within the repository"
+                  },
+                  "with": {
+                    "$ref": "#/definitions/hammerkit/properties/envs",
+                    "description": "Environment values for the current build task\nhttps://hammerkit.dev/docs/build-file/environment-variables"
+                  }
+                },
+                "required": [
+                  "git"
+                ],
+                "additionalProperties": false
+              }
+            ]
           }
         },
         "includes": {
           "type": "object",
           "additionalProperties": {
-            "type": "string"
+            "$ref": "#/definitions/hammerkit/properties/references/additionalProperties"
           }
         },
         "environments": {

package/dist/includes/resolve-git-source.js

@@ -0,0 +1,113 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isWithin = exports.resolveGitSource = exports.describeGitSource = exports.getIncludesDirectory = void 0;
+const child_process_1 = require("child_process");
+const crypto_1 = require("crypto");
+const promises_1 = require("fs/promises");
+const path_1 = require("path");
+const util_1 = require("util");
+const get_cache_directory_1 = require("../optimizer/get-cache-directory");
+const execFileAsync = (0, util_1.promisify)(child_process_1.execFile);
+// Anything but plain network and local transports is refused: `ext::` runs a
+// command, and a remote build file can name any repository.
+const ALLOWED_PROTOCOLS = 'file:git:http:https:ssh';
+function getIncludesDirectory() {
+    return (0, path_1.join)((0, get_cache_directory_1.getHammerkitDirectory)(), 'includes');
+}
+exports.getIncludesDirectory = getIncludesDirectory;
+function getSourceKey(source) {
+    return (0, crypto_1.createHash)('sha256')
+        .update(`${source.git}\n${source.ref ?? ''}`)
+        .digest('hex')
+        .slice(0, 16);
+}
+function describeGitSource(source) {
+    return `${source.git} at ${source.ref ?? 'HEAD'}`;
+}
+exports.describeGitSource = describeGitSource;
+// The cache is the pin (ADR-0004): a source that was resolved once is served from
+// disk, offline too, until it is refreshed or purged. Only a source that was
+// never resolved needs the network.
+async function resolveGitSource(source, environment) {
+    const directory = (0, path_1.join)(getIncludesDirectory(), getSourceKey(source));
+    const cached = await readCached(directory);
+    if (cached) {
+        return cached;
+    }
+    await (0, promises_1.mkdir)(getIncludesDirectory(), { recursive: true });
+    const staging = `${directory}.${(0, crypto_1.randomUUID)()}`;
+    try {
+        await (0, promises_1.mkdir)(staging, { recursive: true });
+        const commit = await fetchInto((0, path_1.join)(staging, 'repo'), source, environment);
+        const stored = { git: source.git, ref: source.ref ?? null, commit };
+        await (0, promises_1.writeFile)((0, path_1.join)(staging, 'source.json'), JSON.stringify(stored, null, 2));
+        try {
+            await (0, promises_1.rename)(staging, directory);
+        }
+        catch (e) {
+            // a parallel run resolved the same source first; both hold the same ref
+            const winner = await readCached(directory);
+            if (!winner) {
+                throw e;
+            }
+        }
+    }
+    finally {
+        await (0, promises_1.rm)(staging, { recursive: true, force: true });
+    }
+    const resolved = await readCached(directory);
+    if (!resolved) {
+        throw new Error(`unable to cache ${describeGitSource(source)}`);
+    }
+    return resolved;
+}
+exports.resolveGitSource = resolveGitSource;
+async function readCached(directory) {
+    try {
+        const stored = JSON.parse(await (0, promises_1.readFile)((0, path_1.join)(directory, 'source.json'), 'utf8'));
+        return { git: stored.git, ref: stored.ref, commit: stored.commit, root: (0, path_1.join)(directory, 'repo') };
+    }
+    catch {
+        return null;
+    }
+}
+async function fetchInto(repo, source, environment) {
+    await git(['init', '--quiet', repo], source, environment);
+    await git(['-C', repo, 'fetch', '--quiet', '--depth', '1', '--', source.git, source.ref ?? 'HEAD'], source, environment);
+    await git(['-C', repo, '-c', 'advice.detachedHead=false', 'checkout', '--quiet', 'FETCH_HEAD'], source, environment);
+    const { stdout } = await git(['-C', repo, 'rev-parse', 'HEAD'], source, environment);
+    return stdout.trim();
+}
+async function git(args, source, environment) {
+    const env = {};
+    for (const [key, value] of Object.entries(environment.processEnvs)) {
+        if (value !== undefined) {
+            env[key] = value;
+        }
+    }
+    // credentials come from the user's git configuration and helpers; there is no
+    // interactive prompt a build could hang on
+    env.GIT_TERMINAL_PROMPT = '0';
+    env.GIT_ALLOW_PROTOCOL = ALLOWED_PROTOCOLS;
+    try {
+        return await execFileAsync('git', args, { env, signal: environment.abortCtrl.signal });
+    }
+    catch (e) {
+        const error = e;
+        if (error.code === 'ENOENT') {
+            throw new Error(`unable to resolve ${describeGitSource(source)}: git is not installed`);
+        }
+        const reason = (error.stderr ?? '')
+            .split('\n')
+            .map((line) => line.trim())
+            .filter((line) => line.length > 0)
+            .pop() ?? error.message;
+        throw new Error(`unable to resolve ${describeGitSource(source)}: ${reason}`);
+    }
+}
+function isWithin(root, path) {
+    const rel = (0, path_1.relative)(root, path);
+    return !rel.startsWith('..') && !(0, path_1.isAbsolute)(rel);
+}
+exports.isWithin = isWithin;
+//# sourceMappingURL=resolve-git-source.js.map

package/dist/schema/build-file-include-schema.js

@@ -0,0 +1,31 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isGitSource = exports.buildFileIncludeSchema = exports.buildFileGitSourceSchema = void 0;
+const zod_1 = require("zod");
+const envs_schema_1 = require("./envs-schema");
+// A repo URL or ref starting with `-` would be read by git as an option
+// (`--upload-pack=…` runs a command), so neither may.
+const notAnOption = (value) => !value.startsWith('-');
+exports.buildFileGitSourceSchema = (0, zod_1.object)({
+    git: (0, zod_1.string)()
+        .min(1)
+        .refine(notAnOption, 'git must be a repository URL or path')
+        .describe('Git repository URL or path'),
+    // branch, tag or commit SHA; the repository's HEAD when omitted
+    ref: (0, zod_1.string)()
+        .min(1)
+        .refine(notAnOption, 'ref must be a branch, tag or commit SHA')
+        .optional()
+        .describe('Branch, tag or commit SHA, the repository HEAD when omitted'),
+    // subpath of the build file or its directory within the repository
+    path: (0, zod_1.string)().optional().describe('Build file or directory within the repository'),
+    // overrides for the top-level `envs` the included file declares as its inputs
+    with: envs_schema_1.envsSchema.optional(),
+}).strict();
+// A string is a local path; an object is a git source (ADR-0004).
+exports.buildFileIncludeSchema = (0, zod_1.union)([(0, zod_1.string)(), exports.buildFileGitSourceSchema]);
+function isGitSource(include) {
+    return typeof include !== 'string';
+}
+exports.isGitSource = isGitSource;
+//# sourceMappingURL=build-file-include-schema.js.map

package/dist/schema/build-file-schema.js

@@ -8,12 +8,13 @@ const build_file_environment_schema_1 = require("./build-file-environment-schema
 const build_file_task_schema_1 = require("./build-file-task-schema");
 const labels_schema_1 = require("./labels-schema");
 const cache_schema_1 = require("./cache-schema");
+const build_file_include_schema_1 = require("./build-file-include-schema");
 exports.buildFileSchema = (0, zod_1.object)({
     envs: envs_schema_1.envsSchema.optional(),
     tasks: (0, zod_1.record)(build_file_task_schema_1.buildFileTaskSchema).optional(),
     services: (0, zod_1.record)(build_file_service_schema_1.buildFileServiceSchema).optional(),
-    references: (0, zod_1.record)((0, zod_1.string)()).optional(),
-    includes: (0, zod_1.record)((0, zod_1.string)()).optional(),
+    references: (0, zod_1.record)(build_file_include_schema_1.buildFileIncludeSchema).optional(),
+    includes: (0, zod_1.record)(build_file_include_schema_1.buildFileIncludeSchema).optional(),
     environments: (0, zod_1.record)(build_file_environment_schema_1.buildFileEnvironmentSchema).optional(),
     caches: (0, zod_1.record)(cache_schema_1.buildFileCacheSchema).optional(),
     labels: labels_schema_1.labelsSchema.optional(),

package/dist/schema/schema-parser.js

@@ -6,6 +6,8 @@ const read_build_file_1 = require("../parser/read-build-file");
 const path_1 = require("path");
 const default_build_file_1 = require("../parser/default-build-file");
 const parse_error_1 = require("./parse-error");
+const build_file_include_schema_1 = require("./build-file-include-schema");
+const resolve_git_source_1 = require("../includes/resolve-git-source");
 async function createParseContext(fileName, environment) {
     const ctx = {
         files: {},
@@ -14,9 +16,10 @@ async function createParseContext(fileName, environment) {
     return { ctx, scope };
 }
 exports.createParseContext = createParseContext;
-async function appendBuildFile(cwd, ctx, environment, fileName, namePrefix) {
+async function appendBuildFile(cwd, ctx, environment, fileName, namePrefix, remote, inputs) {
     const relativeName = namePrefix.join(':');
-    const key = `${cwd}:${fileName}`;
+    // the same file included with other inputs is another set of tasks
+    const key = inputs ? `${cwd}:${fileName}:${JSON.stringify(inputs)}` : `${cwd}:${fileName}`;
     if (ctx.files[key]) {
         return ctx.files[key];
     }
@@ -26,19 +29,18 @@ async function appendBuildFile(cwd, ctx, environment, fileName, namePrefix) {
         const scope = {
             fileName,
             cwd,
-            schema: result.data,
+            schema: inputs ? applyInputs(result.data, inputs, namePrefix, fileName) : result.data,
             namePrefix: relativeName,
+            remote,
             references: {},
         };
         ctx.files[key] = scope;
         if (scope.schema.references) {
             for (const referenceName of Object.keys(scope.schema.references)) {
-                const reference = scope.schema.references[referenceName];
-                const referenceFilename = await (0, default_build_file_1.getBuildFilename)((0, path_1.join)(scope.cwd, reference), environment);
-                const referenceScope = await appendBuildFile((0, path_1.dirname)(referenceFilename), ctx, environment, referenceFilename, [
-                    ...namePrefix,
-                    referenceName,
-                ]);
+                const target = await locateInclude(scope, scope.schema.references[referenceName], 
+                // a remote file's paths resolve within its repository
+                remote ? (0, path_1.dirname)(scope.fileName) : scope.cwd, environment);
+                const referenceScope = await appendBuildFile((0, path_1.dirname)(target.fileName), ctx, environment, target.fileName, [...namePrefix, referenceName], target.remote, target.inputs);
                 if (scope.references[referenceName]) {
                     throw new Error(referenceName + ' already exists');
                 }
@@ -47,9 +49,8 @@ async function appendBuildFile(cwd, ctx, environment, fileName, namePrefix) {
         }
         if (scope.schema.includes) {
             for (const includeName of Object.keys(scope.schema.includes)) {
-                const include = scope.schema.includes[includeName];
-                const includeFilename = await (0, default_build_file_1.getBuildFilename)((0, path_1.join)((0, path_1.dirname)(scope.fileName), include), environment);
-                const includeScope = await appendBuildFile(cwd, ctx, environment, includeFilename, [...namePrefix, includeName]);
+                const target = await locateInclude(scope, scope.schema.includes[includeName], (0, path_1.dirname)(scope.fileName), environment);
+                const includeScope = await appendBuildFile(cwd, ctx, environment, target.fileName, [...namePrefix, includeName], target.remote, target.inputs);
                 if (scope.references[includeName]) {
                     throw new Error(includeName + ' already exists');
                 }
@@ -63,4 +64,35 @@ async function appendBuildFile(cwd, ctx, environment, fileName, namePrefix) {
     }
 }
 exports.appendBuildFile = appendBuildFile;
+// A string is a local path below `base`; an object is a git source, fetched once
+// into the include cache. Inside a cached checkout a local path may not leave it:
+// a remote file contributes definitions from its own repository only.
+async function locateInclude(scope, include, base, environment) {
+    if ((0, build_file_include_schema_1.isGitSource)(include)) {
+        const remote = await (0, resolve_git_source_1.resolveGitSource)(include, environment);
+        const path = (0, path_1.join)(remote.root, include.path ?? '');
+        if (!(0, resolve_git_source_1.isWithin)(remote.root, path)) {
+            throw new Error(`${include.path} in ${include.git} points outside the repository`);
+        }
+        return { fileName: await (0, default_build_file_1.getBuildFilename)(path, environment), remote, inputs: include.with };
+    }
+    const path = (0, path_1.join)(base, include);
+    if (scope.remote && !(0, resolve_git_source_1.isWithin)(scope.remote.root, path)) {
+        throw new Error(`${include} in ${scope.fileName} points outside ${scope.remote.git}`);
+    }
+    return { fileName: await (0, default_build_file_1.getBuildFilename)(path, environment), remote: scope.remote };
+}
+// The top-level `envs` of an included file are its inputs: it declares each one
+// with a default, and `with` replaces the default. A name the file does not
+// declare is rejected, so a typo cannot silently do nothing.
+function applyInputs(schema, inputs, namePrefix, fileName) {
+    const declared = Object.keys(schema.envs ?? {});
+    for (const name of Object.keys(inputs)) {
+        if (!declared.includes(name)) {
+            const available = declared.length > 0 ? `available inputs: ${declared.join(', ')}` : 'it declares no inputs';
+            throw new Error(`unknown input ${name} for ${namePrefix.join(':')} (${fileName}), ${available}`);
+        }
+    }
+    return { ...schema, envs: { ...schema.envs, ...inputs } };
+}
 //# sourceMappingURL=schema-parser.js.map

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "hammerkit",
   "description": "Containerized build tool with incremental caching that runs the same build file locally, in Docker and on Kubernetes.",
-  "version": "1.13.1",
+  "version": "1.14.0",
   "keywords": [
     "build",
     "build-tool",