hammerkit 1.16.0 → 1.18.0

package/dist/cli.js

@@ -16,6 +16,7 @@ const reset_work_tree_1 = require("./executer/reset-work-tree");
 const execute_work_tree_1 = require("./executer/execute-work-tree");
 const package_1 = require("./docker/package");
 const explain_1 = require("./cache/explain");
+const describe_task_1 = require("./planner/describe-task");
 const work_graph_serializer_1 = require("./planner/work-graph-serializer");
 const dry_run_1 = require("./executer/dry-run");
 const cache_sync_1 = require("./cache/cache-sync");
@@ -127,6 +128,9 @@ class Cli {
     }
     // Read-only cache prediction for every task in scope: executes no command,
     // starts no container/service, performs no cache push/pull.
+    explainDefinition() {
+        return [...(0, plan_work_tasks_1.iterateWorkTasks)(this.workTree)].map((task) => (0, describe_task_1.describeTask)(task, this.environment.cwd));
+    }
     async explain(options) {
         return (0, explain_1.explainWorkTree)(this.workTree, options?.cacheDefault ?? 'checksum', this.environment);
     }

package/dist/executer/event-cache.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.cleanCache = exports.storeCache = exports.getArchivePaths = exports.restoreCache = void 0;
+const resolve_git_source_1 = require("../includes/resolve-git-source");
 const remove_orphaned_containers_1 = require("../docker/remove-orphaned-containers");
 const execute_docker_1 = require("./execute-docker");
 const plan_work_tasks_1 = require("../planner/utils/plan-work-tasks");
@@ -92,6 +93,10 @@ async function cleanCache(workTree, environment, options) {
             await service.data.caching.backend.clear(service.id(), environment);
         }
     }
+    if (options?.cache) {
+        // cached remote includes are kept like stored results, so they go with them
+        await (0, resolve_git_source_1.purgeGitSources)();
+    }
     if (workTree.environment.type === 'docker') {
         const removed = await (0, remove_orphaned_containers_1.removeOrphanedContainers)((0, execute_docker_1.getContainerCli)(workTree.environment));
         for (const id of removed) {

package/dist/includes/resolve-git-source.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.isWithin = exports.resolveGitSource = exports.describeGitSource = exports.getIncludesDirectory = void 0;
+exports.isWithin = exports.purgeGitSources = exports.refreshGitSource = exports.resolveGitSource = exports.describeGitSource = exports.getIncludesDirectory = void 0;
 const child_process_1 = require("child_process");
 const crypto_1 = require("crypto");
 const promises_1 = require("fs/promises");
@@ -34,6 +34,34 @@ async function resolveGitSource(source, environment) {
     if (cached) {
         return cached;
     }
+    return fetchIntoCache(source, directory, environment, false);
+}
+exports.resolveGitSource = resolveGitSource;
+// `hammerkit includes pull`: fetch a source again and replace the cached copy
+// with the ref's current commit. The old copy stays when the fetch fails.
+async function refreshGitSource(source, environment) {
+    const directory = (0, path_1.join)(getIncludesDirectory(), getSourceKey(source));
+    const cached = await readCached(directory);
+    const pinned = isCommitSha(source.ref);
+    const resolved = pinned && cached ? cached : await fetchIntoCache(source, directory, environment, cached !== null);
+    return {
+        git: source.git,
+        ref: source.ref ?? null,
+        previous: cached?.commit ?? null,
+        commit: resolved.commit,
+        pinned,
+    };
+}
+exports.refreshGitSource = refreshGitSource;
+// `hammerkit clean --cache`: the next run fetches every source again.
+async function purgeGitSources() {
+    await (0, promises_1.rm)(getIncludesDirectory(), { recursive: true, force: true });
+}
+exports.purgeGitSources = purgeGitSources;
+function isCommitSha(ref) {
+    return ref !== undefined && /^[0-9a-f]{40}$/i.test(ref);
+}
+async function fetchIntoCache(source, directory, environment, replace) {
     await (0, promises_1.mkdir)(getIncludesDirectory(), { recursive: true });
     const staging = `${directory}.${(0, crypto_1.randomUUID)()}`;
     try {
@@ -41,6 +69,9 @@ async function resolveGitSource(source, environment) {
         const commit = await fetchInto((0, path_1.join)(staging, 'repo'), source, environment);
         const stored = { git: source.git, ref: source.ref ?? null, commit };
         await (0, promises_1.writeFile)((0, path_1.join)(staging, 'source.json'), JSON.stringify(stored, null, 2));
+        if (replace) {
+            await (0, promises_1.rm)(directory, { recursive: true, force: true });
+        }
         try {
             await (0, promises_1.rename)(staging, directory);
         }
@@ -61,7 +92,6 @@ async function resolveGitSource(source, environment) {
     }
     return resolved;
 }
-exports.resolveGitSource = resolveGitSource;
 async function readCached(directory) {
     try {
         const stored = JSON.parse(await (0, promises_1.readFile)((0, path_1.join)(directory, 'source.json'), 'utf8'));

package/dist/planner/describe-task.js

@@ -0,0 +1,42 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.describeTask = void 0;
+const path_1 = require("path");
+const work_task_1 = require("./work-task");
+const toPosix = (value) => (path_1.sep === path_1.posix.sep ? value : value.split(path_1.sep).join(path_1.posix.sep));
+function describeTask(item, cwd) {
+    const task = item.data;
+    const remote = task.scope.remote;
+    return {
+        name: item.name,
+        type: (0, work_task_1.isContainerWorkTask)(task) ? 'container' : 'local',
+        description: task.description || null,
+        source: {
+            file: toPosix(remote ? (0, path_1.relative)(remote.root, task.scope.fileName) : (0, path_1.relative)(cwd, task.scope.fileName)),
+            includedAs: task.scope.namePrefix,
+            git: remote?.git ?? null,
+            ref: remote?.ref ?? null,
+            commit: remote?.commit ?? null,
+        },
+        image: (0, work_task_1.isContainerWorkTask)(task) ? task.image : null,
+        shell: task.shell,
+        cmds: task.cmds.map((cmd) => cmd.cmd),
+        src: task.src.map((src) => src.source),
+        generates: task.generates.map((generate) => (0, path_1.relative)(task.cwd, generate.path)),
+        deps: item.deps.map((dep) => dep.name),
+        needs: item.needs.map((need) => need.name),
+        mounts: (0, work_task_1.isContainerWorkTask)(task) ? task.mounts.map((mount) => mount.mount) : [],
+        envs: Object.entries(task.envs.variables).map(([name, value]) => ({
+            name,
+            value,
+            origin: task.envOrigins[name] ?? 'task',
+        })),
+        envReferences: task.envs.replacements.map((replacement) => ({
+            name: replacement.key,
+            from: replacement.name,
+            available: replacement.available,
+        })),
+    };
+}
+exports.describeTask = describeTask;
+//# sourceMappingURL=describe-task.js.map

package/dist/planner/utils/append-work-task.js

@@ -56,6 +56,7 @@ function parseTask(cwd, task, environment, context) {
     const envs = (0, replace_env_variables_1.buildEnvironmentVariables)(task.envs, environment, context);
     const baseWorkNode = {
         envs,
+        envOrigins: getEnvOrigins(task),
         description: (0, template_value_1.templateValue)(task.schema.description || '', envs).trim(),
         name: task.relativeName,
         cwd,
@@ -92,4 +93,22 @@ function parseTask(cwd, task, environment, context) {
         };
     }
 }
+function getEnvOrigins(task) {
+    const origins = {};
+    for (const key of Object.keys(task.envs)) {
+        if (key in (task.schema.envs ?? {})) {
+            origins[key] = 'task';
+        }
+        else if (key in (task.scope.inputs ?? {}) && key in (task.scope.schema.envs ?? {})) {
+            origins[key] = 'input';
+        }
+        else if (key in (task.scope.schema.envs ?? {})) {
+            origins[key] = 'build-file';
+        }
+        else {
+            origins[key] = 'extend';
+        }
+    }
+    return origins;
+}
 //# sourceMappingURL=append-work-task.js.map

package/dist/program.js

@@ -32,6 +32,51 @@ async function createCli(fileName, environment, workScope) {
     return (0, cli_1.getCli)(workTree, environment);
 }
 exports.createCli = createCli;
+const ENV_ORIGINS = {
+    task: () => 'task',
+    input: (includedAs) => `input of include ${includedAs}`,
+    'build-file': () => 'build file',
+    extend: () => 'extended task',
+};
+function printTaskDefinition(environment, definition) {
+    environment.stdout.write(`• ${definition.name}\n`);
+    const source = definition.source;
+    if (source.git) {
+        (0, log_1.printProperty)(environment, 'defined in', `${source.file} of ${source.git} at ${source.ref ?? 'HEAD'} (commit ${(source.commit ?? '').slice(0, 12)})`);
+    }
+    else {
+        (0, log_1.printProperty)(environment, 'defined in', source.file);
+    }
+    if (source.includedAs) {
+        (0, log_1.printProperty)(environment, 'included as', source.includedAs);
+    }
+    if (definition.description) {
+        (0, log_1.printProperty)(environment, 'description', definition.description);
+    }
+    if (definition.image) {
+        (0, log_1.printProperty)(environment, 'image', definition.image);
+    }
+    for (const [name, values] of [
+        ['deps', definition.deps],
+        ['needs', definition.needs],
+        ['src', definition.src],
+        ['generates', definition.generates],
+        ['mounts', definition.mounts],
+    ]) {
+        if (values.length > 0) {
+            (0, log_1.printProperty)(environment, name, values.join(', '));
+        }
+    }
+    for (const cmd of definition.cmds) {
+        (0, log_1.printProperty)(environment, 'cmd', cmd);
+    }
+    for (const env of definition.envs) {
+        (0, log_1.printProperty)(environment, `env ${env.name}`, `${env.value}  ${colors_1.default.grey(`(${ENV_ORIGINS[env.origin](source.includedAs)})`)}`);
+    }
+    for (const reference of definition.envReferences) {
+        (0, log_1.printProperty)(environment, `env ${reference.name}`, `from $${reference.from} ${colors_1.default.grey(reference.available ? '(set)' : '(not set)')}`);
+    }
+}
 function parseWorkLabelScope(options) {
     const scope = {
         environmentName: null,
@@ -278,9 +323,22 @@ async function getProgram(environment, argv) {
             .choices(['checksum', 'modify-date', 'none']))
             .addOption(new commander_1.Option('--json', 'emit the explanation as JSON').default(false))
             .addOption(new commander_1.Option('--check', 'exit with code 1 when any task would be a cache miss').default(false))
+            .addOption(new commander_1.Option('--definition', 'show the resolved definition of the tasks and where it came from instead').default(false))
             .action(async (task, options) => {
             try {
                 const cli = await createCli(fileName, environment, task ? { taskName: task, environmentName: options.env ?? null } : parseWorkLabelScope(options));
+                if (options.definition) {
+                    const definitions = cli.explainDefinition();
+                    if (options.json) {
+                        environment.stdout.write(`${JSON.stringify(definitions, null, 2)}\n`);
+                    }
+                    else {
+                        for (const definition of definitions) {
+                            printTaskDefinition(environment, definition);
+                        }
+                    }
+                    return;
+                }
                 const explanations = await cli.explain({ cacheDefault: options.cache });
                 const missed = explanations.filter((explanation) => explanation.status === 'miss');
                 if (options.json) {
@@ -313,6 +371,43 @@ async function getProgram(environment, argv) {
                 program.error('Explain was not successful', { exitCode: 1 });
             }
         });
+        program
+            .command('includes')
+            .description('manage remote includes (includes pull)')
+            .command('pull')
+            .description('fetch the git repositories of remote includes again, picking up new commits of branches and tags')
+            .action(async () => {
+            try {
+                const { ctx } = await (0, schema_parser_1.createParseContext)(fileName, environment, { refresh: true });
+                const refreshed = [...(ctx.refreshed?.values() ?? [])];
+                for (const source of refreshed) {
+                    const name = `${source.git} at ${source.ref ?? 'HEAD'}`;
+                    const commit = source.commit.slice(0, 12);
+                    if (source.pinned) {
+                        environment.stdout.write(`${name}: pinned to ${commit}\n`);
+                    }
+                    else if (source.previous === source.commit) {
+                        environment.stdout.write(`${name}: up to date at ${commit}\n`);
+                    }
+                    else if (source.previous === null) {
+                        environment.stdout.write(`${name}: fetched ${commit}\n`);
+                    }
+                    else {
+                        environment.stdout.write(`${name}: ${source.previous.slice(0, 12)} -> ${commit}\n`);
+                    }
+                }
+                if (refreshed.length === 0) {
+                    environment.stdout.write('no remote includes\n');
+                }
+            }
+            catch (e) {
+                if (e instanceof commander_1.CommanderError) {
+                    throw e;
+                }
+                environment.console.error((0, log_1.getErrorMessage)(e));
+                program.error('Includes pull was not successful', { exitCode: 1 });
+            }
+        });
         const cacheCommand = program.command('cache').description('inspect, prune and move cache entries');
         for (const direction of ['pull', 'push']) {
             cacheCommand

package/dist/schema/schema-parser.js

@@ -8,9 +8,10 @@ const default_build_file_1 = require("../parser/default-build-file");
 const parse_error_1 = require("./parse-error");
 const build_file_include_schema_1 = require("./build-file-include-schema");
 const resolve_git_source_1 = require("../includes/resolve-git-source");
-async function createParseContext(fileName, environment) {
+async function createParseContext(fileName, environment, options) {
     const ctx = {
         files: {},
+        refreshed: options?.refresh ? new Map() : undefined,
     };
     const scope = await appendBuildFile((0, path_1.dirname)(fileName), ctx, environment, fileName, []);
     return { ctx, scope };
@@ -32,12 +33,13 @@ async function appendBuildFile(cwd, ctx, environment, fileName, namePrefix, remo
             schema: inputs ? applyInputs(result.data, inputs, namePrefix, fileName) : result.data,
             namePrefix: relativeName,
             remote,
+            inputs,
             references: {},
         };
         ctx.files[key] = scope;
         if (scope.schema.references) {
             for (const referenceName of Object.keys(scope.schema.references)) {
-                const target = await locateInclude(scope, scope.schema.references[referenceName], 
+                const target = await locateInclude(ctx, scope, scope.schema.references[referenceName], 
                 // a remote file's paths resolve within its repository
                 remote ? (0, path_1.dirname)(scope.fileName) : scope.cwd, environment);
                 const referenceScope = await appendBuildFile((0, path_1.dirname)(target.fileName), ctx, environment, target.fileName, [...namePrefix, referenceName], target.remote, target.inputs);
@@ -49,7 +51,7 @@ async function appendBuildFile(cwd, ctx, environment, fileName, namePrefix, remo
         }
         if (scope.schema.includes) {
             for (const includeName of Object.keys(scope.schema.includes)) {
-                const target = await locateInclude(scope, scope.schema.includes[includeName], (0, path_1.dirname)(scope.fileName), environment);
+                const target = await locateInclude(ctx, scope, scope.schema.includes[includeName], (0, path_1.dirname)(scope.fileName), environment);
                 const includeScope = await appendBuildFile(cwd, ctx, environment, target.fileName, [...namePrefix, includeName], target.remote, target.inputs);
                 if (scope.references[includeName]) {
                     throw new Error(includeName + ' already exists');
@@ -67,9 +69,9 @@ exports.appendBuildFile = appendBuildFile;
 // A string is a local path below `base`; an object is a git source, fetched once
 // into the include cache. Inside a cached checkout a local path may not leave it:
 // a remote file contributes definitions from its own repository only.
-async function locateInclude(scope, include, base, environment) {
+async function locateInclude(ctx, scope, include, base, environment) {
     if ((0, build_file_include_schema_1.isGitSource)(include)) {
-        const remote = await (0, resolve_git_source_1.resolveGitSource)(include, environment);
+        const remote = await resolveRemote(ctx, include, environment);
         const path = (0, path_1.join)(remote.root, include.path ?? '');
         if (!(0, resolve_git_source_1.isWithin)(remote.root, path)) {
             throw new Error(`${include.path} in ${include.git} points outside the repository`);
@@ -82,6 +84,15 @@ async function locateInclude(scope, include, base, environment) {
     }
     return { fileName: await (0, default_build_file_1.getBuildFilename)(path, environment), remote: scope.remote };
 }
+// `includes pull` fetches each source once, however often the build files name it.
+async function resolveRemote(ctx, source, environment) {
+    const key = `${source.git}\n${source.ref ?? ''}`;
+    if (!ctx.refreshed || ctx.refreshed.has(key)) {
+        return (0, resolve_git_source_1.resolveGitSource)(source, environment);
+    }
+    ctx.refreshed.set(key, await (0, resolve_git_source_1.refreshGitSource)(source, environment));
+    return (0, resolve_git_source_1.resolveGitSource)(source, environment);
+}
 // The top-level `envs` of an included file are its inputs: it declares each one
 // with a default, and `with` replaces the default. A name the file does not
 // declare is rejected, so a typo cannot silently do nothing.

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "hammerkit",
   "description": "Containerized build tool with incremental caching that runs the same build file locally, in Docker and on Kubernetes.",
-  "version": "1.16.0",
+  "version": "1.18.0",
   "keywords": [
     "build",
     "build-tool",