hammerkit 1.21.2 → 1.23.0

package/build.schema.json

@@ -204,7 +204,10 @@
                       "properties": {
                         "from": {
                           "type": "string",
-                          "pattern": "^(env|file):.+$"
+                          "pattern": "^[A-Za-z][A-Za-z0-9_-]*:.+$"
+                        },
+                        "account": {
+                          "type": "string"
                         },
                         "env": {
                           "type": "string",
@@ -221,9 +224,12 @@
                         "from"
                       ],
                       "additionalProperties": false,
-                      "description": "secret from env:NAME or file:path, injected as env or a read-only file"
+                      "description": "secret from env:NAME, file:path or <provider>:<ref>, injected as env or a read-only file"
                     }
                   },
+                  "account": {
+                    "type": "string"
+                  },
                   "resources": {
                     "type": "object",
                     "properties": {
@@ -319,6 +325,9 @@
                   "secrets": {
                     "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/secrets"
                   },
+                  "account": {
+                    "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/account"
+                  },
                   "resources": {
                     "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/resources"
                   },
@@ -446,6 +455,9 @@
                       "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/secrets/items"
                     }
                   },
+                  "account": {
+                    "type": "string"
+                  },
                   "resources": {
                     "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/resources",
                     "description": "cpu and memory requests and limits"
@@ -748,6 +760,74 @@
             "additionalProperties": false
           }
         },
+        "secretProviders": {
+          "type": "object",
+          "additionalProperties": {
+            "type": "object",
+            "properties": {
+              "command": {
+                "type": "array",
+                "items": {
+                  "type": "string"
+                },
+                "minItems": 1,
+                "description": "command and arguments, no shell; {{ref}} is replaced by the secret reference"
+              },
+              "env": {
+                "type": "object",
+                "additionalProperties": {
+                  "type": "string"
+                },
+                "description": "static environment of the command, next to PATH and the account env"
+              },
+              "timeout": {
+                "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/timeout",
+                "description": "how long the command may run, 30s when omitted"
+              }
+            },
+            "required": [
+              "command"
+            ],
+            "additionalProperties": false,
+            "description": "secret provider, a command that prints the secret to stdout"
+          }
+        },
+        "secretAccounts": {
+          "type": "object",
+          "additionalProperties": {
+            "type": "object",
+            "properties": {
+              "default": {
+                "type": "boolean",
+                "description": "the account for secrets and tasks that name none"
+              },
+              "providers": {
+                "type": "object",
+                "additionalProperties": {
+                  "type": "object",
+                  "properties": {
+                    "env": {
+                      "type": "object",
+                      "additionalProperties": {
+                        "type": "string"
+                      }
+                    }
+                  },
+                  "required": [
+                    "env"
+                  ],
+                  "additionalProperties": false
+                },
+                "description": "per provider, the environment that makes its command act as this account"
+              }
+            },
+            "required": [
+              "providers"
+            ],
+            "additionalProperties": false,
+            "description": "service account the secret providers are read as"
+          }
+        },
         "labels": {
           "$ref": "#/definitions/hammerkit/properties/tasks/additionalProperties/anyOf/0/properties/labels"
         }

package/dist/cli.js

@@ -21,6 +21,7 @@ const work_graph_serializer_1 = require("./planner/work-graph-serializer");
 const dry_run_1 = require("./executer/dry-run");
 const cache_sync_1 = require("./cache/cache-sync");
 const cache_inventory_1 = require("./cache/cache-inventory");
+const prefetch_secrets_1 = require("./secrets/prefetch-secrets");
 const isCliTask = (val) => val.type === 'task';
 exports.isCliTask = isCliTask;
 const isCliService = (val) => val.type === 'service';
@@ -30,6 +31,11 @@ class Cli {
         this.workTree = workTree;
         this.environment = environment;
     }
+    // Starts the run's secrets: the values that make up a task id are fetched
+    // before anything asks for an id, see prefetchSecrets.
+    async prepareSecrets(newRun = false) {
+        await (0, prefetch_secrets_1.prefetchSecrets)(this.workTree, this.environment, newRun);
+    }
     setup(type, options) {
         const logMode = options?.logMode ?? (ci_1.isCI ? 'live' : 'interactive');
         const workTree = (0, reset_work_tree_1.resetWorkTree)(this.workTree, type);
@@ -43,6 +49,9 @@ class Cli {
         return {
             state: processWorkTree,
             start: async () => {
+                if (type !== 'down') {
+                    await this.prepareSecrets(true);
+                }
                 (0, check_for_loop_1.checkForLoop)(workTree);
                 const capacity = (0, plan_work_tasks_1.hasError)(workTree) ? null : await (0, resource_budget_1.getResourceCapacity)(workTree, this.environment);
                 if (capacity && type !== 'down' && (options?.resourceCheck ?? true)) {
@@ -100,15 +109,19 @@ class Cli {
         return await run.start();
     }
     async clean(options) {
+        await this.prepareSecrets();
         await (0, event_cache_1.cleanCache)(this.workTree, this.environment, options);
     }
     async restore(path) {
+        await this.prepareSecrets();
         await (0, event_cache_1.restoreCache)(this.environment, path, this.workTree);
     }
     async package(options) {
+        await this.prepareSecrets();
         await (0, package_1.packageWorkTree)(this.workTree, this.environment, options);
     }
     async store(path) {
+        await this.prepareSecrets();
         await (0, event_cache_1.storeCache)(this.environment, path, this.workTree);
     }
     services() {
@@ -132,6 +145,7 @@ class Cli {
         return [...(0, plan_work_tasks_1.iterateWorkTasks)(this.workTree)].map((task) => (0, describe_task_1.describeTask)(task, this.environment.cwd));
     }
     async explain(options) {
+        await this.prepareSecrets();
         return (0, explain_1.explainWorkTree)(this.workTree, options?.cacheDefault ?? 'checksum', this.environment);
     }
     // Serialize the in-scope work graph for rendering (mermaid/dot). Pure read of
@@ -142,25 +156,30 @@ class Cli {
     // Ordered execution plan with predicted cache decisions, reusing the explain
     // engine — executes nothing, starts nothing, performs no cache push/pull.
     async dryRun(options) {
+        await this.prepareSecrets();
         return (0, dry_run_1.planDryRun)(this.workTree, options?.cacheDefault ?? 'checksum', this.environment);
     }
     // Move cache entries between the tasks' own caches and a named remote without
     // executing anything (`cache pull` / `cache push`).
     async syncCache(options) {
+        await this.prepareSecrets();
         return (0, cache_sync_1.syncCache)(this.workTree, options, this.environment);
     }
     // Entries of a named cache (default: the local `default` cache).
     async listCache(cacheName) {
+        await this.prepareSecrets();
         return (0, cache_inventory_1.listCache)(this.workTree, cacheName, this.environment);
     }
     retentionPolicy(cacheName, overrides) {
         return (0, cache_inventory_1.retentionPolicyOf)(this.workTree, cacheName, overrides);
     }
     async pruneCache(cacheName, policy, dryRun = false) {
+        await this.prepareSecrets();
         return (0, cache_inventory_1.pruneCache)(this.workTree, cacheName, policy, { dryRun }, this.environment);
     }
     // Apply declared retention to the local caches this build file uses.
     async autoPrune() {
+        await this.prepareSecrets();
         return (0, cache_inventory_1.autoPrune)(this.workTree, this.environment);
     }
     task(name) {

package/dist/executer/environment-mock.js

@@ -5,14 +5,17 @@ const work_item_status_1 = require("../planner/work-item-status");
 const get_file_context_1 = require("../file/get-file-context");
 const log_1 = require("../log");
 const empty_writable_1 = require("../utils/empty-writable");
+const redact_1 = require("../utils/redact");
 function environmentMock(cwd) {
+    const secrets = (0, redact_1.createSecretRegistry)();
     return {
         cwd,
         file: (0, get_file_context_1.getFileContext)(cwd),
         console: (0, log_1.consoleContext)((0, empty_writable_1.emptyWritable)()),
         abortCtrl: new AbortController(),
         processEnvs: {},
-        status: (0, work_item_status_1.statusConsole)((0, empty_writable_1.emptyWritable)()),
+        status: (0, work_item_status_1.statusConsole)((0, empty_writable_1.emptyWritable)(), secrets),
+        secrets,
         stdout: (0, empty_writable_1.emptyWritable)(),
         stderr: (0, empty_writable_1.emptyWritable)(),
         stdoutColumns: 80,

package/dist/index.js

@@ -4,6 +4,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const log_1 = require("./log");
 const get_file_context_1 = require("./file/get-file-context");
 const work_item_status_1 = require("./planner/work-item-status");
+const redact_1 = require("./utils/redact");
 const empty_writable_1 = require("./utils/empty-writable");
 const run_program_1 = require("./run-program");
 const abort_on_signals_1 = require("./utils/abort-on-signals");
@@ -19,13 +20,15 @@ process.on('beforeExit', () => {
         process.exitCode = 1;
     }
 });
+const secrets = (0, redact_1.createSecretRegistry)();
 (0, run_program_1.runProgram)({
     cwd: process.cwd(),
     abortCtrl,
     processEnvs: process.env,
     file: (0, get_file_context_1.getFileContext)(process.cwd()),
     console: (0, log_1.consoleContext)(process.stdout),
-    status: (0, work_item_status_1.statusConsole)((0, empty_writable_1.emptyWritable)()),
+    status: (0, work_item_status_1.statusConsole)((0, empty_writable_1.emptyWritable)(), secrets),
+    secrets,
     stdout: process.stdout,
     stderr: process.stderr,
     stdoutColumns: process.stdout.columns,

package/dist/planner/service-definition.js

@@ -26,7 +26,9 @@ function getServiceDefinitionHash(item) {
         src: service.src.map((s) => s.absolutePath),
         healthcheck: service.healthcheck?.cmd.cmd ?? null,
         // where secrets go, never their values: a rotated value needs a `down`
-        secrets: service.secrets.map((s) => `${s.source.type}:${(0, work_secret_1.getSecretName)(s)}`),
+        secrets: service.secrets.map((s) => s.source.type === 'provider'
+            ? `provider:${(0, work_secret_1.getSecretName)(s)}:${(0, work_secret_1.describeSecretSource)(s)}`
+            : `${s.source.type}:${(0, work_secret_1.getSecretName)(s)}`),
         // the init task's id covers its image, commands, envs and mounts: a changed
         // init recreates the service, so it runs again
         init: service.init ? service.init.task.id() : null,

package/dist/planner/utils/append-work-service.js

@@ -114,7 +114,7 @@ function parseService(workTree, service, environment, context) {
             caching,
             // set once the service is planned, see appendServiceInit
             init: null,
-            secrets: (0, work_secret_1.parseWorkSecrets)(service.cwd, context.projectRoot, service.schema.secrets, envs, environment),
+            secrets: (0, work_secret_1.parseWorkSecrets)(service.cwd, context.projectRoot, service.schema.secrets, envs, environment, context.secrets, service.schema.account),
             resources: (0, work_resources_1.parseWorkResources)(service.schema.resources),
         };
     }

package/dist/planner/utils/append-work-task.js

@@ -70,7 +70,7 @@ function parseTask(cwd, task, environment, context) {
         shell: task.schema.shell ? (0, template_value_1.templateValue)(task.schema.shell, envs) : '/bin/sh',
         continuous: task.schema.continuous ?? false,
         timeout: task.schema.timeout ? (0, units_1.parseDuration)(task.schema.timeout) : null,
-        secrets: (0, work_secret_1.parseWorkSecrets)(cwd, context.projectRoot, task.schema.secrets, envs, environment),
+        secrets: (0, work_secret_1.parseWorkSecrets)(cwd, context.projectRoot, task.schema.secrets, envs, environment, context.secrets, task.schema.account),
         resources: (0, work_resources_1.parseWorkResources)(task.schema.resources),
     };
     if ((0, build_file_task_schema_1.isBuildFileContainerTaskSchema)(task.schema)) {

package/dist/planner/work-item-status.js

@@ -15,12 +15,13 @@ function logContext(type, item) {
     };
 }
 exports.logContext = logContext;
-function statusConsole(writable) {
+// `secrets` is the registry of the run (its Environment) whose output this console shows.
+function statusConsole(writable, secrets = (0, redact_1.createSecretRegistry)()) {
     const statusBuffer = new buffer_context_1.BufferContext(StatusBufferMax);
     const consoleBuffer = new buffer_context_1.BufferContext(ConsoleBufferMax);
     const emit = (0, emitter_1.emitter)();
     function addMessage(context, unredacted) {
-        const message = { ...unredacted, message: (0, redact_1.redact)(unredacted.message) };
+        const message = { ...unredacted, message: secrets.redact(unredacted.message) };
         emit.emit(message);
         writable.write(JSON.stringify({ context, message }) + '\n');
         if (message.type === 'console') {

package/dist/planner/work-secret.js

@@ -1,13 +1,24 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getSecretDigest = exports.resolveSecretValue = exports.parseWorkSecrets = exports.getPortableSecretName = exports.getSecretName = void 0;
+exports.getSecretDigest = exports.resolveSecretValue = exports.parseWorkSecrets = exports.getPortableSecretName = exports.getSecretName = exports.describeSecretSource = void 0;
 const crypto_1 = require("crypto");
 const os_1 = require("os");
 const fs_1 = require("fs");
 const normalize_path_1 = require("./utils/normalize-path");
 const template_value_1 = require("./utils/template-value");
 const portable_path_1 = require("./utils/portable-path");
-const redact_1 = require("../utils/redact");
+const abort_1 = require("../executer/abort");
+const provider_values_1 = require("../secrets/provider-values");
+// Where the value comes from, for messages and for what makes a service
+// different: never the value.
+function describeSecretSource(secret) {
+    const source = secret.source;
+    if (source.type === 'provider') {
+        return `${source.provider}:${source.ref} as ${source.account}`;
+    }
+    return source.type === 'env' ? `env:${source.name}` : `file:${source.path}`;
+}
+exports.describeSecretSource = describeSecretSource;
 function getSecretName(secret) {
     return secret.target.type === 'env' ? secret.target.name : secret.target.path;
 }
@@ -18,15 +29,16 @@ function getPortableSecretName(projectRoot, secret) {
     return secret.target.type === 'env' ? secret.target.name : (0, portable_path_1.portablePath)(projectRoot, secret.target.path);
 }
 exports.getPortableSecretName = getPortableSecretName;
-function parseWorkSecrets(cwd, projectRoot, secrets, envs, environment) {
+function parseWorkSecrets(cwd, projectRoot, secrets, envs, environment, catalog, 
+// the service account of the task or service the secrets belong to
+account) {
     return (secrets ?? []).map((secret) => {
         const ref = secret.from.substring(secret.from.indexOf(':') + 1);
-        const source = secret.from.startsWith('env:')
-            ? { type: 'env', name: ref }
-            : { type: 'file', path: (0, normalize_path_1.normalizePath)(cwd, (0, os_1.homedir)(), (0, template_value_1.templateValue)(ref, envs)) };
         const target = secret.env
             ? { type: 'env', name: secret.env }
             : { type: 'file', path: (0, normalize_path_1.normalizePath)(cwd, cwd, (0, template_value_1.templateValue)(secret.path ?? '', envs)) };
+        const name = target.type === 'env' ? target.name : secret.path;
+        const source = parseSecretSource(secret, ref, cwd, envs, catalog, account, `secret ${name}`);
         const workSecret = {
             source,
             target,
@@ -37,20 +49,65 @@ function parseWorkSecrets(cwd, projectRoot, secrets, envs, environment) {
     });
 }
 exports.parseWorkSecrets = parseWorkSecrets;
+function parseSecretSource(secret, ref, cwd, envs, catalog, taskAccount, label) {
+    const kind = secret.from.substring(0, secret.from.indexOf(':'));
+    if (kind === 'env') {
+        return { type: 'env', name: ref };
+    }
+    if (kind === 'file') {
+        return { type: 'file', path: (0, normalize_path_1.normalizePath)(cwd, (0, os_1.homedir)(), (0, template_value_1.templateValue)(ref, envs)) };
+    }
+    const provider = catalog.providers[kind];
+    if (!provider) {
+        const declared = Object.keys(catalog.providers);
+        throw new Error(`${label}: unknown secret provider ${kind}, ${declared.length > 0 ? `declared: ${declared.join(', ')}` : 'none is declared'}`);
+    }
+    const providerRef = (0, template_value_1.templateValue)(ref, envs);
+    if (providerRef.length === 0 || providerRef.startsWith('-')) {
+        throw new Error(`${label}: a reference for ${kind} must not be empty or start with -`);
+    }
+    const accountName = secret.account ?? taskAccount ?? Object.values(catalog.accounts).find((a) => a.default)?.name;
+    if (!accountName) {
+        throw new Error(`${label}: ${kind}:${providerRef} needs an account, and none is named or marked default: true`);
+    }
+    const account = catalog.accounts[accountName];
+    if (!account) {
+        throw new Error(`${label}: unknown secret account ${accountName}, declared: ${Object.keys(catalog.accounts).join(', ') || 'none'}`);
+    }
+    const credentials = account.providers[kind];
+    if (!credentials) {
+        throw new Error(`${label}: account ${accountName} has no credentials for provider ${kind}`);
+    }
+    return {
+        type: 'provider',
+        provider: kind,
+        ref: providerRef,
+        account: accountName,
+        binding: { definition: provider, account: { name: accountName, env: credentials.env } },
+    };
+}
 function readSecretValueSync(secret, environment) {
     if (secret.source.type === 'env') {
         const value = environment.processEnvs[secret.source.name];
         if (value === undefined || value === null) {
             throw new Error(`secret ${getSecretName(secret)}: environment variable ${secret.source.name} is not set`);
         }
-        (0, redact_1.registerSecretValue)(value);
+        environment.secrets.register(value);
+        return value;
+    }
+    if (secret.source.type === 'provider') {
+        const value = (0, provider_values_1.peekProviderValue)(secret.source.binding, secret.source.ref, environment);
+        if (value === undefined) {
+            throw new Error(`secret ${getSecretName(secret)}: ${describeSecretSource(secret)} was not fetched before its task id was computed`);
+        }
+        environment.secrets.register(value);
         return value;
     }
     if (!(0, fs_1.existsSync)(secret.source.path)) {
         throw new Error(`secret ${getSecretName(secret)}: file ${secret.source.path} does not exist`);
     }
     const value = (0, fs_1.readFileSync)(secret.source.path, 'utf8');
-    (0, redact_1.registerSecretValue)(value);
+    environment.secrets.register(value);
     return value;
 }
 // The value, read when the task or service starts (or when a cache-affecting
@@ -61,14 +118,17 @@ async function resolveSecretValue(secret, environment) {
         if (value === undefined || value === null) {
             throw new Error(`secret ${getSecretName(secret)}: environment variable ${secret.source.name} is not set`);
         }
-        (0, redact_1.registerSecretValue)(value);
+        environment.secrets.register(value);
         return value;
     }
+    if (secret.source.type === 'provider') {
+        return resolveProviderSecret(secret, secret.source, environment);
+    }
     if (!(await environment.file.exists(secret.source.path))) {
         throw new Error(`secret ${getSecretName(secret)}: file ${secret.source.path} does not exist`);
     }
     const value = await environment.file.read(secret.source.path);
-    (0, redact_1.registerSecretValue)(value);
+    environment.secrets.register(value);
     return value;
 }
 exports.resolveSecretValue = resolveSecretValue;
@@ -80,4 +140,21 @@ function getSecretDigest(projectRoot, cwd, secret, value) {
     return `sha256:${(0, crypto_1.createHash)('sha256').update(`${salt}\0${value}`).digest('hex')}`;
 }
 exports.getSecretDigest = getSecretDigest;
+async function resolveProviderSecret(secret, source, environment) {
+    let value;
+    try {
+        value = await (0, provider_values_1.fetchProviderValue)(source.binding, source.ref, environment);
+    }
+    catch (error) {
+        if (error instanceof abort_1.AbortError) {
+            throw error;
+        }
+        throw new Error(`secret ${getSecretName(secret)} (${describeSecretSource(secret)}): ${error instanceof Error ? error.message : error}`);
+    }
+    if (value.length === 0) {
+        throw new Error(`secret ${getSecretName(secret)} (${describeSecretSource(secret)}): the provider returned no value`);
+    }
+    environment.secrets.register(value);
+    return value;
+}
 //# sourceMappingURL=work-secret.js.map

package/dist/schema/build-file-base-task-schema.js

@@ -34,6 +34,8 @@ exports.buildFileBaseTaskSchema = (0, zod_1.object)({
     // runs longer. Does not affect the cache key (ADR-0002).
     timeout: duration_schema_1.durationSchema.optional(),
     secrets: (0, zod_1.array)(build_file_secret_schema_1.buildFileSecretSchema).optional(),
+    // the service account its provider secrets are read as, unless a secret names one
+    account: (0, zod_1.string)().optional(),
     // CPU and memory limits of the container; a local task is not limited
     resources: resources_schema_1.resourcesSchema.optional(),
 });

package/dist/schema/build-file-container-service-schema.js

@@ -36,6 +36,8 @@ exports.buildFileContainerServiceSchema = (0, zod_1.object)({
     // passed; the service counts as ready for dependents only after it succeeded
     init: (0, zod_1.string)().describe('task run once the service is healthy, before dependents start').optional(),
     secrets: (0, zod_1.array)(build_file_secret_schema_1.buildFileSecretSchema).optional(),
+    // the service account its provider secrets are read as, unless a secret names one
+    account: (0, zod_1.string)().optional(),
     resources: resources_schema_1.resourcesSchema.optional(),
 })
     .strict()

package/dist/schema/build-file-schema.js

@@ -9,6 +9,7 @@ const build_file_task_schema_1 = require("./build-file-task-schema");
 const labels_schema_1 = require("./labels-schema");
 const cache_schema_1 = require("./cache-schema");
 const build_file_include_schema_1 = require("./build-file-include-schema");
+const build_file_secret_provider_schema_1 = require("./build-file-secret-provider-schema");
 exports.buildFileSchema = (0, zod_1.object)({
     envs: envs_schema_1.envsSchema.optional(),
     tasks: (0, zod_1.record)(build_file_task_schema_1.buildFileTaskSchema).optional(),
@@ -17,6 +18,8 @@ exports.buildFileSchema = (0, zod_1.object)({
     includes: (0, zod_1.record)(build_file_include_schema_1.buildFileIncludeSchema).optional(),
     environments: (0, zod_1.record)(build_file_environment_schema_1.buildFileEnvironmentSchema).optional(),
     caches: (0, zod_1.record)(cache_schema_1.buildFileCacheSchema).optional(),
+    secretProviders: build_file_secret_provider_schema_1.buildFileSecretProvidersSchema.optional(),
+    secretAccounts: build_file_secret_provider_schema_1.buildFileSecretAccountsSchema.optional(),
     labels: labels_schema_1.labelsSchema.optional(),
 })
     .strict()

package/dist/schema/build-file-secret-provider-schema.js

@@ -0,0 +1,36 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.buildFileSecretAccountsSchema = exports.buildFileSecretProvidersSchema = exports.buildFileSecretAccountSchema = exports.buildFileSecretProviderSchema = exports.RESERVED_SECRET_PROVIDERS = void 0;
+const zod_1 = require("zod");
+const duration_schema_1 = require("./duration-schema");
+// `env` and `file` are the built-in sources of `from`, so a provider cannot take
+// either name.
+exports.RESERVED_SECRET_PROVIDERS = ['env', 'file'];
+const nameSchema = (0, zod_1.string)()
+    .regex(/^[A-Za-z][A-Za-z0-9_-]*$/, 'a name starts with a letter and holds letters, digits, - and _')
+    .refine((name) => !exports.RESERVED_SECRET_PROVIDERS.includes(name), 'env and file are built-in secret sources');
+// How to fetch a secret: a command that prints the value to stdout (ADR 0008).
+// The command is an argv list, never run through a shell. `{{ref}}` is replaced
+// by the text after `provider:` in a secret's `from`. It holds no credential:
+// those come from the account the secret is read as.
+exports.buildFileSecretProviderSchema = (0, zod_1.object)({
+    command: (0, zod_1.array)((0, zod_1.string)())
+        .min(1)
+        .describe('command and arguments, no shell; {{ref}} is replaced by the secret reference'),
+    env: (0, zod_1.record)((0, zod_1.string)()).optional().describe('static environment of the command, next to PATH and the account env'),
+    timeout: duration_schema_1.durationSchema.optional().describe('how long the command may run, 30s when omitted'),
+})
+    .strict()
+    .describe('secret provider, a command that prints the secret to stdout');
+// As whom a provider is read: the environment its command runs with. Values
+// are `${HOST_VARIABLE}` references to a credential the machine running
+// hammerkit holds, so a build file never carries one.
+exports.buildFileSecretAccountSchema = (0, zod_1.object)({
+    default: (0, zod_1.boolean)().optional().describe('the account for secrets and tasks that name none'),
+    providers: (0, zod_1.record)(nameSchema, (0, zod_1.object)({ env: (0, zod_1.record)((0, zod_1.string)()) }).strict()).describe('per provider, the environment that makes its command act as this account'),
+})
+    .strict()
+    .describe('service account the secret providers are read as');
+exports.buildFileSecretProvidersSchema = (0, zod_1.record)(nameSchema, exports.buildFileSecretProviderSchema);
+exports.buildFileSecretAccountsSchema = (0, zod_1.record)(nameSchema, exports.buildFileSecretAccountSchema);
+//# sourceMappingURL=build-file-secret-provider-schema.js.map

package/dist/schema/build-file-secret-schema.js

@@ -3,11 +3,14 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.buildFileSecretSchema = void 0;
 const zod_1 = require("zod");
 // A credential handed to a task or service without putting it in the build
-// file, the cache or the logs: `from: env:NAME` or `from: file:path`, injected
+// file, the cache or the logs: `from: env:NAME`, `from: file:path` or
+// `from: <provider>:<ref>` (a secret provider, read as an account), injected
 // as an env variable (`env`) or a read-only file (`path`). `cache: true` makes
 // a salted digest of the value part of the task's cache key (ADR 0002).
 exports.buildFileSecretSchema = (0, zod_1.object)({
-    from: (0, zod_1.string)().regex(/^(env|file):.+$/, 'from must be env:NAME or file:path'),
+    from: (0, zod_1.string)().regex(/^[A-Za-z][A-Za-z0-9_-]*:.+$/, 'from must be env:NAME, file:path or <provider>:<ref>'),
+    // the service account a provider secret is read as, else the task's or service's, else the default
+    account: (0, zod_1.string)().optional(),
     env: (0, zod_1.string)()
         .regex(/^[A-Za-z_][A-Za-z0-9_]*$/)
         .optional(),
@@ -18,5 +21,5 @@ exports.buildFileSecretSchema = (0, zod_1.object)({
     .refine((secret) => (secret.env === undefined) !== (secret.path === undefined), {
     message: 'a secret needs exactly one of env or path',
 })
-    .describe('secret from env:NAME or file:path, injected as env or a read-only file');
+    .describe('secret from env:NAME, file:path or <provider>:<ref>, injected as env or a read-only file');
 //# sourceMappingURL=build-file-secret-schema.js.map

package/dist/schema/reference-parser.js

@@ -9,6 +9,7 @@ const label_values_1 = require("../executer/label-values");
 const merge_environment_variables_1 = require("../environment/merge-environment-variables");
 const resolve_cache_1 = require("../cache/resolve-cache");
 const project_root_1 = require("../planner/project-root");
+const secret_catalog_1 = require("../secrets/secret-catalog");
 function combinePath(namePrefix, name) {
     return namePrefix.length > 0 ? `${namePrefix}:${name}` : name;
 }
@@ -21,6 +22,7 @@ async function parseReferences(ctx, mainScope, environment) {
         envFiles: {},
         environments: {},
         caches: {},
+        secrets: (0, secret_catalog_1.emptySecretCatalog)(),
         projectRoot: await (0, project_root_1.findProjectRoot)(mainScope.cwd, environment),
     };
     for (const file of Object.values(ctx.files)) {
@@ -90,6 +92,7 @@ async function parseReferences(ctx, mainScope, environment) {
         resolveInit(reference, service);
     }
     reference.caches = (0, resolve_cache_1.withBuiltinCaches)(declaredCaches);
+    reference.secrets = (0, secret_catalog_1.collectSecretCatalog)(Object.values(ctx.files), environment, reference);
     return reference;
 }
 exports.parseReferences = parseReferences;

package/dist/secrets/command-secret-provider.js

@@ -0,0 +1,108 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createCommandSecretProvider = void 0;
+const child_process_1 = require("child_process");
+const abort_1 = require("../executer/abort");
+const STDERR_LIMIT = 500;
+// Runs the provider's command and takes its stdout, verbatim, as the value: a
+// key file needs its trailing newline. No shell, so a reference is one argument
+// and never syntax. The command sees PATH, the provider's own `env` and the
+// account's `env` and nothing else of this process: a personal login (`gcloud
+// auth login`, the 1Password app) is never picked up, and a missing credential
+// fails instead of falling back to one.
+function createCommandSecretProvider(binding) {
+    const { definition, account } = binding;
+    return {
+        async resolve(ref, environment) {
+            const [command, ...args] = definition.command.map((part) => expand(part, environment, `provider ${definition.name}`).replace(/\{\{ref\}\}/g, () => ref));
+            const env = {
+                ...minimalEnvironment(environment),
+                ...expandAll(definition.env, environment, `provider ${definition.name}`),
+                ...expandAll(account.env, environment, `account ${account.name}, provider ${definition.name}`),
+            };
+            return run(command, args, env, definition.timeoutMs, environment);
+        },
+    };
+}
+exports.createCommandSecretProvider = createCommandSecretProvider;
+function minimalEnvironment(environment) {
+    const names = process.platform === 'win32' ? ['PATH', 'Path', 'PATHEXT', 'SystemRoot'] : ['PATH'];
+    const result = {};
+    for (const name of names) {
+        const value = environment.processEnvs[name];
+        if (value !== undefined && value !== null) {
+            result[name] = value;
+        }
+    }
+    return result;
+}
+function expandAll(values, environment, owner) {
+    return Object.fromEntries(Object.entries(values).map(([key, value]) => [key, expand(value, environment, owner)]));
+}
+// `${NAME}` from the process environment; one that is not set stops the secret
+// before the command runs.
+function expand(value, environment, owner) {
+    return value.replace(/\$\{([A-Za-z_][A-Za-z0-9_]*)\}/g, (_match, name) => {
+        const resolved = environment.processEnvs[name];
+        if (resolved === undefined || resolved === null || resolved === '') {
+            throw new Error(`${owner}: ${name} is not set`);
+        }
+        return resolved;
+    });
+}
+function run(command, args, env, timeoutMs, environment) {
+    return new Promise((resolve, reject) => {
+        const signal = environment.abortCtrl.signal;
+        if (signal.aborted) {
+            reject(new abort_1.AbortError());
+            return;
+        }
+        const child = (0, child_process_1.spawn)(command, args, { env, shell: false, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] });
+        const stdout = [];
+        const stderr = [];
+        let failure = null;
+        const finish = (error, value) => {
+            clearTimeout(timer);
+            signal.removeEventListener('abort', onAbort);
+            if (error) {
+                reject(error);
+            }
+            else {
+                resolve(value ?? '');
+            }
+        };
+        const stop = (error) => {
+            failure = failure ?? error;
+            child.kill('SIGKILL');
+        };
+        const onAbort = () => stop(new abort_1.AbortError());
+        const timer = setTimeout(() => stop(new Error(`${command} did not finish within ${timeoutMs}ms`)), timeoutMs);
+        signal.addEventListener('abort', onAbort);
+        child.stdout.on('data', (chunk) => stdout.push(chunk));
+        child.stderr.on('data', (chunk) => stderr.push(chunk));
+        child.on('error', (error) => {
+            finish(error.code === 'ENOENT' ? new Error(`${command} was not found on PATH`) : error);
+        });
+        child.on('close', (code) => {
+            if (failure) {
+                finish(failure);
+            }
+            else if (code !== 0) {
+                finish(new Error(`${command} exited with code ${code}${describeStderr(stderr, environment)}`));
+            }
+            else {
+                finish(null, Buffer.concat(stdout).toString('utf8'));
+            }
+        });
+    });
+}
+// What the CLI said, as the first lines of a failure; its own output may hold a
+// value it was asked for, so it is masked like all output.
+function describeStderr(chunks, environment) {
+    const text = environment.secrets.redact(Buffer.concat(chunks).toString('utf8').trim());
+    if (text.length === 0) {
+        return '';
+    }
+    return `: ${text.length > STDERR_LIMIT ? `${text.substring(0, STDERR_LIMIT)}…` : text}`;
+}
+//# sourceMappingURL=command-secret-provider.js.map

package/dist/secrets/prefetch-secrets.js

@@ -0,0 +1,27 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.prefetchSecrets = void 0;
+const plan_work_tasks_1 = require("../planner/utils/plan-work-tasks");
+const work_secret_1 = require("../planner/work-secret");
+const provider_values_1 = require("./provider-values");
+// Fetches the values a task id needs; a run drops the ones of the previous run
+// first (`newRun`), any other command reuses what the run already fetched. A task id is computed synchronously and holds the
+// digest of each `cache: true` secret, so those values must be in memory before
+// anything asks for an id. Secrets that don't affect the id are read when their
+// task starts, and a plan without provider secrets calls no provider.
+async function prefetchSecrets(workTree, environment, newRun = false) {
+    if (newRun) {
+        (0, provider_values_1.beginSecretRun)(environment);
+    }
+    const fetches = [];
+    for (const task of (0, plan_work_tasks_1.iterateWorkTasks)(workTree)) {
+        for (const secret of task.data.secrets) {
+            if (secret.cache && secret.source.type === 'provider') {
+                fetches.push((0, work_secret_1.resolveSecretValue)(secret, environment));
+            }
+        }
+    }
+    await Promise.all(fetches);
+}
+exports.prefetchSecrets = prefetchSecrets;
+//# sourceMappingURL=prefetch-secrets.js.map

package/dist/secrets/provider-values.js

@@ -0,0 +1,47 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.peekProviderValue = exports.fetchProviderValue = exports.beginSecretRun = void 0;
+const secret_provider_1 = require("./secret-provider");
+// The values fetched in the current run, by provider, account and reference.
+// Concurrent requests share one fetch: a secret manager is rate limited (a
+// 1Password service account per token) and several tasks read the same value.
+// Held in memory only, never written anywhere, and dropped when the next run
+// begins so a rotated value is picked up by it.
+const runs = new WeakMap();
+function beginSecretRun(environment) {
+    runs.set(environment, new Map());
+}
+exports.beginSecretRun = beginSecretRun;
+function keyOf(binding, ref) {
+    return [binding.definition.name, binding.account.name, ref].join('\0');
+}
+function fetchProviderValue(binding, ref, environment) {
+    let values = runs.get(environment);
+    if (!values) {
+        values = new Map();
+        runs.set(environment, values);
+    }
+    const key = keyOf(binding, ref);
+    let fetched = values.get(key);
+    if (!fetched) {
+        environment.status
+            .context({ type: 'cli', name: 'hammerkit' })
+            .write('debug', `secret ${ref} via ${binding.definition.name}/${binding.account.name}`);
+        const entry = { promise: (0, secret_provider_1.createSecretProvider)(binding).resolve(ref, environment) };
+        entry.promise = entry.promise.then((value) => {
+            entry.value = value;
+            return value;
+        });
+        fetched = entry;
+        values.set(key, fetched);
+    }
+    return fetched.promise;
+}
+exports.fetchProviderValue = fetchProviderValue;
+// The value of a fetch that has finished in this run, for the sync part of
+// planning (a task id); undefined when it was not fetched, see prefetchSecrets.
+function peekProviderValue(binding, ref, environment) {
+    return runs.get(environment)?.get(keyOf(binding, ref))?.value;
+}
+exports.peekProviderValue = peekProviderValue;
+//# sourceMappingURL=provider-values.js.map

package/dist/secrets/secret-catalog.js

@@ -0,0 +1,78 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.collectSecretCatalog = exports.emptySecretCatalog = void 0;
+const merge_environment_variables_1 = require("../environment/merge-environment-variables");
+const replace_env_variables_1 = require("../environment/replace-env-variables");
+const units_1 = require("../utils/units");
+const DEFAULT_TIMEOUT = '30s';
+function emptySecretCatalog() {
+    return { providers: {}, accounts: {} };
+}
+exports.emptySecretCatalog = emptySecretCatalog;
+const FULL_COMMIT = /^[0-9a-f]{40}$/i;
+// Providers and accounts are global names, like caches, collected from every
+// loaded build file. A name declared twice is an error rather than a silent
+// override: which definition runs with a service account's credential must not
+// depend on include order.
+function collectSecretCatalog(files, environment, context) {
+    const catalog = emptySecretCatalog();
+    for (const file of files) {
+        const providers = Object.entries(file.schema.secretProviders ?? {});
+        const accounts = Object.entries(file.schema.secretAccounts ?? {});
+        if (providers.length === 0 && accounts.length === 0) {
+            continue;
+        }
+        requirePinnedSource(file);
+        const envs = (0, replace_env_variables_1.buildEnvironmentVariables)((0, merge_environment_variables_1.mergeEnvironmentVariables)(file.schema.envs, null), environment, context).variables;
+        for (const [name, provider] of providers) {
+            if (catalog.providers[name]) {
+                throw new Error(`secret provider ${name} is declared in ${catalog.providers[name].declaredIn} and in ${file.fileName}`);
+            }
+            catalog.providers[name] = {
+                name,
+                type: 'command',
+                command: provider.command.map((part) => applyEnvs(part, envs)),
+                env: provider.env ?? {},
+                timeoutMs: (0, units_1.parseDuration)(provider.timeout ?? DEFAULT_TIMEOUT),
+                declaredIn: file.fileName,
+            };
+        }
+        for (const [name, account] of accounts) {
+            if (catalog.accounts[name]) {
+                throw new Error(`secret account ${name} is declared in ${catalog.accounts[name].declaredIn} and in ${file.fileName}`);
+            }
+            catalog.accounts[name] = {
+                name,
+                default: account.default ?? false,
+                providers: account.providers,
+                declaredIn: file.fileName,
+            };
+        }
+    }
+    const defaults = Object.values(catalog.accounts).filter((account) => account.default);
+    if (defaults.length > 1) {
+        throw new Error(`more than one default secret account: ${defaults.map((a) => a.name).join(', ')}`);
+    }
+    for (const account of Object.values(catalog.accounts)) {
+        for (const provider of Object.keys(account.providers)) {
+            if (!catalog.providers[provider]) {
+                throw new Error(`secret account ${account.name} (${account.declaredIn}) names provider ${provider}, which is not declared`);
+            }
+        }
+    }
+    return catalog;
+}
+exports.collectSecretCatalog = collectSecretCatalog;
+// A tag or branch can be moved, and a moved file runs with the credentials of a
+// service account: providers and accounts from a git include are pinned.
+function requirePinnedSource(file) {
+    if (file.remote && !FULL_COMMIT.test(file.remote.ref ?? '')) {
+        throw new Error(`${file.fileName} declares secret providers or accounts, so its git include ${file.remote.git} needs ref to be a full commit SHA`);
+    }
+}
+// `${NAME}` that the build file's own `envs` define; any other reference stays
+// for the host environment.
+function applyEnvs(value, envs) {
+    return value.replace(/\$\{([A-Za-z_][A-Za-z0-9_]*)\}/g, (match, name) => name in envs && !envs[name].includes('$') ? envs[name] : match);
+}
+//# sourceMappingURL=secret-catalog.js.map

package/dist/secrets/secret-provider.js

@@ -0,0 +1,19 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createSecretProvider = exports.registerSecretProvider = void 0;
+const command_secret_provider_1 = require("./command-secret-provider");
+const factories = new Map();
+function registerSecretProvider(type, factory) {
+    factories.set(type, factory);
+}
+exports.registerSecretProvider = registerSecretProvider;
+function createSecretProvider(binding) {
+    const factory = factories.get(binding.definition.type);
+    if (!factory) {
+        throw new Error(`unknown secret provider type: ${binding.definition.type}`);
+    }
+    return factory(binding);
+}
+exports.createSecretProvider = createSecretProvider;
+registerSecretProvider('command', command_secret_provider_1.createCommandSecretProvider);
+//# sourceMappingURL=secret-provider.js.map

package/dist/utils/redact.js

@@ -1,29 +1,85 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.redact = exports.registerSecretValue = void 0;
-// Secret values hammerkit has read, masked in everything it logs, including the
-// output it streams from a task or service. Logs are per line, so each line of
-// a multi-line value (a key file) is masked on its own. Values shorter than
-// MIN_LENGTH are left alone: masking them would mangle unrelated output.
+exports.getEncodedForms = exports.createSecretRegistry = void 0;
+// Secret values a run has read, masked in everything it logs, including the
+// output it streams from a task or service. The registry belongs to one run
+// (its Environment): a process that serves several runs must not mask one
+// run's values in another's output, and must not keep them after the run.
+//
+// Logs are per line, so each line of a multi-line value (a key file) is masked
+// on its own. Values shorter than MIN_LENGTH are left alone: masking them would
+// mangle unrelated output. The usual encodings of a value (base64, hex, URL,
+// JSON) are masked too, since a task or tool often prints those instead; an
+// encoding must be MIN_ENCODED_LENGTH long to be matched, for the same reason.
 const MIN_LENGTH = 4;
+const MIN_ENCODED_LENGTH = 8;
 const MASK = '***';
-const values = new Set();
-function registerSecretValue(value) {
-    for (const candidate of [value, ...value.split(/\r?\n/)]) {
-        const trimmed = candidate.trim();
-        if (trimmed.length >= MIN_LENGTH) {
-            values.add(trimmed);
+function createSecretRegistry() {
+    const values = new Set();
+    // longest first, so a value containing another is masked as a whole
+    let sorted = null;
+    function add(candidate, minLength) {
+        if (candidate.length >= minLength && !values.has(candidate)) {
+            values.add(candidate);
+            sorted = null;
         }
     }
+    return {
+        register(value) {
+            // the whole value, then each line of it; encodings are taken of each as it was written
+            for (const base of new Set([value, ...value.split(/\r?\n/)])) {
+                if (base.trim().length < MIN_LENGTH) {
+                    continue;
+                }
+                add(base.trim(), MIN_LENGTH);
+                for (const form of getEncodedForms(base)) {
+                    add(form.value, form.minLength);
+                }
+            }
+        },
+        redact(message) {
+            if (values.size === 0) {
+                return message;
+            }
+            sorted = sorted ?? [...values].sort((a, b) => b.length - a.length);
+            return sorted.reduce((result, value) => result.split(value).join(MASK), message);
+        },
+    };
 }
-exports.registerSecretValue = registerSecretValue;
-function redact(message) {
-    if (values.size === 0) {
-        return message;
+exports.createSecretRegistry = createSecretRegistry;
+// The encodings of a value that are likely to show up in output. Base64 is
+// matched wherever it starts inside a longer base64 string: the value begins
+// at byte offset 0, 1 or 2 of a 3-byte group, and the characters that mix in
+// bytes before or after it are dropped, since they depend on unknown data.
+function getEncodedForms(value) {
+    const bytes = Buffer.from(value, 'utf8');
+    const forms = [];
+    // the value on its own, as a task or tool usually prints it
+    const whole = bytes.toString('base64');
+    forms.push({ value: whole, minLength: MIN_ENCODED_LENGTH });
+    forms.push({ value: whole.replace(/\+/g, '-').replace(/\//g, '_'), minLength: MIN_ENCODED_LENGTH });
+    for (let offset = 0; offset < 3; offset++) {
+        const encoded = Buffer.concat([Buffer.alloc(offset), bytes])
+            .toString('base64')
+            .replace(/=+$/, '');
+        const leading = [0, 2, 3][offset];
+        const trailing = (bytes.length + offset) % 3 === 0 ? 0 : 1;
+        const form = encoded.substring(leading, encoded.length - trailing);
+        forms.push({ value: form, minLength: MIN_ENCODED_LENGTH });
+        forms.push({ value: form.replace(/\+/g, '-').replace(/\//g, '_'), minLength: MIN_ENCODED_LENGTH });
     }
-    // longest first, so a value containing another is masked as a whole
-    const sorted = [...values].sort((a, b) => b.length - a.length);
-    return sorted.reduce((result, value) => result.split(value).join(MASK), message);
+    const hex = bytes.toString('hex');
+    forms.push({ value: hex, minLength: MIN_ENCODED_LENGTH });
+    forms.push({ value: hex.toUpperCase(), minLength: MIN_ENCODED_LENGTH });
+    const url = encodeURIComponent(value);
+    if (url !== value) {
+        forms.push({ value: url, minLength: MIN_LENGTH });
+    }
+    const json = JSON.stringify(value).slice(1, -1);
+    if (json !== value) {
+        forms.push({ value: json, minLength: MIN_LENGTH });
+    }
+    return forms;
 }
-exports.redact = redact;
+exports.getEncodedForms = getEncodedForms;
 //# sourceMappingURL=redact.js.map

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "hammerkit",
   "description": "Containerized build tool with incremental caching that runs the same build file locally, in Docker and on Kubernetes.",
-  "version": "1.21.2",
+  "version": "1.23.0",
   "keywords": [
     "build",
     "build-tool",