@olivervea/pipelines 2.0.0-beta.7 → 2.0.0-beta.8

package/README.md

@@ -15,7 +15,7 @@ sdk/
   src/index.ts            authoring API (`@olivervea/pipelines`)
   src/core/               models, secrets, config, steps, implement()
   src/builder/            pipeline builder (types + runtime records), Ref<T>
-  src/library/            library steps (script, gitHubSource, imageBuild, helmDeployOverSsh, helmDeployInCluster, composeDeploy), their exported helpers (httpHealthCheck, registryTag, npmDistTag) and SDK models
+  src/library/            library steps (gitHubSource, imageBuild, helmDeployOverSsh, helmDeployInCluster, composeDeploy), their exported helpers (httpHealthCheck, sshSession, checkSpa) and SDK models; public API only
   src/describe/           describe: manifest, checks, mapper recording + source analysis, manifest schema
   src/harness/            harness CLI modes, env contract, envelope, result upload (`@olivervea/pipelines/harness`)
   src/ctx/                the ctx handed to handlers
@@ -93,10 +93,6 @@ export const deployBeta = step({
   timeoutSeconds: 900,                         // optional
   retries: 1,                                  // optional: a failed attempt is retried (fresh pod) up to n times
 });
-
-import testSh from "./scripts/test.sh" with { type: "file" };
-export const test = steps.script({ name: "test", description: "Runs the tests.", image: "…", script: testSh, input: { source: Source }, output: TestReport,
-  env: { TAG: ({ source }, run) => `${run.version}-${source.sha.slice(0, 7)}` } });   // env values may be functions of the inputs
 ```
 
 Rules:
@@ -139,37 +135,26 @@ target: { host: deploy@box }
 - Handlers see `ctx.config` with the output view (defaults applied); a discriminated union narrows as usual.
 - `secretRef(S)` is a field whose value is `{ secret: "<S.name>" }`: it names a secret, never holds its value. The handler reads the value from `ctx.secrets`. A step that receives an entry containing a `secretRef` must declare that secret.
 - Config is per **run**: the run pins a config snapshot at start; continues and step reruns reuse it. Secrets are per attempt.
-- Library steps take **config references** in place of values: `host: TARGET.host`, `target: TARGET.beta`. A reference is a typed proxy (`ConfigRef<T>`, `T` = the field's output type; `Configurable<T> = T | ConfigRef<T>`), so a `number` field or an optional field can't go where a `string` is required. Fields named `kind`, `name`, `description` or `schema` (the entry's own properties) are reached through `$` (`TARGET.$.name`); `TARGET.$` and `TARGET` itself reference the whole entry. The step's `config` gets every referenced entry (`configEntries` in the manifest); `manifestConfig` records each reference as `{"$config": "<entry>.<path>"}` (field names must not contain `.`; literal keys starting with `$` are reserved); `run()` resolves them from `ctx.config` (defaults applied) before doing anything. `describe` and reconcile reject a reference to an entry the step doesn't receive or a field the entry doesn't have.
+- Library steps (and any step's `settings`) take **config references** in place of values: `host: TARGET.host`, `target: TARGET.beta`. A reference is a typed proxy (`ConfigRef<T>`, `T` = the field's output type; `Configurable<T> = T | ConfigRef<T>`), so a `number` field or an optional field can't go where a `string` is required. Fields named `kind`, `name`, `description` or `schema` (the entry's own properties) are reached through `$` (`TARGET.$.name`); `TARGET.$` and `TARGET` itself reference the whole entry. The step's `config` gets every entry referenced in its `settings` (`configEntries` in the manifest); the manifest records each reference as `{"$config": "<entry>.<path>"}` (field names must not contain `.`; literal keys starting with `$` are reserved); the handler resolves them with `resolveConfigRefs(value, ctx.config)` (defaults applied). `describe` and reconcile reject a reference to an entry the step doesn't receive or a field the entry doesn't have.
 - `clusterAccess: true` on a step asks for the instance's step ServiceAccount token (`Kubernetes:StepServiceAccount`; manifest `clusterAccess: true`, omitted otherwise). Without the instance setting the pod gets no token, like every step.
 
-**`steps.script` contract** (for the script):
-
-| Name | What it holds |
-|---|---|
-| `$OLVE_INPUT` | Path to the validated input as JSON. |
-| `INPUT_<KEY>` | One variable per input key. `deployTo` becomes `INPUT_DEPLOY_TO`. Strings are passed raw; everything else is JSON. |
-| `$OLVE_OUTPUT` | Set only when the step has an output model. Write the output JSON to this path. |
-
-In the output JSON, an object of the form `{"$file": "<path>"}` or `{"$dir": "<path>"}` becomes a file reference. The step's declared secrets are environment variables named after the secrets.
-
 ### Library steps
 
-Library steps carry their own implementation and image; they take typed config and use the SDK models `Source`, `BuiltImage`, `PushedImage` and `HelmRelease`. The set grows from real use: a step or an option exists because a pipeline uses it.
+Library steps carry their own implementation and image; they take typed settings and use the SDK models `Source`, `BuiltImage`, `PushedImage` and `HelmRelease`. Each is a function returning `implement(step({ kind, settings, … }), handler)`, built on the public API only (a test enforces it), so a step package outside the SDK is written the same way (see [Reusable steps](#reusable-steps)). The set grows from real use: a step or an option exists because several pipelines use it; what one pipeline needs stays a custom step there.
 
 | Step | Image | Input → output | Config |
 |---|---|---|---|
-| `steps.script` | yours | yours | `script` (embedded), `env` (see above) |
 | `steps.gitHubSource` | `alpine:3.22` (work happens in the harness) | `{ repo: "owner/name", sha }` (plain schemas: needs a mapper) → `Source` | `token?: Secret` (sent as `Authorization: Bearer` when set; unset = anonymous; becomes the step's secret), `apiUrl?` (default `https://api.github.com`) |
-| `steps.imageBuild` | `gcr.io/kaniko-project/executor:debug` | `{ source: Source }` → `BuiltImage { repository, tag, digest, tar }`; with `push`: `PushedImage { repository, tag, digest }` | `repository` (image name; tag = run version, `-attempt<n>` on reruns), `context?` (dir in the source tree; default the root), `dockerfile?` (relative to the context; default `Dockerfile`), `push?` (push to `repository` instead of a tarball), `contentTag?` (with `push`: tag `src-<context hash>`, skip when the registry has it), `cacheRepo?` (kaniko layer cache), `registryMap?` (kaniko `--registry-map`: registry → mirror prefix; default: the homelab pull-through caches, `{}` = off), `stallTimeoutSeconds?` (default 300, 0 = off), `retries?` (default 2), `registryAuth?` (`{ username, password: Secret }`: log in to the `repository` registry; or a config reference to `{ username, password: { secret: NAME } }`, with `NAME` in `secrets`), `insecureRegistry?` (that registry is plain HTTP), `secrets?` (declared secrets a config `registryAuth` may name), `enabled?` (`false` skips the step) |
-| `steps.helmDeployOverSsh` | `alpine:3.22` (+ `apk add openssh-client` at run time) | `{ source: Source, <key>: BuiltImage }` per `images` (`PushedImage` with `fromRegistry`) → `HelmRelease { release, namespace, image }` (`image`: the first image) | `host` (`user@host`), `sshKey: Secret`, `release`, `namespace`, `chart` (dir in the source), `values?` (chart-relative `-f`), `set?`, `images?` (input key → values prefix; default `{ image: "image" }`), `fromRegistry?`, `imageRefs?` (`PushedImage` input key → values prefix: set as `<prefix>.repository`/`.tag`/`.digest`, not pulled; works next to tarball `images`), `deployments?` (default `[release]`), `rolloutTimeoutSeconds?` (default 120), `healthUrl?` (URL or `{ url, attempts?, delaySeconds? }`) |
+| `steps.imageBuild` | `gcr.io/kaniko-project/executor:debug` | `{ source: Source }` → `BuiltImage { repository, tag, digest, tar }`; with `push`: `PushedImage { repository, tag, digest }` | `repository` (image name; tag = run version, `-attempt<n>` on reruns), `context?` (dir in the source tree with the `Dockerfile`; default the root), `push?` (push to `repository` instead of a tarball), `contentTag?` (with `push`: tag `src-<context hash>`, skip when the registry has it), `registryMap?` (kaniko `--registry-map`: registry → mirror prefix; default: the homelab pull-through caches, `{}` = off), `stallTimeoutSeconds?` (default 300, 0 = off), `retries?` (default 2), `registryAuth?` (`{ username, password: Secret }`: log in to the `repository` registry; or a config reference to `{ username, password: { secret: NAME } }`, with `NAME` in `secrets`), `insecureRegistry?` (that registry is plain HTTP), `secrets?` (declared secrets a config `registryAuth` may name), `enabled?` (`false` skips the step) |
+| `steps.helmDeployOverSsh` | `alpine:3.22` (+ `apk add openssh-client` at run time) | `{ source: Source, <key>: BuiltImage }` per `images` (`PushedImage` with `fromRegistry`) → `HelmRelease { release, namespace, image? }` (`image`: the first image; absent without images) | `host` (`user@host`), `sshKey: Secret`, `release`, `namespace`, `chart` (dir in the source), `values?` (chart-relative `-f`), `set?`, `images?` (input key → values prefix; default `{ image: "image" }`; `{}` = the chart alone), `fromRegistry?`, `imageRefs?` (`PushedImage` input key → values prefix: set as `<prefix>.repository`/`.tag`/`.digest`, not pulled; works next to tarball `images`), `deployments?` (default `[release]`, none without images), `rolloutTimeoutSeconds?` (default 120), `healthUrl?` (URL or `{ url, attempts?, delaySeconds? }`) |
 | `steps.helmDeployInCluster` (and the `target` form of `helmDeployOverSsh`) | `alpine/helm:3.16.2` (ssh: `alpine:3.22`) | `{ source: Source, <key>: PushedImage }` per `images`, `<key>: optional(PushedImage)` per `imageRefs` → `HelmRelease` (`image`: `repository:tag@digest` of the first image) | `target` (a `select` case's reference to a `helmTarget()` variant: `{ kind: "inCluster", namespace, release, values, set, url? }` for helmDeployInCluster, the same plus `host` with `kind: "ssh"` for helmDeployOverSsh; `url` is where step pods reach the release, default its in-cluster Service URL, and is passed on in `HelmRelease.url`), `sshKey: Secret` (ssh only), `chart`, `inlineValues?` (objects written as values files after the target's), `images?`, `imageRefs?`, `deployments?` (ssh), `rolloutTimeoutSeconds?` (default 120), `healthPath?`; helmDeployInCluster declares `clusterAccess` |
 | `steps.composeDeploy` | `alpine:3.22` (work happens in the harness) | `{ image: PushedImage }` → `HelmRelease { release: <service>, namespace: "compose", image: <repository>@<digest> }` | `target` (a `select` case's reference to the compose variant of `deployTarget()`: `{ kind: "compose", deployerUrl, service, token: { secret }, healthUrl? }`), `token: Secret` (must be the secret the target names; the step's only secret), `timeoutSeconds?` |
 
 - `gitHubSource` downloads `<apiUrl>/repos/<repo>/tarball/<sha>` (retrying 5xx), strips the top-level directory and outputs it as `tree`.
-- `imageBuild` copies the tree to `/kaniko/build-context` (kaniko wipes other directories between multi-stage builds), builds `--context=/kaniko/build-context[/<context>] --dockerfile=<that>/<dockerfile>` with `--no-push --tar-path --digest-file`, and outputs the tarball. A missing context or Dockerfile fails the step before kaniko starts. `repository` in the output is fully qualified the way a runtime names a loaded image (`x` → `docker.io/library/x`). A **stall watchdog** kills kaniko when it prints nothing for `stallTimeoutSeconds` (a base-image pull can stall on an idle connection at "Unpacking rootfs", and kaniko has no read timeout), failing the attempt; `retries` then starts a fresh attempt. The retry is deliberately a new pod, never a second kaniko in the same container: kaniko unpacks the base image over the container root and does not clean up when killed.
-- **Caches never block a build.** `registryMap` defaults to the homelab pull-through caches (`DEFAULT_REGISTRY_MAP`: `mcr.microsoft.com` → `mcr-cache.ovea.pro`, `index.docker.io` → `hub-cache.ovea.pro`; kaniko under gVisor stalls pulling from mcr.microsoft.com directly). A map you pass replaces the default; `{}` turns mapping off. Before kaniko starts, every mirror host and the `cacheRepo` registry gets a 5 s `GET https://<host>/v2/` (200 or 401 = up); one that doesn't answer is dropped with a log line (`mcr-cache.ovea.pro unreachable; pulling from mcr.microsoft.com directly`, or no `--cache` for the layer cache). After that, kaniko itself copes: it falls back to the original registry when a mirror can't serve an image's manifest (`RetrieveRemoteImage`; `--skip-default-registry-fallback` is never set), a layer-cache lookup error is a cache miss, a cache push error is a warning, and `--skip-push-permission-check` stops its startup check of the cache repo from exiting. A cache that dies mid-download (after the manifest came from it) fails or stalls the attempt; `retries` starts a fresh one, which probes again.
-- **Registry mode** (`push: true`, for images too big for the step's ~1 GiB scratch space and Garage): kaniko runs without `--no-push`/`--tar-path`, pushes to `repository` (it must start with a registry host; anonymous unless `registryAuth`) and the output is a `PushedImage` with the digest from `--digest-file`. With `contentTag`, the tag is `src-` + the first 12 hex of the sha256 of the context directory's deterministic tar (the bytes `ctx.output.dir` would write), computed from the fetched tree; a `HEAD https://<host>/v2/<path>/manifests/<tag>` (answering a Basic or Bearer challenge: anonymous token, or the `registryAuth` login; an anonymous lookup that is denied counts as "not there") that answers a `Docker-Content-Digest` skips kaniko and outputs that digest. With `registryAuth`, the step declares the password secret and writes `$DOCKER_CONFIG/config.json` (`/kaniko/.docker` in the kaniko image) for that registry before kaniko runs, so pushes, base-image and cache pulls from it are logged in. `insecureRegistry` makes the lookups plain HTTP and adds kaniko `--insecure --insecure-registry=<host>`.
-- `helmDeployOverSsh` gets each image into the host's **k3s** containerd: a `BuiltImage` tar is streamed in (`nerdctl --address /run/k3s/containerd/containerd.sock --namespace k8s.io load`), a `PushedImage` (`fromRegistry`) is pulled there (`… pull -q`, so the rollout doesn't wait on the pull). It copies the chart to `/tmp/<release>-helm-<namespace>`, runs `helm upgrade --install` from inside it (values, then `set`, then per image `<prefix>.repository`/`.tag`/`.pullPolicy` — `Never` for tarballs, `IfNotPresent` for registry images — which always win), waits for `kubectl rollout status` of each of `deployments` in order (`deploy/<name>` for a bare name), and with `healthUrl` polls it (5 × 5 s unless given) from the harness. Several images (one chart, two Deployments):
+- `imageBuild` copies the tree to `/kaniko/build-context` (kaniko wipes other directories between multi-stage builds), builds `--context=/kaniko/build-context[/<context>] --dockerfile=<that>/Dockerfile` with `--no-push --tar-path --digest-file`, and outputs the tarball. A missing context or Dockerfile fails the step before kaniko starts. `repository` in the output is fully qualified the way a runtime names a loaded image (`x` → `docker.io/library/x`). A **stall watchdog** kills kaniko when it prints nothing for `stallTimeoutSeconds` (a base-image pull can stall on an idle connection at "Unpacking rootfs", and kaniko has no read timeout), failing the attempt; `retries` then starts a fresh attempt. The retry is deliberately a new pod, never a second kaniko in the same container: kaniko unpacks the base image over the container root and does not clean up when killed.
+- **Caches never block a build.** `registryMap` defaults to the homelab pull-through caches (`DEFAULT_REGISTRY_MAP`: `mcr.microsoft.com` → `mcr-cache.ovea.pro`, `index.docker.io` → `hub-cache.ovea.pro`; kaniko under gVisor stalls pulling from mcr.microsoft.com directly). A map you pass replaces the default; `{}` turns mapping off. Before kaniko starts, every mirror host gets a 5 s `GET https://<host>/v2/` (200 or 401 = up); one that doesn't answer is dropped with a log line (`mcr-cache.ovea.pro unreachable; pulling from mcr.microsoft.com directly`). After that, kaniko itself copes: it falls back to the original registry when a mirror can't serve an image's manifest (`RetrieveRemoteImage`; `--skip-default-registry-fallback` is never set). A cache that dies mid-download (after the manifest came from it) fails or stalls the attempt; `retries` starts a fresh one, which probes again.
+- **Registry mode** (`push: true`, for images too big for the step's ~1 GiB scratch space and Garage): kaniko runs without `--no-push`/`--tar-path`, pushes to `repository` (it must start with a registry host; anonymous unless `registryAuth`) and the output is a `PushedImage` with the digest from `--digest-file`. With `contentTag`, the tag is `src-` + the first 12 hex of the sha256 of the context directory's deterministic tar (the bytes `ctx.output.dir` would write), computed from the fetched tree; a `HEAD https://<host>/v2/<path>/manifests/<tag>` (answering a Basic or Bearer challenge: anonymous token, or the `registryAuth` login; an anonymous lookup that is denied counts as "not there") that answers a `Docker-Content-Digest` skips kaniko and outputs that digest. With `registryAuth`, the step declares the password secret and writes `$DOCKER_CONFIG/config.json` (`/kaniko/.docker` in the kaniko image) for that registry before kaniko runs, so pushes and base-image pulls from it are logged in. `insecureRegistry` makes the lookups plain HTTP and adds kaniko `--insecure --insecure-registry=<host>`.
+- `helmDeployOverSsh` gets each image into the host's **k3s** containerd: a `BuiltImage` tar is streamed in (`nerdctl --address /run/k3s/containerd/containerd.sock --namespace k8s.io load`), a `PushedImage` (`fromRegistry`) is pulled there (`… pull -q`, so the rollout doesn't wait on the pull). It copies the chart to `/tmp/<release>-helm-<namespace>`, runs `helm upgrade --install` from inside it (values, then `set`, then per image `<prefix>.repository`/`.tag`/`.pullPolicy` — `Never` for tarballs, `IfNotPresent` for registry images — which always win), waits for `kubectl rollout status` of each of `deployments` in order (`deploy/<name>` for a bare name), and with `healthUrl` polls it (5 × 5 s unless given) from the harness. `images: {}` deploys the chart alone: the step's only input is `source`, no image is imported and no image flag set, and nothing is waited on unless `deployments` names it. Several images (one chart, two Deployments):
 
   ```ts
   steps.helmDeployOverSsh({ name: "deploy-beta", description: "", host: "oliver@bulwark-m2", sshKey: SSH_PRIVATE_KEY,
@@ -187,13 +172,33 @@ Library steps carry their own implementation and image; they take typed config a
 | `httpHealthCheck(ctx, url, { attempts?, delaySeconds?, headers? })` | GETs `url` until it returns 2xx (redirects followed), retrying network errors and non-2xx statuses; defaults 5 attempts, 5 s apart (v1's deploy-beta loop). Throws the last error when the attempts run out. Runs in the harness, so the image needs no curl. `helmDeployOverSsh`'s `healthUrl` uses it. |
 | `serviceUrl(release)` | `http://<release>.<namespace>.svc.cluster.local` for a `HelmRelease` (charts that name the Service after the release). |
 | `checkSpa(ctx, url)` | `GET <url>/` must reference an `/assets/*.js` bundle (Vite's layout) and the bundle must be served as JavaScript; returns `{ html, bundle }` for app-specific checks. |
-| `sshSession(ctx, { host, key })` | Installs openssh-client (apk), writes the key to `<tmpDir>/ssh/id` (trailing newline, mode 600; host key not verified) and returns `{ run(remote), text(remote), copy(local, remote), shell(remote) }`; `shell` is the quoted ssh command for pipes (`ctx.sh(`printf '%s' "$V" \| ${s.shell("kubectl apply -f -")}`)`). `helmDeployOverSsh` uses it. `shq(s)` is the POSIX quoting it uses. |
+| `sshSession(ctx, { host, key })` | Installs openssh-client (apk), writes the key to `<tmpDir>/ssh/id` (trailing newline, mode 600; host key not verified) and returns `{ run(remote), text(remote), copy(local, remote), shell(remote) }`; `shell` is the quoted ssh command for pipes (`ctx.sh(`printf '%s' "$V" \| ${s.shell("kubectl apply -f -")}`)`). `helmDeployOverSsh` uses it. |
 
 ```ts
 import { httpHealthCheck, implement } from "@olivervea/pipelines";
 export default implement(smoke, async (_, ctx) => { await httpHealthCheck(ctx, "http://app.ns.svc.cluster.local/api/health", { attempts: 10 }); });
 ```
 
+### Reusable steps
+
+A step several pipelines use is a function returning `implement(step(…), handler)`, staged directly: it carries its handler, so it has no `handlers/` file. The library steps are written this way, so a module in a repo or a package of its own works the same.
+
+```ts
+import { implement, resolveConfigRefs, serviceUrl, step, HelmRelease, type Configurable, type LiteralName } from "@olivervea/pipelines";
+
+export function smokeTest<const N extends string>(def: { name: LiteralName<N>; description: string; path: Configurable<string> }) {
+  return implement(
+    step({ name: def.name, description: def.description, kind: "smokeTest", image: "alpine:3.22", input: { release: HelmRelease }, settings: { path: def.path } }),
+    async ({ release }, ctx) => void (await ctx.http.expectOk(`${release.url ?? serviceUrl(release)}${resolveConfigRefs(def.path, ctx.config)}`)),
+  );
+}
+```
+
+- **`kind`**: a label, required for a step that carries its implementation (`custom` is a step with a handler file). Prefix a package's kinds (`acme/smokeTest`).
+- **`settings`**: what the step was set up with, recorded as the step's manifest `config` (so part of its version). JSON values and config references; the step receives every entry its settings reference.
+- **Identity**: `implement` and `step` record the module that called them (from the stack; a tail call can name a module further up, which only adds to what is hashed). The build bundles those modules, minus `pipeline.ts` and `models.ts`, like a handler, and the kind's hash (`kindHashes` in the generated `handlers.ts`) goes into each such step's `handlerHash`. A step that carries its implementation and has no other module (defined and implemented in a contract file) is a build error.
+- `implement` refuses a step that already carries an implementation; a `handlers/*.ts` file for such a step is a describe error.
+
 ### Handlers
 
 ```ts
@@ -259,7 +264,7 @@ const r = await testHandler(deployBeta, { artifact }, { secrets: { SSH_PRIVATE_K
 expect(r).toMatchObject({ status: "succeeded", output: { release: "olve-pipelines" } });   // r.filesDir, r.logs, r.exitCode
 ```
 
-`testHandler` uses the same validation path as a real run. Invalid input gives `status: "failed"` with `kind: "input"`; an invalid return value gives `kind: "output"`. Any part of `ctx` can be replaced; whatever you don't replace is the real implementation. `testHandler` also accepts library steps.
+`testHandler` uses the same validation path as a real run. Invalid input gives `status: "failed"` with `kind: "input"`; an invalid return value gives `kind: "output"`. Any part of `ctx` can be replaced; whatever you don't replace is the real implementation. `testHandler` takes any implementation: a handler module's default export, a library step or a reusable step.
 
 ## Harness contract (service ⇄ step binary)
 
@@ -369,7 +374,7 @@ Exact format: [`manifest.schema.json`](manifest.schema.json) (JSON Schema 2020-1
 - **`apiVersion`** is `"2.0"`.
 - **`harness`** is `{ bun, sdk, sdkHash }`.
 - **`models[name]`** is `{ output, input? }`.
-- **Step `kind`** is `custom` or the library step's name (`imageBuild`, `composeDeploy`, …). To the service it is a label: any non-empty kind is accepted, so a new library step needs no service change.
+- **Step `kind`** is `custom` (a handler file) or the kind of a step that carries its implementation (`imageBuild`, `composeDeploy`, a package's `acme/…`). To the service it is a label: any non-empty kind is accepted, so a new library or package step needs no service change.
   - `output` is the `z.output` view, with `additionalProperties: false` on every object. It validates outputs, the pool and trigger results.
   - `input` is the `z.input` view (defaults optional). Use it for trigger forms and manual input. It is emitted only when it differs from `output` beyond `additionalProperties`.
 - **Step `input`** maps each input key to a mapping:
@@ -381,7 +386,7 @@ Exact format: [`manifest.schema.json`](manifest.schema.json) (JSON Schema 2020-1
 - **`inputTypes`** declares each input key: `{model, optional}`, or `{schema: {output, input?}}` for a plain schema.
 - **`handlerHash`** is the step's implementation identity. A step version can be computed from image + `inputTypes`/model versions + `output` model version + `secrets` + `configEntries` (and those entries' schemas) + `input` mappings + `handlerHash`. Stage placement, `when` and the step's `description` are not part of it.
   - Custom steps: `sha256(canonical {bundle, harness})`. `bundle` hashes the handler bundled on its own, with `pipeline.ts`, `models.ts` and `@olivervea/pipelines` external. It is tree-shaken and includes embedded scripts, so it changes only when that handler's code (or code it uses) changes.
-  - Library steps: `sha256(canonical {library: kind, config, harness})`.
+  - Steps that carry their implementation: `sha256(canonical {library: kind, config, harness, code})`, `code` being the hash of the modules that define and implement steps of that kind (see [Reusable steps](#reusable-steps)).
   - Because `harness` is part of both hashes, a Bun or SDK bump changes every handler.
 - **`stages`** is an array whose items are either `{steps: [Step…]}` (parallel, with a barrier) or `{when: {field: [values]}, stages: [...]}`.
 - **`triggers`**:
@@ -391,13 +396,13 @@ Exact format: [`manifest.schema.json`](manifest.schema.json) (JSON Schema 2020-1
 - **`failureHandlers`** is a list of `{step: Step, steps: [names] | null}`. Each handler's inputs are all FailureContext (`{"ref": "failure"}`).
 - **`secrets`** lists every secret used by steps, polls and webhooks, sorted, as `{name, description}`.
 - **`config`** lists every config entry steps declare, sorted by name, as `{name, description, required, schema}`. `schema` is the `z.input` view (defaults listed); it validates the entry's value in the config document. `required` is false when the schema accepts a missing value. A `secretRef` field is marked `"x-olve-kind": "secretRef"`, with the secret's name as `properties.secret.const`. Omitted when there are no entries.
-- **Step `configEntries`** names the entries the step receives (sorted; omitted when none). It is part of the step's content, like `secrets`. The step key `config` is the library-step configuration.
+- **Step `configEntries`** names the entries the step receives (sorted; omitted when none). It is part of the step's content, like `secrets`. The step key `config` is the step's `settings`.
 
 Describe runs these checks; reconcile should re-run the manifest-level ones:
 
 - step name format and reserved names (`input`, `failure`), and duplicate step names;
 - one schema per model name, and JSON-representable models;
-- missing handler file, and a handler file without a step (or for a library step);
+- missing handler file, a handler file without a step (or for a step that carries its implementation), and a step that carries its implementation without a kind;
 - availability, ambiguity, maybe-available as required, `optional` without a producer, plain schema without a mapper;
 - mapper source analysis and recording, unknown fields in refs, mapper keys that are missing or extra;
 - `when` on an unknown or non-literal field, or with an invalid value;
@@ -413,7 +418,7 @@ olve-build <pipelineDir> --out <dir> [--target bun-linux-x64|bun-linux-x64-musl]
 olve-build <pipelineDir> --hash-only          # print handler hashes (diagnostics)
 ```
 
-1. Pair each `handlers/*.ts` with the step its `implement(step, …)` names, and hash it (see `handlerHash`).
+1. Pair each `handlers/*.ts` with the step its `implement(step, …)` names, and hash it (see `handlerHash`); hash the modules of each kind of step that carries its implementation.
 2. Generate `<pipelineDir>/.olve/gen/handlers.ts` (a registry of literal `import()` paths, plus hashes and harness info) and `entry.ts`. Add `.olve/` to `.gitignore`.
 3. Run `bun build --compile --target=bun-linux-x64 --no-compile-autoload-dotenv --no-compile-autoload-bunfig --bytecode --format=esm` → `<out>/step`. The output is never minified, because mapper source analysis needs real source.
 4. Check that `PT_INTERP` is `/lib64/ld-linux-x86-64.so.2`. On alpine, Bun silently produces a musl binary; the check fails it.

package/manifest.schema.json

@@ -382,7 +382,7 @@
           "additionalProperties": {
             "$ref": "#/$defs/Json"
           },
-          "description": "Library-step configuration (display only; already covered by handlerHash). A config reference is recorded as {\"$config\": \"<entry>.<field>…\"}; the entry is in configEntries.",
+          "description": "The step's settings (display only; already covered by handlerHash). A config reference is recorded as {\"$config\": \"<entry>.<field>…\"}; the entry is in configEntries.",
           "propertyNames": {
             "type": "string"
           },
@@ -402,7 +402,7 @@
           "type": "string"
         },
         "handlerHash": {
-          "description": "Implementation identity: handler bundle (contract files excluded) + harness (bun + sdkHash); for library steps kind + config + harness.",
+          "description": "Implementation identity: handler bundle (contract files excluded) + harness (bun + sdkHash); for a step that carries its implementation kind + config + harness + the hash of the code of its kind.",
           "pattern": "^sha256:[0-9a-f]{64}$",
           "type": "string"
         },
@@ -432,7 +432,7 @@
           "type": "object"
         },
         "kind": {
-          "description": "'custom' = handlers/<name>.ts; others are SDK library steps. A label: the service accepts any kind.",
+          "description": "'custom' = the handler is a handlers/*.ts file; any other kind is a step that carries its implementation (an SDK or package step). A label: the service accepts any kind.",
           "minLength": 1,
           "type": "string"
         },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@olivervea/pipelines",
-  "version": "2.0.0-beta.7",
+  "version": "2.0.0-beta.8",
   "description": "Olve.Pipelines v2 SDK: typed pipeline authoring, the step harness, and the build tool.",
   "type": "module",
   "license": "MIT",

package/src/build/build.ts

@@ -7,8 +7,8 @@ import type { HarnessInfo } from "../util/handler-hash.ts";
 import { writeTarZst } from "../util/tar.ts";
 import { MANIFEST_BEGIN, MANIFEST_END, uploadFile } from "../harness/result.ts";
 import { GLIBC_INTERP, MUSL_INTERP, readInterpreter } from "./elf.ts";
-import { handlerHashes, type HandlerHash } from "./hashes.ts";
-import { harnessInfo, loadProject, relPosix, type Project } from "./project.ts";
+import { handlerHashes, kindHashes, type HandlerHash } from "./hashes.ts";
+import { harnessInfo, loadPipeline, loadProject, relPosix, type Project } from "./project.ts";
 
 export const BUILD_FLAGS = ["--no-compile-autoload-dotenv", "--no-compile-autoload-bunfig", "--bytecode", "--format=esm"] as const;
 export type Target = "bun-linux-x64" | "bun-linux-x64-musl";
@@ -41,7 +41,7 @@ export interface BuildResult {
 }
 
 /** Writes `<dir>/.olve/gen/{handlers,entry}.ts`: the literal-import handler registry + binary entry. */
-export function generateEntry(project: Project, hashes: Record<string, string>, harness: HarnessInfo): string {
+export function generateEntry(project: Project, hashes: Record<string, string>, harness: HarnessInfo, kinds: Record<string, string> = {}): string {
   const gen = join(project.dir, ".olve", "gen");
   mkdirSync(gen, { recursive: true });
   const registry = project.handlers.map(([name, file]) => `  ${JSON.stringify(name)}: () => import(${JSON.stringify(relPosix(gen, file))}),`).join("\n");
@@ -51,6 +51,7 @@ export function generateEntry(project: Project, hashes: Record<string, string>,
       `import type { HandlerRegistry, HarnessInfo } from "@olivervea/pipelines/harness";\n\n` +
       `export const handlers: HandlerRegistry = {\n${registry}\n};\n\n` +
       `export const handlerHashes: Record<string, string> = ${JSON.stringify(hashes, null, 2)};\n\n` +
+      `export const kindHashes: Record<string, string> = ${JSON.stringify(kinds, null, 2)};\n\n` +
       `export const harness: HarnessInfo = ${JSON.stringify(harness, null, 2)};\n`,
   );
   const entry = join(gen, "entry.ts");
@@ -59,8 +60,8 @@ export function generateEntry(project: Project, hashes: Record<string, string>,
     `// GENERATED by olve-build — do not edit. Entry point of the step binary.\n` +
       `import pipeline from ${JSON.stringify(relPosix(gen, project.pipelineFile))};\n` +
       `import { runHarness } from "@olivervea/pipelines/harness";\n` +
-      `import { handlers, handlerHashes, harness } from "./handlers.ts";\n\n` +
-      `process.exit(await runHarness({ pipeline, handlers, handlerHashes, harness }, process.argv.slice(2)));\n`,
+      `import { handlers, handlerHashes, kindHashes, harness } from "./handlers.ts";\n\n` +
+      `process.exit(await runHarness({ pipeline, handlers, handlerHashes, kindHashes, harness }, process.argv.slice(2)));\n`,
   );
   return entry;
 }
@@ -76,7 +77,8 @@ export async function build(dir: string, opts: BuildOptions): Promise<BuildResul
 
   const t0 = performance.now();
   const hashes = await handlerHashes(project, harness);
-  const entry = generateEntry(project, Object.fromEntries(Object.entries(hashes).map(([k, v]) => [k, v.handlerHash])), harness);
+  const kinds = await kindHashes(project, await loadPipeline(project));
+  const entry = generateEntry(project, Object.fromEntries(Object.entries(hashes).map(([k, v]) => [k, v.handlerHash])), harness, kinds);
   log(`olve-build: ${project.handlers.length} handler(s) hashed in ${((performance.now() - t0) / 1000).toFixed(2)}s`);
 
   // 1. compile
@@ -112,7 +114,7 @@ export async function build(dir: string, opts: BuildOptions): Promise<BuildResul
   await writeTarZst(archive, [{ path: "step", type: "file", mode: 0o755, source: binary }]);
   writeFileSync(
     join(out, "build.json"),
-    canonicalJson({ target, flags: BUILD_FLAGS, harness, handlers: hashes, binarySha256: sha256File(binary), archiveSha256: sha256File(archive) }),
+    canonicalJson({ target, flags: BUILD_FLAGS, harness, handlers: hashes, kinds, binarySha256: sha256File(binary), archiveSha256: sha256File(archive) }),
   );
   log(`olve-build: wrote ${archive}`);
 

package/src/build/hashes.ts

@@ -2,8 +2,12 @@
 // (tree-shaken → function-level granularity; embedded scripts are bundle outputs too), then hash the
 // outputs. Contract changes are versioned through the manifest's model/step entries, SDK changes
 // through `harness`; both are combined in composeHandlerHash.
-import { dirname, join, resolve } from "node:path";
+import { dirname, join, relative, resolve } from "node:path";
 import type { BunPlugin } from "bun";
+import { isImplementation } from "../core/handlers.ts";
+import { originsOf } from "../core/origin.ts";
+import { pipelineData } from "../builder/pipeline.ts";
+import { allSteps } from "../harness/main.ts";
 import { composeHandlerHash, type HarnessInfo } from "../util/handler-hash.ts";
 import { sha256Hex } from "../util/hash.ts";
 import type { Project } from "./project.ts";
@@ -33,11 +37,12 @@ function externals(project: Project): BunPlugin {
 }
 
 export async function bundleHash(project: Project, entry: string): Promise<{ hash: string; outputs: string[] }> {
+  const inside = !relative(project.dir, entry).startsWith("..");
   const r = await Bun.build({
     entrypoints: [entry],
     target: "bun",
     format: "esm",
-    root: project.dir,
+    root: inside ? project.dir : dirname(entry),
     naming: { entry: "[dir]/[name].[ext]", chunk: "[dir]/[name].[ext]", asset: "[dir]/[name].[ext]" },
     plugins: [externals(project)],
     throw: false,
@@ -65,3 +70,27 @@ export async function handlerHashes(project: Project, harness: HarnessInfo): Pro
   }
   return out;
 }
+
+/**
+ * Kind → hash of the modules that implement the steps of that kind which carry their implementation
+ * (each module bundled like a handler: the SDK and contract files external).
+ */
+export async function kindHashes(project: Project, pipeline: unknown): Promise<Record<string, string>> {
+  const origins = new Map<string, Set<string>>();
+  for (const s of allSteps(pipelineData(pipeline))) {
+    if (!isImplementation(s)) continue;
+    const files = originsOf(s).filter((f) => !project.contractFiles.includes(f));
+    if (files.length === 0)
+      throw new Error(`step '${s.name}' carries its implementation but is defined in ${project.contractFiles.map((f) => relative(project.dir, f)).join(" or ")}; define and implement it in a module of its own (or a package)`);
+    const set = origins.get(s.kind) ?? new Set<string>();
+    for (const f of files) set.add(f);
+    origins.set(s.kind, set);
+  }
+  const out: Record<string, string> = {};
+  for (const [kind, files] of [...origins].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))) {
+    const parts: string[] = [];
+    for (const f of [...files].sort()) parts.push((await bundleHash(project, f)).hash);
+    out[kind] = "sha256:" + sha256Hex(parts.join("\n"));
+  }
+  return out;
+}

package/src/build/project.ts

@@ -35,6 +35,13 @@ export async function loadProject(dirArg: string): Promise<Project> {
   return { dir, pipelineFile, contractFiles, handlers: await resolveHandlers(hdir, files) };
 }
 
+/** The default export of pipeline.ts. */
+export async function loadPipeline(project: Project): Promise<unknown> {
+  const mod = (await import(project.pipelineFile)) as { default?: unknown };
+  if (!mod.default) throw new ProjectError(`${project.pipelineFile} has no default export (export default pipeline({...})...)`);
+  return mod.default;
+}
+
 /** Pairs each handler file with the step its default export implements (file names don't matter). */
 async function resolveHandlers(hdir: string, files: string[]): Promise<[string, string][]> {
   const byStep = new Map<string, string>();

package/src/core/brands.ts

@@ -2,6 +2,7 @@
 // a dev runner and the user's installed package — still recognise each other's objects.
 export const STEP = Symbol.for("olve.pipelines.step");
 export const IMPLEMENTATION = Symbol.for("olve.pipelines.implementation");
+export const ORIGIN = Symbol.for("olve.pipelines.origin");
 export const SKIPPED = Symbol.for("olve.pipelines.skipped");
 export const PIPELINE = Symbol.for("olve.pipelines.pipeline");
 export const REF_PATH = Symbol.for("olve.pipelines.refPath");
@@ -11,3 +12,4 @@ export declare const ModelBrand: unique symbol;
 export declare const RefBrand: unique symbol;
 export declare const StepBrand: unique symbol;
 export declare const SkippedBrand: unique symbol;
+export declare const ImplementationBrand: unique symbol;

package/src/core/config.ts

@@ -12,7 +12,7 @@ interface RefTarget {
 }
 
 /**
- * A typed reference to a config value (`TARGET.beta.host`), accepted by library steps in place of a
+ * A typed reference to a config value (`TARGET.beta.host`), accepted in a step's settings in place of a
  * literal. It is recorded in the manifest as `{"$config": "target.beta.host"}` and resolved from the
  * run's config when the step runs; the step receives the entry automatically.
  */

package/src/core/handlers.ts

@@ -1,6 +1,7 @@
-import { IMPLEMENTATION, SKIPPED, type SkippedBrand } from "./brands.ts";
+import { IMPLEMENTATION, ORIGIN, SKIPPED, type ImplementationBrand, type SkippedBrand } from "./brands.ts";
 import type { Ctx } from "../ctx/types.ts";
-import type { AnyStep, ConfigOf, LibraryStep, OutputValue, SecretsOf, StepInput } from "./steps.ts";
+import { callerFile, originsOf } from "./origin.ts";
+import type { AnyStep, ConfigOf, OutputValue, SecretsOf, StepInput } from "./steps.ts";
 import type { Awaitable } from "./types.ts";
 
 /** Returned by `ctx.skip(reason)`: the step produced no output on purpose. */
@@ -18,10 +19,13 @@ export function isSkipped(v: unknown): v is Skipped {
 export type HandlerResult<S extends AnyStep> = OutputValue<S> | Skipped;
 export type Handler<S extends AnyStep> = (input: StepInput<S["input"]>, ctx: Ctx<SecretsOf<S>, ConfigOf<S>>) => Awaitable<HandlerResult<S>>;
 
-export interface Implementation<S extends AnyStep = AnyStep> {
+/** A step with its handler. It is itself a step: stage it directly, or default-export it from `handlers/*.ts`. */
+export type Implementation<S extends AnyStep = AnyStep> = S & {
+  readonly [ImplementationBrand]: true;
+  /** The step as defined. */
   readonly step: S;
   readonly handler: Handler<S>;
-}
+};
 
 /** Block-bodied/async returns get no excess-property check; this restores it for handler outputs. */
 type NoExtraFields<R, T> = R extends Skipped | void
@@ -29,14 +33,16 @@ type NoExtraFields<R, T> = R extends Skipped | void
   : { [K in Exclude<keyof R, keyof T>]: `❌ '${K & string}' is not a field of the output model` };
 
 /**
- * Implement a custom step. Input, ctx and return types come from the step definition alone.
- * Default-export the result from `handlers/<step-name>.ts`.
+ * Implement a step. Input, ctx and return types come from the step definition alone. Default-export
+ * the result from a `handlers/*.ts` file, or stage it directly (a reusable step: give it a `kind`).
  */
 export function implement<S extends AnyStep, R extends HandlerResult<S>>(
-  step: S & (S extends LibraryStep ? "❌ library steps already carry their implementation" : unknown),
+  step: S & (S extends { readonly [ImplementationBrand]: true } ? "❌ this step already carries its implementation" : unknown),
   handler: (input: StepInput<S["input"]>, ctx: Ctx<SecretsOf<S>, ConfigOf<S>>) => Awaitable<R & NoExtraFields<R, OutputValue<S>>>,
 ): Implementation<S> {
-  return Object.freeze({ [IMPLEMENTATION]: true, step, handler: handler as unknown as Handler<S> }) as Implementation<S>;
+  if (isImplementation(step)) throw new TypeError(`step '${(step as AnyStep).name}' already carries its implementation`);
+  const origins = [...new Set([...originsOf(step), callerFile()].filter((f): f is string => f !== undefined))];
+  return Object.freeze({ ...step, [IMPLEMENTATION]: true, [ORIGIN]: origins, step, handler: handler as unknown as Handler<S> }) as unknown as Implementation<S>;
 }
 
 export function isImplementation(v: unknown): v is Implementation {

package/src/core/origin.ts

@@ -0,0 +1,20 @@
+import { ORIGIN } from "./brands.ts";
+
+/**
+ * The file that called the function calling this one, from the stack. A tail call drops its caller's
+ * frame, so this can name a module further up; the build only uses it to choose what to hash.
+ */
+export function callerFile(): string | undefined {
+  const files = (new Error().stack ?? "").split("\n").slice(1).map((f) => f.match(FRAME)?.[1]);
+  const own = files[0];
+  const callee = files[1];
+  const file = files.slice(2).find((f) => f && f !== own && f !== callee) ?? undefined;
+  return file?.startsWith("file://") ? new URL(file).pathname : file;
+}
+const FRAME = /\(?((?:file:\/\/)?\/[^()]*?):\d+(?::\d+)?\)?\s*$/;
+
+/** The modules that defined and implemented a step (where `step` and `implement` were called). */
+export function originsOf(v: unknown): string[] {
+  const o = (v as Record<symbol, unknown>)[ORIGIN];
+  return Array.isArray(o) ? o.filter((x): x is string => typeof x === "string") : [];
+}

package/src/core/steps.ts

@@ -1,5 +1,6 @@
 import type { z } from "zod";
-import { STEP, type StepBrand } from "./brands.ts";
+import { ORIGIN, STEP, type StepBrand } from "./brands.ts";
+import { callerFile } from "./origin.ts";
 import type { AnyModel, Infer, Optional } from "./models.ts";
 import type { AnyConfigEntry } from "./config.ts";
 import { configRefsIn } from "./config-refs.ts";
@@ -18,8 +19,12 @@ export type InferEntry<E> =
 /** What a handler receives: the output view of every input entry. */
 export type StepInput<I> = Simplify<{ -readonly [K in keyof I]: InferEntry<I[K]> }>;
 
-/** `custom` steps have a handler file; the others are library steps implemented by the SDK. */
-export type StepKind = "custom" | "script" | "gitHubSource" | "imageBuild" | "helmDeployOverSsh" | "helmDeployInCluster" | "composeDeploy";
+/**
+ * A label for what a step is, shown in the UI. `custom` (the default) is a step whose handler is in
+ * `handlers/`; a step that carries its implementation (`implement(step, …)` staged directly) names its
+ * kind, e.g. `gitHubSource`. Packages prefix theirs (`acme/deploy`).
+ */
+export type StepKind = string;
 
 export interface Step<
   N extends string = string,
@@ -42,17 +47,9 @@ export interface Step<
   readonly retries?: number;
   /** The step pod gets the instance's step ServiceAccount token (`Kubernetes:StepServiceAccount`). */
   readonly clusterAccess?: boolean;
-  /** Library steps carry their own implementation and config. */
-  readonly builtin?: LibraryImplementation;
+  /** How the step is set up, as written in the definition (may hold config references). */
+  readonly settings?: Readonly<Record<string, unknown>>;
 }
-/** Runtime part of a library step (not part of the public contract). */
-export interface LibraryImplementation {
-  /** Config as emitted in the manifest (JSON only; must not contain paths). */
-  readonly manifestConfig: () => Record<string, unknown>;
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  readonly run: (input: any, ctx: any) => Promise<unknown>;
-}
-export type LibraryStep = { readonly builtin: LibraryImplementation };
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 export type AnyStep = Step<string, any, AnyModel | null, any, any>;
 
@@ -89,16 +86,23 @@ export interface StepDef<N extends string, I extends InputSpec, O extends AnyMod
    * (`Kubernetes:StepServiceAccount`), if it grants one. Default false (no token).
    */
   clusterAccess?: boolean;
+  /** A label for the step's kind (default `custom`). */
+  kind?: StepKind;
+  /**
+   * How the step is set up, shown in the manifest and part of the step's identity: the parameters of
+   * a reusable step. JSON values and config references (`target.prod.host`); every config entry it
+   * references is delivered to the step, and `resolveConfigRefs(value, ctx.config)` resolves them.
+   */
+  settings?: Readonly<Record<string, unknown>>;
 }
 
-/** `configFrom`: a library step's definition; the config entries it references are added to the step's `config`. */
-export function makeStep(kind: StepKind, def: Record<string, unknown>, builtin?: LibraryImplementation, configFrom?: unknown): AnyStep {
+function makeStep(def: Record<string, unknown>, origin: string | undefined): AnyStep {
   const config = new Map<string, AnyConfigEntry>();
   for (const e of (def.config as readonly AnyConfigEntry[] | undefined) ?? []) if (!config.has(e.name)) config.set(e.name, e);
-  configRefsIn(configFrom, config);
+  configRefsIn(def.settings, config);
   const s = {
     [STEP]: true,
-    kind,
+    kind: def.kind ?? "custom",
     name: def.name,
     description: def.description,
     image: def.image,
@@ -109,7 +113,8 @@ export function makeStep(kind: StepKind, def: Record<string, unknown>, builtin?:
     ...(def.timeoutSeconds !== undefined ? { timeoutSeconds: def.timeoutSeconds } : {}),
     ...(def.retries !== undefined ? { retries: def.retries } : {}),
     ...(def.clusterAccess === true ? { clusterAccess: true } : {}),
-    ...(builtin ? { builtin } : {}),
+    ...(def.settings !== undefined ? { settings: def.settings } : {}),
+    [ORIGIN]: origin === undefined ? [] : [origin],
   };
   return Object.freeze(s) as unknown as AnyStep;
 }
@@ -118,7 +123,10 @@ export function isStep(v: unknown): v is AnyStep {
   return typeof v === "object" && v !== null && (v as Record<symbol, unknown>)[STEP] === true;
 }
 
-/** Define a custom step: a contract (name, image, inputs, output, secrets). Its handler lives in `handlers/<name>.ts`. */
+/**
+ * Define a step: a contract (name, image, inputs, output, secrets). Its handler is a `handlers/*.ts`
+ * default-exporting `implement(step, …)`, or, for a reusable step, `implement(step, …)` staged directly.
+ */
 export function step<
   const N extends string,
   const I extends InputSpec = {},
@@ -126,5 +134,5 @@ export function step<
   const Sec extends readonly Secret[] = [],
   const Cfg extends readonly AnyConfigEntry[] = [],
 >(def: StepDef<N, I, O, Sec, Cfg>): Step<N, I, O, Sec[number]["name"], Cfg[number]> {
-  return makeStep("custom", def as never) as never;
+  return makeStep(def as never, callerFile()) as never;
 }