@kici-dev/compiler 0.1.16 → 0.1.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (138) hide show
  1. package/dist/auth/headless-detect.js +1 -1
  2. package/dist/cli-banner.js +1 -1
  3. package/dist/cli.js +56 -19
  4. package/dist/commands/approve.js +1 -1
  5. package/dist/commands/compile.js +19 -17
  6. package/dist/commands/diagnostics.d.ts +7 -0
  7. package/dist/commands/diagnostics.js +58 -0
  8. package/dist/commands/docs.js +1 -1
  9. package/dist/commands/drain-worker.js +1 -1
  10. package/dist/commands/endpoints.js +1 -1
  11. package/dist/commands/fixture.js +1 -1
  12. package/dist/commands/held-run-client.js +1 -1
  13. package/dist/commands/held-run-resolve.js +1 -1
  14. package/dist/commands/hook.js +1 -1
  15. package/dist/commands/index.d.ts +16 -4
  16. package/dist/commands/index.js +10 -4
  17. package/dist/commands/init.js +2 -2
  18. package/dist/commands/login.d.ts +0 -2
  19. package/dist/commands/login.js +1 -3
  20. package/dist/commands/logout.js +1 -1
  21. package/dist/commands/orchestrators.d.ts +25 -0
  22. package/dist/commands/orchestrators.js +115 -0
  23. package/dist/commands/org.js +1 -1
  24. package/dist/commands/reject.js +1 -1
  25. package/dist/commands/run.d.ts +2 -9
  26. package/dist/commands/run.js +256 -315
  27. package/dist/commands/runs/cancel.d.ts +6 -0
  28. package/dist/commands/runs/cancel.js +32 -0
  29. package/dist/commands/runs/list.d.ts +6 -0
  30. package/dist/commands/runs/list.js +49 -0
  31. package/dist/commands/runs/logs.d.ts +7 -0
  32. package/dist/commands/runs/logs.js +67 -0
  33. package/dist/commands/runs/rerun.d.ts +5 -0
  34. package/dist/commands/runs/rerun.js +21 -0
  35. package/dist/commands/runs/show.d.ts +5 -0
  36. package/dist/commands/runs/show.js +65 -0
  37. package/dist/commands/secrets-list.d.ts +7 -9
  38. package/dist/commands/secrets-list.js +40 -64
  39. package/dist/commands/test.d.ts +4 -0
  40. package/dist/commands/test.js +4 -6
  41. package/dist/commands/types.d.ts +3 -5
  42. package/dist/commands/types.js +13 -37
  43. package/dist/commands/verify-attestation.d.ts +12 -0
  44. package/dist/commands/verify-attestation.js +69 -0
  45. package/dist/commands/watch.js +1 -1
  46. package/dist/commands/workflows.js +14 -34
  47. package/dist/errors/capability-gap.js +1 -1
  48. package/dist/errors/formatter.js +1 -1
  49. package/dist/errors/index.js +1 -1
  50. package/dist/execution/executor.js +1 -1
  51. package/dist/execution/index.js +1 -1
  52. package/dist/execution/sdk-alias.js +1 -1
  53. package/dist/execution/ts-loader.js +1 -1
  54. package/dist/fixtures/compiler.js +1 -1
  55. package/dist/fixtures/defaults/index.js +1 -1
  56. package/dist/format.js +1 -1
  57. package/dist/generators/secrets-dts.js +2 -2
  58. package/dist/hooks/detector.js +1 -1
  59. package/dist/hooks/index.js +1 -1
  60. package/dist/hooks/installer.js +1 -1
  61. package/dist/hooks/templates.js +1 -1
  62. package/dist/index.js +1 -1
  63. package/dist/llm-context/llms-full.txt +1549 -1282
  64. package/dist/llm-context/llms.txt +6 -2
  65. package/dist/local-executor/dag-scheduler.js +1 -1
  66. package/dist/local-executor/index.js +1 -1
  67. package/dist/local-executor/job-runner.js +40 -18
  68. package/dist/local-executor/materializer.js +1 -1
  69. package/dist/local-executor/output-streamer.js +1 -1
  70. package/dist/local-executor/payload-generator.js +1 -1
  71. package/dist/local-executor/picker.js +1 -1
  72. package/dist/local-executor/runs-on-display.d.ts +9 -0
  73. package/dist/local-executor/runs-on-display.js +23 -0
  74. package/dist/local-executor/secret-loader.js +1 -1
  75. package/dist/local-executor/to-event-payload.js +1 -1
  76. package/dist/local-executor/types.js +1 -1
  77. package/dist/local-executor/workflow-lock.js +0 -0
  78. package/dist/lockfile/generator.d.ts +9 -5
  79. package/dist/lockfile/generator.js +58 -38
  80. package/dist/lockfile/hash-files.js +1 -1
  81. package/dist/lockfile/hasher.js +1 -1
  82. package/dist/lockfile/index.js +1 -1
  83. package/dist/lockfile/purity-analyzer.js +1 -1
  84. package/dist/postinstall.js +1 -1
  85. package/dist/provenance-trust-root.d.ts +14 -0
  86. package/dist/provenance-trust-root.js +49 -0
  87. package/dist/remote/config.d.ts +6 -2
  88. package/dist/remote/config.js +7 -4
  89. package/dist/remote/dashboard-client.d.ts +92 -0
  90. package/dist/remote/dashboard-client.js +168 -0
  91. package/dist/remote/encryption.js +3 -4
  92. package/dist/remote/history.js +1 -1
  93. package/dist/remote/oauth.js +1 -1
  94. package/dist/remote/oidc-discovery.js +1 -1
  95. package/dist/remote/output/json.js +1 -1
  96. package/dist/remote/output/junit.js +1 -1
  97. package/dist/remote/output/streaming.js +1 -1
  98. package/dist/remote/output/summary.js +1 -1
  99. package/dist/remote/platform-client.d.ts +142 -0
  100. package/dist/remote/platform-client.js +187 -0
  101. package/dist/remote/prod-defaults.js +1 -1
  102. package/dist/remote/render.d.ts +11 -0
  103. package/dist/remote/render.js +48 -0
  104. package/dist/remote/secret-upload.js +1 -1
  105. package/dist/remote/uploader.d.ts +9 -0
  106. package/dist/remote/uploader.js +51 -4
  107. package/dist/templates/agents-md.d.ts +1 -1
  108. package/dist/templates/agents-md.js +8 -3
  109. package/dist/templates/index.js +1 -1
  110. package/dist/templates/package-json.js +4 -5
  111. package/dist/templates/tsconfig-json.js +1 -1
  112. package/dist/templates/workflows/hello-world.js +2 -2
  113. package/dist/templates/workflows/hello-world.ts +1 -1
  114. package/dist/templates/workflows/pr-checks.js +3 -3
  115. package/dist/templates/workflows/pr-checks.ts +2 -2
  116. package/dist/test-runner/dry-run.js +1 -1
  117. package/dist/test-runner/event-types.js +1 -1
  118. package/dist/test-runner/git-detector.js +1 -1
  119. package/dist/test-runner/index.js +1 -1
  120. package/dist/test-runner/job-executor.js +3 -2
  121. package/dist/test-runner/output-formatter.js +1 -1
  122. package/dist/test-runner/payload-builder.js +1 -1
  123. package/dist/test-runner/rule-evaluator.js +1 -1
  124. package/dist/test-runner/secrets-file.js +1 -1
  125. package/dist/test-runner/step-context.js +10 -6
  126. package/dist/types.d.ts +29 -8
  127. package/dist/types.js +2 -1
  128. package/dist/validation/index.js +1 -1
  129. package/dist/validation/validator.js +1 -1
  130. package/dist/workflows/hello-world.ts +1 -1
  131. package/dist/workflows/pr-checks.ts +2 -2
  132. package/package.json +10 -9
  133. package/sbom.spdx.json +465 -410
  134. package/dist/commands/cancel.js +0 -124
  135. package/dist/commands/status.js +0 -212
  136. package/dist/remote/client.js +0 -203
  137. package/dist/remote/observer.js +0 -174
  138. /package/dist/{chunk-gOLHoazu.js → chunk-BTugEXQM.js} +0 -0
@@ -26,7 +26,7 @@ Why the lock file must stay in sync with workflow source, how to commit both tog
26
26
 
27
27
  ### [CLI reference](cli-reference.md)
28
28
 
29
- All CLI commands provided by `@kici-dev/compiler`: `kici compile` (with watch mode and check mode), `kici run` (local and remote execution), `kici test` (event simulation with dry-run, filtering, debug output, and custom payloads), `kici login`/`logout`/`org` (authentication and org management), `kici status`/`cancel` (run management), `kici secrets` (secret listing), `kici types` (type generation), `kici fixture` (generate test payloads), `kici init` (interactive project scaffolding), `kici hook` (pre-commit hook installation), `kici endpoints` (webhook entrypoints), and `kici workflows` (workflow listing). Includes environment variables and exit codes.
29
+ All CLI commands provided by `@kici-dev/compiler`: `kici compile` (with watch mode and check mode), `kici run` (local and remote execution), `kici test` (event simulation with dry-run, filtering, debug output, and custom payloads), `kici login`/`logout`/`org` (authentication and org management), `kici diagnostics` (infrastructure tree) and `kici runs` (`list`/`show`/`logs`/`rerun`/`cancel` run management), `kici secrets` (secret listing), `kici types` (type generation), `kici fixture` (generate test payloads), `kici init` (interactive project scaffolding), `kici hook` (pre-commit hook installation), `kici endpoints` (webhook entrypoints), and `kici workflows` (workflow listing). Includes environment variables and exit codes.
30
30
 
31
31
  ### [Workflow patterns](workflow-patterns.md)
32
32
 
@@ -215,6 +215,8 @@ This workflow:
215
215
  - Runs a `lint` job first
216
216
  - Runs a `test` job after lint succeeds (`needs: [lint]`)
217
217
 
218
+ `runsOn` selects which agents may run a job. Every agent self-reports `kici:os:<platform>`, `kici:arch:<cpu>`, and `kici:host:<hostname>`, so `runsOn: 'kici:os:linux'` targets any connected Linux agent with zero configuration — `kici init` scaffolds workflows with exactly that. Use a custom label such as `'linux'` or `'gpu'` (defined in your scaler's `labelSet`) to target a specific pool instead. See the [runsOn forms](./sdk/core.md#runson-forms) reference for the full label model.
219
+
218
220
  **Single-step shortcut.** If a job only has one step, pass `run` directly to `job()` instead of building a `steps: [step(...)]` array:
219
221
 
220
222
  ```typescript
@@ -310,14 +312,14 @@ This updates `.kici/package.json` and generates (or updates) `package-lock.json`
310
312
 
311
313
  ### Dependency resolution contract
312
314
 
313
- Every `.kici/` dependency must be resolvable from the **single cloned repository**. When a job runs, the agent clones only this repository and installs `.kici/` dependencies with your repo's package manager (npm or pnpm; yarn is not supported the agent rejects it with an actionable error). A dependency that points outside the cloned repo cannot be resolved.
315
+ Every `.kici/` dependency must be resolvable from the **single cloned repository**. When a job runs, the agent clones only this repository and installs `.kici/` dependencies with your repo's package manager npm, pnpm, yarn classic (v1), and yarn berry (v2+). A dependency that points outside the cloned repo cannot be resolved.
314
316
 
315
317
  In practice:
316
318
 
317
319
  - **From a registry** — the common case. Pin a published version (a private registry works — see [Private registries](./private-registries.md)). Available for any package manager.
318
- - **From an in-repo workspace sibling** — if your `.kici/` is a member of a **pnpm workspace**, it can depend on a sibling package in the same repo via `workspace:*`. The whole repo is cloned, so the sibling is present and resolves; the agent also builds your `.kici/` dependency closure after install, so a sibling's build output exists before the workflow that imports it loads. A `file:`/`link:`/`portal:` path is allowed only when it stays inside the repository.
320
+ - **From an in-repo workspace sibling** — if your `.kici/` is a member of a **pnpm workspace** or a **yarn berry workspace** (a `workspaces` array in the repo-root `package.json`), it can depend on a sibling package in the same repo via `workspace:*` (yarn berry also accepts `portal:`). The whole repo is cloned, so the sibling is present and resolves; the agent also builds your `.kici/` dependency closure after install, so a sibling's build output exists before the workflow that imports it loads. A `file:`/`link:`/`portal:` path is allowed only when it stays inside the repository.
319
321
 
320
- What fails fast (with an actionable error naming the dependency, not a raw package-manager error): a `workspace:` dependency in an **npm** project (npm has no workspace protocol — pin a published version or switch to pnpm), and any `file:`/`link:`/`portal:` path that points outside the cloned repo.
322
+ What fails fast (with an actionable error naming the dependency, not a raw package-manager error): a `workspace:` dependency in an **npm** project (npm has no workspace protocol — pin a published version or switch to pnpm), a `workspace:`/`portal:` dependency in a **yarn classic** project (v1 has neither — use a version range, pnpm, or yarn berry), a `workspace:` dependency in a **yarn berry** project whose repo-root `package.json` has no `workspaces` array, and any `file:`/`link:`/`portal:` path that points outside the cloned repo.
321
323
 
322
324
  Then use the package in your workflow:
323
325
 
@@ -416,6 +418,8 @@ Or add the flag to your root `package.json`:
416
418
 
417
419
  ## Authoring KiCI workflows with LLM coding agents
418
420
 
421
+ KiCI is LLM-ready by design. Because workflows are real, typed TypeScript, coding agents reason over the SDK's `.d.ts` signatures instead of guessing a bespoke YAML DSL — and they verify their own pipelines with the same `kici test` and `kici run local` loop you use, so there's no push-to-find-out round-trip. First-class agent context ships in the box, so an agent is briefed the moment it opens the project.
422
+
419
423
  KiCI ships first-class context for LLM coding agents (Claude Code, Cursor, Aider, etc.). When you scaffold a project with `kici init`, the CLI writes `.kici/AGENTS.md`, a one-page briefing that tells the agent:
420
424
 
421
425
  - where the SDK type declarations live (`node_modules/@kici-dev/sdk/dist/index.d.ts`)
@@ -1146,6 +1150,46 @@ export const postDeploy = workflow('post-deploy', {
1146
1150
  });
1147
1151
  ```
1148
1152
 
1153
+ `workflowComplete()` / `jobComplete()` start a **separate** workflow run that reacts to the prior one finishing, gated on its status. They are the right tool when a _different_ workflow should respond. When you instead need to add more jobs to the **same** run based on what a job just produced — fanning out follow-up work from a prior job's outputs — use a result-aware generator (next section), not a completion-event chain.
1154
+
1155
+ ### Same-run discovery → fan-out
1156
+
1157
+ A result-aware [`dynamicJob(group, { needs, generate })`](../sdk/rules-matrix-dynamic.md#dynamicjob--result-aware-generation) is deferred until its declared upstreams complete, then runs with their frozen outputs as `ctx.needs` — so a discovery job can emit a list at runtime and the generator fans out one follow-up job per item, all in the same run:
1158
+
1159
+ ```typescript
1160
+ import { workflow, job, step, push, dynamicJob, z } from '@kici-dev/sdk';
1161
+
1162
+ const discover = job('discover', {
1163
+ runsOn: 'linux',
1164
+ steps: [
1165
+ step('list-services', {
1166
+ outputs: { services: z.array(z.string()) },
1167
+ run: async ({ $ }) => {
1168
+ const out = await $`ls services/`;
1169
+ return { services: out.stdout.trim().split('\n') };
1170
+ },
1171
+ }),
1172
+ ],
1173
+ });
1174
+
1175
+ const deployEach = dynamicJob('deploys', {
1176
+ needs: ['discover'],
1177
+ generate: async ({ ctx }) =>
1178
+ ctx.needs.discover.result.services.map((svc) =>
1179
+ job(`deploy-${svc}`, {
1180
+ runsOn: 'linux',
1181
+ run: async ({ $ }) => {
1182
+ await $`./scripts/deploy.sh ${svc}`;
1183
+ },
1184
+ }),
1185
+ ),
1186
+ });
1187
+
1188
+ export default workflow('deploy-discovered-services', { on: push(), jobs: [discover, deployEach] });
1189
+ ```
1190
+
1191
+ Contrast: this keeps everything in one run with results flowing job→job. A cross-workflow `jobComplete()` chain (above) reacts to a job finishing but only sees its _status_, in a new run — use that when the reacting logic belongs to a different workflow.
1192
+
1149
1193
  ### Using custom events
1150
1194
 
1151
1195
  For richer payload data, emit custom events from steps using `ctx.emit()`:
@@ -1888,33 +1932,33 @@ function job(options: JobOptions): Job;
1888
1932
 
1889
1933
  **Parameters:**
1890
1934
 
1891
- | Parameter | Type | Required | Description |
1892
- | -------------------------- | --------------------------------------------------------------- | -------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- |
1893
- | `name` | `string` | no | Job name (auto-generated UUID if omitted) |
1894
- | `options.runsOn` | `RunsOn` | yes | Runner label(s) and optional exclusions (see below) |
1895
- | `options.steps` | `StepInput[]` | yes (or use `run`) | Steps to execute in order. Mutually exclusive with `run`. |
1896
- | `options.run` | `(ctx) => Promise<unknown>` | yes (or use `steps`) | Single-step shorthand -- see [Single-step job shorthand](#single-step-job-shorthand). Mutually exclusive with `steps`. |
1897
- | `options.needs` | `NeedsEntry[]` | no | Job dependencies (must complete first) -- see [Job dependencies (`needs`)](#job-dependencies-needs) |
1898
- | `options.rules` | `Rule[]` | no | Conditions for conditional execution |
1899
- | `options.description` | `string` | no | Human-readable description |
1900
- | `options.matrix` | `Matrix` | no | Matrix configuration for job expansion |
1901
- | `options.include` | `MatrixInclude[]` | no | Additional matrix combinations |
1902
- | `options.exclude` | `MatrixExclude[]` | no | Matrix combinations to remove |
1903
- | `options.checkout` | `boolean` | no (default: `true`) | When `false`, agent skips git clone. Useful for deploy/notify jobs. |
1904
- | `options.container` | `string \| ContainerConfig` | no | Docker image for job execution. String form is the image name; object form adds `env`. All steps run inside the container. |
1905
- | `options.environment` | `string \| ((event) => string \| Promise<string>)` | no | Deployment environment for this job. Static string or async/dynamic function -- see [Dynamic values](../dynamic-values.md). |
1906
- | `options.env` | `Record<string, string> \| ((event) => Record<string, string>)` | no | Environment variables. Static object or async/dynamic function -- see [Dynamic values](../dynamic-values.md). |
1907
- | `options.concurrencyGroup` | `string \| ((event) => string \| Promise<string>)` | no | Concurrency group name (defaults to environment name) -- see [Concurrency](../concurrency.md). |
1908
- | `options.onCancel` | `HookInput` | no | Hook that runs when the job is cancelled |
1909
- | `options.cleanup` | `HookInput` | no | Hook that always runs after completion |
1910
- | `options.onSuccess` | `HookInput` | no | Hook that runs when the job succeeds |
1911
- | `options.onFailure` | `HookInput` | no | Hook that runs when the job fails |
1912
- | `options.beforeStep` | `HookInput` | no | Hook that runs before each step |
1913
- | `options.afterStep` | `HookInput` | no | Hook that runs after each step |
1914
- | `options.gracePeriod` | `number` | no | Seconds before SIGKILL after SIGTERM during cancellation -- see [Hooks](../hooks.md#hook-timeout). |
1915
- | `options.timeout` | `number` | no | Total job wall-clock timeout in milliseconds (init + all steps + hooks). On breach the job is aborted and reported timed out. See [Timeouts](#timeouts). |
1916
- | `options.resources` | `ResourceRequest` | no | Per-job CPU / memory request and limit. See [Per-job resources](#per-job-resources) below. |
1917
- | `options.init` | `GenericInitConfig \| GenericInitConfig[] \| false` | no | Per-job initialization run after clone, before steps -- provisions a toolchain. See [Per-job init](#per-job-init) below. |
1935
+ | Parameter | Type | Required | Description |
1936
+ | -------------------------- | --------------------------------------------------------------- | -------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
1937
+ | `name` | `string` | no | Job name (auto-generated UUID if omitted) |
1938
+ | `options.runsOn` | `RunsOn` | yes | Runner label(s) and optional exclusions (see below) |
1939
+ | `options.steps` | `StepInput[]` | yes (or use `run`) | Steps to execute in order. Mutually exclusive with `run`. |
1940
+ | `options.run` | `(ctx) => Promise<unknown>` | yes (or use `steps`) | Single-step shorthand -- see [Single-step job shorthand](#single-step-job-shorthand). Mutually exclusive with `steps`. |
1941
+ | `options.needs` | `NeedsEntry[]` | no | Job dependencies (must complete first) -- see [Job dependencies (`needs`)](#job-dependencies-needs) |
1942
+ | `options.rules` | `Rule[]` | no | Conditions for conditional execution |
1943
+ | `options.description` | `string` | no | Human-readable description |
1944
+ | `options.matrix` | `Matrix` | no | Matrix configuration for job expansion |
1945
+ | `options.include` | `MatrixInclude[]` | no | Additional matrix combinations |
1946
+ | `options.exclude` | `MatrixExclude[]` | no | Matrix combinations to remove |
1947
+ | `options.checkout` | `boolean` | no (default: `true`) | When `false`, agent skips git clone. Useful for deploy/notify jobs. |
1948
+ | `options.container` | `string \| ContainerConfig` | no | Docker image for job execution. String form is the image name; object form adds `env`. All steps run inside the container. |
1949
+ | `options.environment` | `string \| ((event) => string \| Promise<string>)` | no | Deployment environment for this job. Static string or async/dynamic function -- see [Dynamic values](../dynamic-values.md). |
1950
+ | `options.env` | `Record<string, string> \| ((event) => Record<string, string>)` | no | Environment variables. Static object or async/dynamic function -- see [Dynamic values](../dynamic-values.md). |
1951
+ | `options.concurrencyGroup` | `string \| ((event) => string \| Promise<string>)` | no | Concurrency group name (defaults to environment name) -- see [Concurrency](../concurrency.md). |
1952
+ | `options.onCancel` | `HookInput` | no | Hook that runs when the job is cancelled |
1953
+ | `options.cleanup` | `HookInput` | no | Hook that always runs after completion |
1954
+ | `options.onSuccess` | `HookInput` | no | Hook that runs when the job succeeds |
1955
+ | `options.onFailure` | `HookInput` | no | Hook that runs when the job fails |
1956
+ | `options.beforeStep` | `HookInput` | no | Hook that runs before each step |
1957
+ | `options.afterStep` | `HookInput` | no | Hook that runs after each step |
1958
+ | `options.gracePeriod` | `number` | no | Seconds before SIGKILL after SIGTERM during cancellation -- see [Hooks](../hooks.md#hook-timeout). |
1959
+ | `options.timeout` | `number` | no | Total job wall-clock timeout in milliseconds (init + all steps + hooks). On breach the job is aborted and reported timed out. See [Timeouts](#timeouts). |
1960
+ | `options.resources` | `ResourceRequest` | no | Per-job CPU / memory request and limit. See [Per-job resources](#per-job-resources) below. |
1961
+ | `options.init` | `InitConfig` | no | Per-job initialization run after clone, before steps -- provisions a toolchain. A generic config, a typed preset (`'mise'` / `{ mise }`), `'auto'`, or `false`. See [Per-job init](#per-job-init) below. |
1918
1962
 
1919
1963
  **Returns:** `Job` -- an immutable job definition.
1920
1964
 
@@ -1935,20 +1979,27 @@ const build = job({
1935
1979
 
1936
1980
  #### runsOn forms
1937
1981
 
1938
- The `runsOn` parameter accepts three forms for targeting agents:
1982
+ A job's `runsOn` selects which agents may run it. Every label listed must be present on the agent (a subset match). It accepts three forms, and each label can be an exact string, a glob, or a regular expression (see [Targeting by pattern](#targeting-by-pattern) below):
1939
1983
 
1940
1984
  ```typescript
1941
1985
  // 1. Simple string -- agent must have this label
1942
- runsOn: 'linux'
1986
+ runsOn: 'kici:os:linux'
1943
1987
 
1944
1988
  // 2. Array of required labels -- agent must have ALL labels
1945
- runsOn: ['linux', 'docker']
1989
+ runsOn: ['kici:os:linux', 'gpu']
1946
1990
 
1947
1991
  // 3. Object form with exclusions -- agent must have ALL required labels
1948
1992
  // and NONE of the excluded labels
1949
- runsOn: { labels: ['linux', 'docker'], exclude: ['gpu'] }
1993
+ runsOn: { labels: ['kici:os:linux'], exclude: ['kici:host:box-01'] }
1950
1994
  ```
1951
1995
 
1996
+ **The label model:**
1997
+
1998
+ - Every agent automatically reports `kici:os:<platform>`, `kici:arch:<cpu>`, and `kici:host:<hostname>`, so `runsOn: 'kici:os:linux'` targets any connected Linux agent without configuring labels — a fresh `kici init` matches out of the box.
1999
+ - Use **custom labels** (e.g. `'gpu'`, `'prod-pool'`) — defined in your scaler's `labelSet` — to target a specific agent pool.
2000
+ - You can also target scaler-assigned labels (`kici:agent:<backend>`, `kici:scaler:<name>`), but those names are deployment-specific, so custom labels are more portable.
2001
+ - `runsOn` is a _requirement_ on candidate agents, never a _grant_: targeting a label only narrows the candidate set. Users cannot _set_ `kici:` labels on agents — that namespace is reserved for the scaler and the agent's self-reported platform facts — but they may freely _target_ any label in `runsOn`.
2002
+
1952
2003
  **Semantics:**
1953
2004
 
1954
2005
  - **Required labels:** The agent must have every label in the `labels` array (or the string/array form).
@@ -1970,6 +2021,48 @@ const deploy = job('deploy', {
1970
2021
  });
1971
2022
  ```
1972
2023
 
2024
+ #### Targeting by pattern
2025
+
2026
+ Every selector element — in `runsOn`, in `runsOnAll`, on both the include and the exclude side — can be a plain string, a glob pattern, or a regular expression. KiCI picks the matching mode from the value itself:
2027
+
2028
+ - **Plain string → exact match.** `'kici:os:linux'` matches the label `kici:os:linux` and nothing else.
2029
+ - **String with glob metacharacters (`*`, `?`, `[]`, `{}`) → glob.** `'kici:host:web-*'` matches every host label starting with `kici:host:web-`. `'kici:host:box-0[1-3]'` matches `box-01`, `box-02`, `box-03`.
2030
+ - **`RegExp` literal → regular expression.** `/kici:host:box-0[1-3]/` matches any label the expression matches.
2031
+
2032
+ Both the required (include) side and the excluded side accept all three forms:
2033
+
2034
+ ```typescript
2035
+ // Glob include + regex exclude, single-agent targeting.
2036
+ const build = job('build', {
2037
+ runsOn: { labels: ['kici:os:linux', 'kici:host:web-*'], exclude: [/.*-canary$/] },
2038
+ steps: [compile],
2039
+ });
2040
+
2041
+ // A bare regex picks any agent whose label the expression matches.
2042
+ const probe = job('probe', {
2043
+ runsOn: /kici:host:box-0[1-3]/,
2044
+ steps: [smoke],
2045
+ });
2046
+ ```
2047
+
2048
+ In the `runsOnAll` array form, a leading `!` still routes an entry to the exclude side. The `!` is stripped **before** the matching mode is decided, so `'!kici:host:box-*'` is an exclude **glob** and `'!box-01'` an exclude **exact** match. Regular-expression exclusions use the structured `exclude: [/…/]` form (a `RegExp` cannot carry a `!` prefix). The structured `runsOnAll` form below targets every Linux host in the `db` or `replica` role except those whose hostname ends in `-canary`:
2049
+
2050
+ ```typescript
2051
+ const fanout = job('deploy', {
2052
+ runsOnAll: {
2053
+ include: [{ all: ['kici:os:linux', 'kici:host:web-*'] }],
2054
+ exclude: [/.*-canary$/],
2055
+ },
2056
+ run: async (ctx) => {
2057
+ /* runs once per matched host */
2058
+ },
2059
+ });
2060
+ ```
2061
+
2062
+ **Edge case — custom labels that contain glob metacharacters.** Because the matching mode is inferred from the value, a custom label that literally contains `*`, `?`, `[]`, or `{}` is always treated as a glob and can no longer be matched exactly. Avoid glob metacharacters in label names you intend to target by exact string.
2063
+
2064
+ **ReDoS protection.** Glob patterns are linear by construction. A regular expression you supply is validated for catastrophic-backtracking (ReDoS) when you run `kici compile` — a pattern that could hang on a crafted input is rejected with an error, so it never reaches the orchestrator. The orchestrator re-validates every pattern when it loads the lock file.
2065
+
1973
2066
  ### step(name, run) / step(name, options)
1974
2067
 
1975
2068
  Create a step with a run function or with typed outputs.
@@ -2108,7 +2201,37 @@ The agent reads both files after the command succeeds, applies the delta, and th
2108
2201
 
2109
2202
  **Arrays run in order.** Passing `GenericInitConfig[]` runs the inits sequentially; each one's `$KICI_ENV` / `$KICI_PATH` delta is applied before the next runs, so a later init sees an earlier init's tools on `PATH`. The first init to fail stops the sequence and fails the job.
2110
2203
 
2111
- **`init: false`** is an explicit opt-out (reserved for a future auto-detect layer); it behaves the same as omitting `init`.
2204
+ **`init: false`** is an explicit opt-out; it behaves the same as omitting `init`.
2205
+
2206
+ #### Toolchain presets
2207
+
2208
+ For the common case, a typed preset removes the hand-written `run` block entirely. The agent expands the preset to the same generic init it would otherwise run.
2209
+
2210
+ - **`init: 'mise'`** -- zero-config. Installs mise, trusts and runs `mise install` against the committed mise config (`mise.toml` / `.mise.toml` / `.tool-versions`), hands mise's env + shims dir to subsequent steps, and caches mise's data dir under a key derived from the committed config (so a config change rotates the cache). The committed config is trusted automatically — committing it to your repo is the trust signal.
2211
+ - **`init: { mise: { cache, timeout, env, shell } }`** -- the same preset with overrides. These tune the generic fields a hand-written init exposes (minus `run`): `cache: false` disables caching, a `CacheSpec` replaces the default key/paths, and `timeout` / `env` / `shell` map straight through. `init: 'mise'` is exactly `init: { mise: {} }`.
2212
+
2213
+ ```typescript
2214
+ const build = job('build', {
2215
+ runsOn: 'linux',
2216
+ init: 'mise', // committed mise.toml pins the toolchain; jq, node, etc. land on PATH
2217
+ steps: [
2218
+ step('show-jq-version', async (ctx) => {
2219
+ const { stdout } = await ctx.$`jq --version`;
2220
+ ctx.log.info(`jq version: ${stdout.trim()}`);
2221
+ }),
2222
+ ],
2223
+ });
2224
+ ```
2225
+
2226
+ #### Auto-detect (`init: 'auto'`)
2227
+
2228
+ **`init: 'auto'`** detects the toolchain from committed files instead of naming a preset. The agent scans the clone root and selects a preset when a marker is present: `mise.toml` / `.mise.toml` / `.tool-versions` -> the mise preset. With no markers found, `'auto'` is a logged no-op.
2229
+
2230
+ `'auto'` is opt-in: an **unset** `init` does nothing even when the repo carries a `mise.toml` for local development. Use `'auto'` to enable detection and `false` to keep the explicit opt-out.
2231
+
2232
+ #### Cross-platform
2233
+
2234
+ The mise preset works on Linux, macOS, and Windows. On Linux and macOS mise is installed via its standalone install script; on Windows it is installed from its standalone GitHub release. The resulting toolchain reaches every step the same way on all three. On Windows the standalone mise binary requires the Microsoft Visual C++ runtime (`vc_redist.x64`) to be present on the agent host — install it once when provisioning a Windows agent that uses the mise preset.
2112
2235
 
2113
2236
  ## Step & job authoring patterns
2114
2237
 
@@ -3049,6 +3172,8 @@ Every variant carries the shared `EventBase` fields — `type`, `action`, `targe
3049
3172
 
3050
3173
  Matrix configurations expand a single job into multiple instances, one per parameter combination. Maximum 256 combinations.
3051
3174
 
3175
+ Expansion happens at **dispatch time**: the orchestrator materializes the matrix into N execution jobs — one per combination, each dispatched to its own agent — before any job runs. Each instance receives its combination as `ctx.matrix`. This is identical whether the workflow runs via `kici run local` or remotely through a webhook trigger, and the dashboard groups the N instances under one parent node.
3176
+
3052
3177
  ### Static array (single dimension)
3053
3178
 
3054
3179
  ```typescript
@@ -3103,6 +3228,8 @@ The function receives a `DynamicMatrixContext`:
3103
3228
 
3104
3229
  Must return `string[]` (single dimension) or `Record<string, string[]>` (multi-dimensional).
3105
3230
 
3231
+ A dynamic matrix is resolved at runtime, then materialized into N instances exactly like a static matrix. Because the combinations are not known until the function runs, the 256-combination cap (and the "zero combinations" guard) is enforced at that point: a dynamic matrix that resolves to more than 256 combinations, or to none, fails the job with a matrix-expansion error rather than dispatching.
3232
+
3106
3233
  ### Include and exclude
3107
3234
 
3108
3235
  Fine-tune matrix combinations on multi-dimensional matrices:
@@ -3141,6 +3268,58 @@ interface MatrixValues {
3141
3268
  }
3142
3269
  ```
3143
3270
 
3271
+ ### Bounding matrix concurrency (maxParallel / failFast)
3272
+
3273
+ A matrix fan-out runs every combination at once by default. The fan-out-generic
3274
+ `maxParallel` and `failFast` job options bound it the same way they bound a
3275
+ [`runsOnAll`](./runs-on-all.md#rolling-rollout-maxparallel--failfast) host fan-out:
3276
+
3277
+ ```typescript
3278
+ const test = job('test', {
3279
+ runsOn: 'linux',
3280
+ matrix: { os: ['ubuntu', 'macos', 'windows'] },
3281
+ maxParallel: 1, // run one combination at a time (sliding window)
3282
+ failFast: true, // stop launching combinations after the first failure
3283
+ run: async (ctx) => {
3284
+ /* ctx.matrix.os */
3285
+ },
3286
+ });
3287
+ ```
3288
+
3289
+ `maxParallel` is a sliding window (each combination that finishes releases the next;
3290
+ `1` = serial; must be `>= 1`); `failFast` halts the fan-out on the first failure and
3291
+ skips the held remainder (default `false`). They are ignored on a job with no `matrix`
3292
+ or `runsOnAll`.
3293
+
3294
+ ### Consuming matrix outputs downstream
3295
+
3296
+ A downstream job that lists a matrix job in its `needs` receives a **keyed envelope** instead of a flat outputs object, because the upstream produced N sets of outputs (one per combination). `ctx.jobOutputs(matrixJob)` returns a `MatrixJobOutputs`:
3297
+
3298
+ ```typescript
3299
+ interface MatrixJobOutputs<T = Record<string, unknown>> {
3300
+ /** Keyed by the combination suffix — the text inside `(...)` of the child name. */
3301
+ byMatrix: Record<string, T>;
3302
+ /** Last-write-wins flat merge across children, in child (name) order. */
3303
+ merged: T;
3304
+ }
3305
+ ```
3306
+
3307
+ The suffix key matches the child job's display name: `byMatrix['a']` for a single-dimension `['a', 'b']` matrix, `byMatrix['linux, arm64']` for a multi-dimension combination. Use `isMatrixJobOutputs` (or `'byMatrix' in result`) to discriminate:
3308
+
3309
+ ```typescript
3310
+ import { isMatrixJobOutputs } from '@kici-dev/sdk';
3311
+
3312
+ step('collect', async ({ jobOutputs }) => {
3313
+ const out = jobOutputs(buildMatrixJob);
3314
+ if (isMatrixJobOutputs(out)) {
3315
+ console.log(out.byMatrix['a']); // outputs of the `a` combination
3316
+ console.log(out.merged); // last-write-wins across all combinations
3317
+ }
3318
+ });
3319
+ ```
3320
+
3321
+ The downstream job waits for **all** matrix combinations to terminate before it dispatches. A non-matrix upstream keeps the flat outputs shape. The envelope is identical under `kici run local` and the remote path.
3322
+
3144
3323
  ### Matrix type guards
3145
3324
 
3146
3325
  ```typescript
@@ -3201,6 +3380,53 @@ export default workflow('ci', {
3201
3380
  });
3202
3381
  ```
3203
3382
 
3383
+ ### dynamicJob — result-aware generation
3384
+
3385
+ `dynamicJob(group, fnOrConfig)` tags a generator with a group name (so static jobs can depend on it via `needs: [dynamicGroup('group')]`). It is polymorphic:
3386
+
3387
+ - **Function form** — event-only, dispatched at webhook time: `dynamicJob('shards', async ({ ctx }) => [...])`.
3388
+ - **Options-object form** — result-aware, deferred until its declared `needs` complete, then run with the upstreams' frozen outputs as `ctx.needs`: `dynamicJob('reports', { needs, generate })`.
3389
+
3390
+ ```typescript
3391
+ import { workflow, job, step, dynamicJob, dynamicGroup, z } from '@kici-dev/sdk';
3392
+
3393
+ // Upstream job A discovers a list of targets at runtime.
3394
+ const discover = job('discover', {
3395
+ runsOn: 'linux',
3396
+ steps: [
3397
+ step('emit', {
3398
+ outputs: { targets: z.array(z.string()) },
3399
+ run: async () => ({ targets: ['api', 'web'] }),
3400
+ }),
3401
+ ],
3402
+ });
3403
+
3404
+ // Result-aware generator fans out one report job per discovered target.
3405
+ const reports = dynamicJob('reports', {
3406
+ needs: ['discover'],
3407
+ generate: async ({ ctx }) => {
3408
+ const targets = ctx.needs.discover.result.targets; // OutputProxy over discover's outputs
3409
+ return targets.map((target) =>
3410
+ job(`report-${target}`, {
3411
+ runsOn: 'linux',
3412
+ run: async ({ log }) => log.info(`reporting on ${target}`),
3413
+ }),
3414
+ );
3415
+ },
3416
+ });
3417
+
3418
+ export default workflow('discovery-fan-out', { jobs: [discover, reports] });
3419
+ ```
3420
+
3421
+ `ctx.needs` shape:
3422
+
3423
+ | Need form | `ctx.needs[...]` value |
3424
+ | ------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- |
3425
+ | `'jobName'` / `{ name, ifFailed }` | `{ result }` — `result` is an `OutputProxy` (`ctx.needs.<job>.result.<step>.<field>`; single-step `run` jobs flatten to `ctx.needs.<job>.result.<field>`) |
3426
+ | `dynamicGroup('g')` / `dynamicGroup('g', { ifFailed })` | ordered array of `{ name, result }`, one per group member |
3427
+
3428
+ `ctx.needs` is deterministic — a snapshot of upstream outputs frozen at first eval and replayed unchanged on re-eval, like `ctx.event`. Use result-aware generation for same-run fan-out from a prior job's result; use [`jobComplete()`](./triggers.md) for cross-workflow reactions to a job finishing. See the architecture deep-dive in [dynamic jobs](../../architecture/execution/dynamic-jobs.md#result-aware-generation).
3429
+
3204
3430
  ### JobOrFactory
3205
3431
 
3206
3432
  The `jobs` array in `WorkflowOptions` accepts both static jobs and dynamic generators:
@@ -3229,6 +3455,218 @@ for (const item of workflow.jobs) {
3229
3455
 
3230
3456
  ---
3231
3457
 
3458
+ ## SDK reference: runsOnAll host fan-out
3459
+
3460
+ Source: https://docs.kici.dev/user/sdk/runs-on-all/
3461
+
3462
+ ## runsOnAll
3463
+
3464
+ `runsOnAll` fans a single job out to **every** host in the orchestrator's declared
3465
+ roster that matches a label predicate — one pinned execution per host. Use it for
3466
+ fleet-wide operations: patch every web tier, smoke-test every node, collect uptime
3467
+ from the fleet.
3468
+
3469
+ `runsOnAll` is mutually exclusive with [`runsOn`](/user/sdk/core/): a job declares one
3470
+ or the other. Where `runsOn` picks a **single** agent that satisfies the labels,
3471
+ `runsOnAll` targets **all** matching hosts and runs the job once on each, pinned to
3472
+ that specific host.
3473
+
3474
+ ```typescript
3475
+ import { job } from '@kici-dev/sdk';
3476
+
3477
+ // Run on every host labelled role:web.
3478
+ const patch = job('patch', {
3479
+ runsOnAll: 'role:web',
3480
+ run: async (ctx) => {
3481
+ await ctx.$`sudo apt-get update && sudo apt-get upgrade -y`;
3482
+ ctx.log.info(`patched ${ctx.host}`);
3483
+ },
3484
+ });
3485
+ ```
3486
+
3487
+ ### Input forms
3488
+
3489
+ `runsOnAll` accepts three shapes:
3490
+
3491
+ - **Bare string** — one required label.
3492
+
3493
+ ```typescript
3494
+ runsOnAll: 'role:web';
3495
+ ```
3496
+
3497
+ - **Array** — every positive entry is required (AND); a `!`-prefixed entry excludes a host.
3498
+
3499
+ ```typescript
3500
+ runsOnAll: ['kici:os:linux', 'role:db', '!kici:host:db-01'];
3501
+ ```
3502
+
3503
+ - **Structured** — explicit OR-of-AND include groups plus excludes.
3504
+
3505
+ ```typescript
3506
+ runsOnAll: {
3507
+ include: [{ all: ['kici:os:linux', 'role:db'] }, { all: ['role:replica'] }],
3508
+ exclude: ['kici:host:db-01'],
3509
+ };
3510
+ ```
3511
+
3512
+ A host matches when it satisfies **any** include group (all labels in that group)
3513
+ and carries **none** of the exclude labels.
3514
+
3515
+ #### Targeting by pattern
3516
+
3517
+ Every entry in any of these forms — include or exclude — can be an exact string, a
3518
+ glob, or a regular expression, exactly like [`runsOn`](./core.md#targeting-by-pattern):
3519
+
3520
+ - **Plain string → exact match** (`'role:web'`).
3521
+ - **String with glob metacharacters (`*`, `?`, `[]`, `{}`) → glob** (`'kici:host:web-*'`).
3522
+ - **`RegExp` literal → regular expression** (`/.*-canary$/`).
3523
+
3524
+ In the array form, a leading `!` routes an entry to the exclude side and is stripped
3525
+ **before** the matching mode is decided, so `'!kici:host:box-*'` is an exclude **glob**
3526
+ and `'!box-01'` an exclude **exact** match. A regular-expression exclusion uses the
3527
+ structured `exclude: [/…/]` form (a `RegExp` cannot carry a `!` prefix):
3528
+
3529
+ ```typescript
3530
+ const fanout = job('deploy', {
3531
+ runsOnAll: {
3532
+ include: [{ all: ['kici:os:linux', 'kici:host:web-*'] }],
3533
+ exclude: [/.*-canary$/],
3534
+ },
3535
+ run: async (ctx) => {
3536
+ /* runs once per matched host */
3537
+ },
3538
+ });
3539
+ ```
3540
+
3541
+ A custom label that literally contains glob metacharacters is always treated as a glob
3542
+ and can no longer be matched exactly. A regular expression you supply is validated for
3543
+ catastrophic-backtracking (ReDoS) when you run `kici compile` and rejected if it could
3544
+ hang on a crafted input.
3545
+
3546
+ ### Per-host execution model
3547
+
3548
+ Each matching host runs the job as its own pinned child, named `<job> (<hostname>)`
3549
+ (e.g. `patch (web-01)`). The children fan in for downstream `needs:` exactly like a
3550
+ matrix job — a downstream that needs the base job waits for every host child.
3551
+
3552
+ The job runs once per host with concurrency `unlimited` (all hosts in parallel).
3553
+
3554
+ ### ctx.host and ctx.agent
3555
+
3556
+ Inside a `runsOnAll` step, two extra context fields identify the host the child is
3557
+ running on:
3558
+
3559
+ - `ctx.host` — the hostname (string).
3560
+ - `ctx.agent` — the resolved agent facts: `{ host, labels, platform?, arch? }`.
3561
+
3562
+ ```typescript
3563
+ run: async (ctx) => {
3564
+ ctx.log.info(`running on ${ctx.host} (${ctx.agent?.platform}/${ctx.agent?.arch})`);
3565
+ };
3566
+ ```
3567
+
3568
+ Both are `undefined` for jobs that do not use `runsOnAll`.
3569
+
3570
+ ### byHost outputs
3571
+
3572
+ A downstream that `needs:` a `runsOnAll` job receives a **byHost** envelope instead
3573
+ of a flat outputs object — keyed by hostname, with a per-host summary:
3574
+
3575
+ ```typescript
3576
+ import { isHostJobOutputs } from '@kici-dev/sdk';
3577
+
3578
+ const report = job('report', {
3579
+ runsOn: 'role:control',
3580
+ needs: [patch],
3581
+ run: async (ctx) => {
3582
+ const outputs = ctx.jobOutputs(patch);
3583
+ if (isHostJobOutputs(outputs)) {
3584
+ ctx.log.info(`succeeded: ${outputs.summary.succeededHosts.join(', ')}`);
3585
+ ctx.log.info(`failed: ${outputs.summary.failedHosts.join(', ')}`);
3586
+ // Per-host outputs, keyed by hostname:
3587
+ const version = outputs.byHost['web-01']?.version;
3588
+ // Array view of one output key across every host:
3589
+ const allVersions = outputs.summary.outputs.version;
3590
+ }
3591
+ },
3592
+ });
3593
+ ```
3594
+
3595
+ Unlike the matrix envelope's last-write-wins `merged`, the host summary never collapses
3596
+ to a single scalar: `summary.outputs[key]` is an array of every host's value, and
3597
+ `succeededHosts` / `failedHosts` record each host's terminal outcome.
3598
+
3599
+ ### onUnreachable: skip | fail | hold
3600
+
3601
+ Resolution is backed by the **declared host roster** (see the operator
3602
+ [host roster](/operator/orchestrator/host-roster/) doc), not just the live registry.
3603
+ This lets KiCI surface an expected-but-absent host instead of silently fanning out to a
3604
+ partial fleet. The `onUnreachable` policy controls what happens when a **durable**
3605
+ (static) host in the roster is matched but not currently connected:
3606
+
3607
+ - **`hold`** (default) — queue a pinned child for the absent host and wait for it to
3608
+ reconnect. The fan-out is honest: a 5-host fleet with 1 host rebooting reports
3609
+ `4 ran, 1 held`, not a silent 4-of-5 success.
3610
+ - **`skip`** — omit the absent durable host and run only on the reachable hosts.
3611
+ - **`fail`** — fail the run init if any expected durable host is unreachable.
3612
+
3613
+ ```typescript
3614
+ const patch = job('patch', {
3615
+ runsOnAll: 'role:web',
3616
+ onUnreachable: 'skip',
3617
+ run: async (ctx) => {
3618
+ /* ... */
3619
+ },
3620
+ });
3621
+ ```
3622
+
3623
+ Ephemeral (scaled-down) hosts that are no longer connected are **always** skipped,
3624
+ independent of `onUnreachable` — a scaled-down node may never return. A `runsOnAll`
3625
+ that matches zero usable hosts fails the run rather than reporting a silent zero-child
3626
+ success.
3627
+
3628
+ ### Rolling rollout: maxParallel + failFast
3629
+
3630
+ By default a `runsOnAll` fan-out dispatches to every matched host at once — fine for
3631
+ collecting state across the fleet, dangerous for a deploy that takes the whole tier
3632
+ down simultaneously. Two job options bound the rollout:
3633
+
3634
+ - **`maxParallel`** — the fan-out width: at most this many hosts run at once. It is a
3635
+ sliding window — each host that finishes (success or failure) releases the next held
3636
+ host. `maxParallel: 1` is a strictly serial, one-host-at-a-time rolling deploy. Must
3637
+ be `>= 1`.
3638
+ - **`failFast`** — when `true`, the first host failure halts the rollout: no further
3639
+ held hosts are started, and the remaining ones are marked skipped. Default `false`
3640
+ (every host runs regardless of sibling outcomes — the same as the unbounded fan-out).
3641
+
3642
+ ```typescript
3643
+ const deploy = job('deploy', {
3644
+ runsOnAll: 'role:web',
3645
+ onUnreachable: 'skip', // see the caveat below
3646
+ maxParallel: 1, // strictly one host at a time
3647
+ failFast: true, // stop the roll on the first failure
3648
+ run: async (ctx) => {
3649
+ /* patch ctx.host */
3650
+ },
3651
+ });
3652
+ ```
3653
+
3654
+ Both options are **fan-out-generic** — they bound a `matrix` fan-out exactly the same
3655
+ way (the children are matrix combinations instead of hosts). They are ignored on a job
3656
+ with neither `matrix` nor `runsOnAll` (there is no fan-out to bound).
3657
+
3658
+ **Caveat — use `onUnreachable: 'skip'` or `'fail'` for rolling deploys, not `'hold'`.**
3659
+ A held host occupies a wave slot indefinitely while it waits to reconnect, stalling the
3660
+ roll behind an absent box. `skip` (run only reachable hosts) or `fail` (refuse the roll
3661
+ if any expected host is down) keep the window moving.
3662
+
3663
+ ### Limits (v0)
3664
+
3665
+ - Per-host secret scoping is not yet available — all hosts receive the job's resolved
3666
+ secrets.
3667
+
3668
+ ---
3669
+
3232
3670
  ## SDK reference: runtime
3233
3671
 
3234
3672
  Source: https://docs.kici.dev/user/sdk/runtime/
@@ -3239,25 +3677,26 @@ All types are exported from `@kici-dev/sdk` as type-only imports.
3239
3677
 
3240
3678
  ### Core types
3241
3679
 
3242
- | Type | Description |
3243
- | ----------------- | ------------------------------------------------------------------------------------- |
3244
- | `Workflow` | Workflow definition returned by `workflow()` |
3245
- | `WorkflowOptions` | Options for `workflow()` factory |
3246
- | `Job` | Job definition returned by `job()` |
3247
- | `JobOptions` | Options for `job()` factory |
3248
- | `Step<TOutputs>` | Step definition returned by `step()` |
3249
- | `StepOptions<T>` | Options for `step()` factory (full form with outputs) |
3250
- | `StepRunFn` | Simple step function type: `(ctx) => Promise<void>` |
3251
- | `BareStepFn` | Bare step function (no options, just `(ctx) => ...`) |
3252
- | `StepInput` | Union of step input forms accepted by `job()` |
3253
- | `OutputSchema` | Record of Zod types for step outputs |
3254
- | `InferOutputs<T>` | Infer output type from output schema |
3255
- | `ContainerConfig` | Container config for job execution (`image`, `env?`) |
3256
- | `RunsOn` | Union of `runsOn` forms: `string \| string[] \| RunsOnSelector` |
3257
- | `RunsOnSelector` | Object form for `runsOn` with `labels` (required) and `exclude` (optional) properties |
3258
- | `Fixture` | Test fixture definition returned by `fixture()` |
3259
- | `FixtureOptions` | Options for `fixture()` factory |
3260
- | `Registry` | Private npm registry declaration used in `WorkflowOptions.registries` |
3680
+ | Type | Description |
3681
+ | ----------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
3682
+ | `Workflow` | Workflow definition returned by `workflow()` |
3683
+ | `WorkflowOptions` | Options for `workflow()` factory |
3684
+ | `Job` | Job definition returned by `job()` |
3685
+ | `JobOptions` | Options for `job()` factory |
3686
+ | `Step<TOutputs>` | Step definition returned by `step()` |
3687
+ | `StepOptions<T>` | Options for `step()` factory (full form with outputs) |
3688
+ | `StepRunFn` | Simple step function type: `(ctx) => Promise<void>` |
3689
+ | `BareStepFn` | Bare step function (no options, just `(ctx) => ...`) |
3690
+ | `StepInput` | Union of step input forms accepted by `job()` |
3691
+ | `OutputSchema` | Record of Zod types for step outputs |
3692
+ | `InferOutputs<T>` | Infer output type from output schema |
3693
+ | `ContainerConfig` | Container config for job execution (`image`, `env?`) |
3694
+ | `RunsOn` | Union of `runsOn` forms: `string \| RegExp \| (string \| RegExp)[] \| RunsOnSelector`. A plain string matches exactly, a string with glob metacharacters (`*?[]{}`) is a glob, and a `RegExp` is a regular expression. See [Targeting by pattern](./core.md#targeting-by-pattern). |
3695
+ | `RunsOnSelector` | Object form for `runsOn` with `labels` (required) and `exclude` (optional) properties. Each element accepts the exact / glob / regex forms on both sides. |
3696
+ | `RunsOnAllInput` | Union of `runsOnAll` host fan-out forms: `string \| RegExp \| (string \| RegExp)[] \| { include: { all: (string \| RegExp)[] }[]; exclude?: (string \| RegExp)[] }`. Same exact / glob / regex semantics per element. See [runsOnAll](./runs-on-all.md#targeting-by-pattern). |
3697
+ | `Fixture` | Test fixture definition returned by `fixture()` |
3698
+ | `FixtureOptions` | Options for `fixture()` factory |
3699
+ | `Registry` | Private npm registry declaration used in `WorkflowOptions.registries` |
3261
3700
 
3262
3701
  ### Trigger types
3263
3702
 
@@ -3556,6 +3995,68 @@ const deploy = job('deploy', {
3556
3995
  - Maximum 20 secret outputs per job
3557
3996
  - Maximum 64 KB per value
3558
3997
 
3998
+ ### ctx.kici.oidc.token({ audience })
3999
+
4000
+ Request a short-lived OIDC ID token for the current job, bound to an `audience`. The token is a signed JWT whose identity claims (`repository`, `ref`, `sha`, `kici_run_id`, `kici_job_id`) are derived by the build platform from the run context — a step cannot spoof them. Use it to authenticate the build to an external service that trusts the platform's OIDC issuer (for example, when generating build provenance).
4001
+
4002
+ ```typescript
4003
+ const publish = job('publish', {
4004
+ steps: [
4005
+ step('mint', async (ctx) => {
4006
+ const { token, expiresIn } = await ctx.kici.oidc.token({ audience: 'sigstore' });
4007
+ ctx.log.info(`Got an ID token valid for ${expiresIn}s`);
4008
+ // Hand `token` to a tool that exchanges it with the trusting service.
4009
+ }),
4010
+ ],
4011
+ });
4012
+ ```
4013
+
4014
+ **Behavior:**
4015
+
4016
+ - The token is short-lived (about 10 minutes) and scoped to the current run and job.
4017
+ - The returned token value is automatically masked in step logs.
4018
+ - The step never holds platform credentials — the request is relayed through the orchestrator, which mints the token on the step's behalf.
4019
+ - Only available inside a running job step; calling it outside one (for example, during local execution) rejects with a clear error.
4020
+
4021
+ ### ctx.attestProvenance({ subject })
4022
+
4023
+ Build, sign, and persist a build-provenance attestation for an artifact your step produced. KiCI assembles an in-toto SLSA v1.0 provenance statement whose build identity (`repository`, `ref`, `sha`, run/job ids) comes from the platform — not from the step — so it cannot be spoofed, signs it, and stores a verifiable bundle that the dashboard surfaces and the `kici verify-attestation` CLI checks.
4024
+
4025
+ The artifact is **caller-supplied**: give it either a precomputed digest or a path (relative to the step working directory) that KiCI digests with SHA-256. For a container image, pass the manifest digest your build tool emitted.
4026
+
4027
+ ```typescript
4028
+ const publish = job('publish', {
4029
+ steps: [
4030
+ step('build', async (ctx) => {
4031
+ await ctx.$`npm pack`;
4032
+ }),
4033
+ step('attest', async (ctx) => {
4034
+ // Digest a file KiCI hashes for you:
4035
+ const result = await ctx.attestProvenance({
4036
+ subject: { name: 'my-pkg-1.2.3.tgz', path: 'my-pkg-1.2.3.tgz' },
4037
+ });
4038
+ ctx.log.info(`Attestation stored at ${result.storageKey}`);
4039
+
4040
+ // Or supply a precomputed digest (e.g. a container manifest digest):
4041
+ await ctx.attestProvenance({
4042
+ subject: { name: 'ghcr.io/acme/app', digest: { sha256: '<manifest-digest>' } },
4043
+ });
4044
+ }),
4045
+ ],
4046
+ });
4047
+ ```
4048
+
4049
+ **Behavior:**
4050
+
4051
+ - The attestation is a signed [DSSE](https://github.com/secure-systems-lab/dsse) envelope over an [in-toto](https://in-toto.io) statement carrying the [SLSA v1.0](https://slsa.dev/spec/v1.0/provenance) provenance predicate.
4052
+ - It is signed with an ephemeral key bound to a platform-minted identity token, so it is **offline-verifiable** against the platform's published signing keys — no online lookup needed at verify time.
4053
+ - The bundle is persisted to object storage and recorded so the dashboard can show it and `kici verify-attestation` can retrieve it.
4054
+ - The returned `{ storageKey, subjectDigest, bundleMediaType }` identifies the stored bundle.
4055
+ - Only available inside a running job step; calling it outside one (for example, during local execution) rejects with a clear error.
4056
+
4057
+ See the [build provenance guide](../provenance.md) for the end-to-end attest →
4058
+ verify → view journey, including how to verify a bundle with `kici verify-attestation`.
4059
+
3559
4060
  ## Secrets
3560
4061
 
3561
4062
  Workflows access secrets through `ctx.secrets` on `StepContext`. Use `await ctx.secrets.get('KEY')` to retrieve a value (rejects with `SecretNotFoundError` if the key is missing, fail-fast on typos), `ctx.secrets.has('KEY')` for a synchronous existence check, and `await ctx.secrets.expose('KEY')` when you need the value as a `process.env` entry for a child process.
@@ -4126,6 +4627,8 @@ jobComplete({ workflow: 'CI', job: 'build', status: ['success'] }); // Success o
4126
4627
  jobComplete({ workflow: 'CI', job: 'build', source: 'org/repo' }); // Cross-repo
4127
4628
  ```
4128
4629
 
4630
+ `jobComplete()` starts a **new** workflow run that reacts to another job finishing (gated on the prior job's status). For same-run fan-out — generating follow-up jobs from a prior job's _outputs_ within the same run — use a result-aware [`dynamicJob(group, { needs, generate })`](./rules-matrix-dynamic.md#dynamicjob--result-aware-generation) instead.
4631
+
4129
4632
  ### genericWebhook()
4130
4633
 
4131
4634
  Create a generic webhook trigger. Fires when a non-GitHub webhook is received from an external source configured via the admin API. Returns a frozen `GenericWebhookTriggerConfig`.
@@ -4603,7 +5106,7 @@ This:
4603
5106
 
4604
5107
  1. Revokes the PAT on the server (preventing further use)
4605
5108
  2. Clears auth fields from the local config file
4606
- 3. Preserves non-auth settings (endpoint, routing key, connection mode)
5109
+ 3. Preserves non-auth settings (active org, default clusters, Platform endpoint)
4607
5110
 
4608
5111
  ## Organization management
4609
5112
 
@@ -4623,6 +5126,10 @@ kici org use <name-or-id>
4623
5126
 
4624
5127
  Name matching is case-insensitive. You can also use the organization ID directly.
4625
5128
 
5129
+ The active organization is both the scope for org-scoped commands (`kici runs list`, `kici diagnostics`, `kici secrets list`, …) **and** the default target for `kici run remote`. After `kici login` and `kici org use <org>`, `kici run remote` dispatches to that org through the Platform — that is the complete path to a remote run. Override the target for a single run with `kici run remote --org <id>`.
5130
+
5131
+ If an organization has more than one connected orchestrator cluster, set its default cluster once with `kici orchestrators use <name>` (list them with `kici orchestrators list`). `kici run remote` then targets that cluster unless you pass `--orchestrator <name>`. With a single connected orchestrator the cluster is selected automatically.
5132
+
4626
5133
  ### Show current organization
4627
5134
 
4628
5135
  ```bash
@@ -4633,18 +5140,15 @@ Displays the currently active organization name and ID.
4633
5140
 
4634
5141
  ## Auth status
4635
5142
 
4636
- The `kici status <run-id>` command displays run details including your authentication state:
5143
+ `kici org current` shows your current login state and active organization:
4637
5144
 
4638
5145
  ```bash
4639
- kici status <run-id>
5146
+ kici org current
4640
5147
  ```
4641
5148
 
4642
- The auth-related output includes:
4643
-
4644
- - Login state (logged in / not logged in)
4645
- - Active organization name
4646
- - PAT expiry date and time remaining
4647
- - Warning if PAT expires within 7 days
5149
+ It reports whether you are logged in and which organization is active. PAT
5150
+ expiry and the full list of your tokens are managed from the dashboard (see
5151
+ "Dashboard management" below).
4648
5152
 
4649
5153
  ## Personal access tokens
4650
5154
 
@@ -4723,7 +5227,6 @@ The dashboard is a browser SPA on top of the same `/api/v1/*` surface, so anythi
4723
5227
  - **Workflows & runs:** `/orgs/:customerId/{runs,registrations,workflows,held-runs,environments,secrets,global-workflows}`
4724
5228
  - **Webhooks & event log:** `/orgs/:customerId/{sources,webhook-endpoints,event-log}`
4725
5229
  - **Diagnostics & activity:** `/orgs/:customerId/{diagnostics,activity,access-log}`
4726
- - **Admin (kici-admin org only):** `/admin/{orgs,connections,audit-log,grafana/*}`
4727
5230
 
4728
5231
  The full route tree is the source of truth — every method, request schema, and response schema is enumerated server-side. There is currently no auto-generated OpenAPI spec; the typed `DashboardApiType` export is the canonical contract for TypeScript clients.
4729
5232
 
@@ -4767,7 +5270,8 @@ The CLI stores authentication data in `~/.kici/config` with `0600` permissions (
4767
5270
  - PAT token
4768
5271
  - PAT expiry date
4769
5272
  - Active organization ID
4770
- - Server endpoint URL
5273
+ - Per-org default orchestrator clusters
5274
+ - Platform endpoint URL
4771
5275
 
4772
5276
  ## Troubleshooting
4773
5277
 
@@ -5026,7 +5530,9 @@ kici run local push --keep-going
5026
5530
 
5027
5531
  Execute fixtures remotely through the full CI pipeline. Fixtures are defined in `.kici/tests/*.ts` using the `fixture()` factory function. Without arguments, lists available fixtures.
5028
5532
 
5029
- Requires `kici login` (an authenticated session) and a target orchestrator that has **cache storage configured** (`KICI_STORAGE_TYPE` = `s3` or `filesystem`) the command uploads your working-tree overlay to that storage for the agent to fetch. The quickstart orchestrators do not enable storage by default; see the [testing guide](testing-guide.md) for setup (including non-public / self-hosted S3 endpoints).
5533
+ Remote runs route through the Platform. Authenticate with a personal access token (`kici login`), then target an organization with `kici org use <org>` or the `--org` flag. The Platform relays the run to the org's orchestrator, while your working-tree overlay uploads directly to object storage see [How the run is routed](#how-the-run-is-routed) and [The two planes](#the-two-planes) below.
5534
+
5535
+ The orchestrator must have **cache storage configured** (`KICI_STORAGE_TYPE` = `s3` or `filesystem`) with a dev-reachable upload endpoint so the CLI's direct upload succeeds; see the [testing guide](testing-guide.md) and [Storage layout](../operator/orchestrator/storage-layout.md) for setup.
5030
5536
 
5031
5537
  ```bash
5032
5538
  kici run remote [fixture] [options]
@@ -5040,21 +5546,22 @@ kici run remote [fixture] [options]
5040
5546
 
5041
5547
  **Options:**
5042
5548
 
5043
- | Option | Default | Description |
5044
- | --------------------------- | ------- | --------------------------------------------------------------------------------------------------------------------- |
5045
- | `--all` | `false` | Run all fixtures |
5046
- | `--workflow <name>` | none | Run a specific workflow directly (bypass triggers) |
5047
- | `--parallel` | `false` | Run multiple fixtures concurrently |
5048
- | `--no-wait` | - | Fire and forget (print runIds, don't stream) |
5049
- | `--quiet` | `false` | Minimal output (only final result) |
5050
- | `--json` | `false` | Machine-readable JSON output |
5051
- | `--junit <path>` | none | JUnit XML output to file for CI integration |
5052
- | `--history` | `false` | Show table of recent test runs |
5053
- | `--routing-key <key>` | none | Override routing key for this run |
5054
- | `--context <ctx.key=value>` | none | Inject a namespaced context secret, uploaded encrypted to the orchestrator (repeatable) |
5055
- | `--env <KEY=VALUE>` | none | Provide a per-run secret, uploaded encrypted to the orchestrator (repeatable) — see [testing guide](testing-guide.md) |
5056
- | `--debug` | `false` | Verbose internals |
5057
- | `--kici-dir <path>` | `.kici` | Path to .kici directory |
5549
+ | Option | Default | Description |
5550
+ | --------------------------- | ------- | ------------------------------------------------------------------------------------------------- |
5551
+ | `--org <id>` | active | Target organization for this run (overrides `kici org use`) |
5552
+ | `--orchestrator <name>` | default | Target orchestrator cluster within the org (overrides the per-org default) |
5553
+ | `--all` | `false` | Run all fixtures |
5554
+ | `--workflow <name>` | none | Run a specific workflow directly (bypass triggers) |
5555
+ | `--parallel` | `false` | Run multiple fixtures concurrently |
5556
+ | `--no-wait` | - | Fire and forget (print runIds, don't stream) |
5557
+ | `--quiet` | `false` | Minimal output (only final result) |
5558
+ | `--json` | `false` | Machine-readable JSON output |
5559
+ | `--junit <path>` | none | JUnit XML output to file for CI integration |
5560
+ | `--history` | `false` | Show table of recent test runs |
5561
+ | `--context <ctx.key=value>` | none | Inject a namespaced context secret, uploaded encrypted (repeatable) |
5562
+ | `--env <KEY=VALUE>` | none | Provide a per-run secret, uploaded encrypted (repeatable) — see [testing guide](testing-guide.md) |
5563
+ | `--debug` | `false` | Verbose internals |
5564
+ | `--kici-dir <path>` | `.kici` | Path to .kici directory |
5058
5565
 
5059
5566
  **Examples:**
5060
5567
 
@@ -5062,9 +5569,15 @@ kici run remote [fixture] [options]
5062
5569
  # List available fixtures
5063
5570
  kici run remote
5064
5571
 
5065
- # Run a single fixture
5572
+ # Run a single fixture against the active org
5066
5573
  kici run remote push-main
5067
5574
 
5575
+ # Target a specific org for this run
5576
+ kici run remote push-main --org xyz789ghi012
5577
+
5578
+ # Target a specific orchestrator cluster within the org
5579
+ kici run remote push-main --orchestrator us-east
5580
+
5068
5581
  # Run all push-related fixtures
5069
5582
  kici run remote push-*
5070
5583
 
@@ -5094,19 +5607,75 @@ kici run remote --history
5094
5607
  | 0 | All matched workflows passed |
5095
5608
  | 1 | One or more workflows failed |
5096
5609
 
5610
+ #### How the run is routed
5611
+
5612
+ A remote run is dispatched to your **active organization** — the one set with `kici org use <org>`, or overridden per-run with `--org <id>`. The org is resolved in this order:
5613
+
5614
+ 1. The `--org <id>` flag, if provided.
5615
+ 2. Otherwise the active org saved in your global config by `kici org use <org>`.
5616
+ 3. If neither is set, the command errors and asks you to select an org with `kici org use` or pass `--org`.
5617
+
5618
+ The orchestrator anchors the org without any manual webhook source: it auto-provisions a system-managed **remote source** (routing key `remote:<orgId>`) that maps to its bound organization, so even a zero-source org is immediately routable for remote runs. You never set a routing key for a remote run — selecting the org is enough.
5619
+
5620
+ When an org has more than one connected orchestrator cluster, the CLI picks the target cluster in this order:
5621
+
5622
+ 1. The `--orchestrator <name>` flag, if provided.
5623
+ 2. Otherwise the per-org default cluster, set with `kici orchestrators use <name>`.
5624
+ 3. If the org has exactly **one** connected orchestrator, it is auto-selected.
5625
+ 4. Otherwise the run errors with the list of connected clusters, and you pass `--orchestrator <name>` to choose one. Run `kici orchestrators list` to see the available cluster names.
5626
+
5627
+ #### The two planes
5628
+
5629
+ `kici run remote` uses two independent paths:
5630
+
5631
+ - **Control plane** — run initiation, trigger, status, log retrieval, and cancellation flow from your machine through the Platform, which relays them over a WebSocket connection to the org's orchestrator. Logs are delivered by the CLI polling the Platform for log chunks (tracked by a monotonic line cursor) and run status until the run reaches a terminal state; there is no direct streaming socket to the orchestrator.
5632
+ - **Data plane** — your working-tree overlay tarball uploads **directly** from your machine to the orchestrator's object store via a presigned PUT URL. The overlay never passes through the Platform. This is why the orchestrator's object-store upload endpoint must be reachable from your machine; see [Storage layout](../operator/orchestrator/storage-layout.md).
5633
+
5634
+ An orchestrator with no Platform connection cannot serve remote runs — the Platform is the service that offers them. For executing workflow steps on your own machine without an orchestrator (no scaler, agents, or environments), use [`kici run local`](#kici-run-local).
5635
+
5097
5636
  #### Fresh repos (no GitHub remote)
5098
5637
 
5099
5638
  `kici run remote` works even if the repo has never been pushed to GitHub. When no remote is detected:
5100
5639
 
5101
5640
  - The entire repo content is uploaded (not just a diff overlay)
5102
- - A synthetic routing key `local:<repo-name>` is used
5103
5641
  - The lock file is sent inline (no GitHub API fetch)
5104
5642
  - Steps that use git commands will fail (no `.git` directory in the remote workspace)
5105
5643
  - Build cache (`__build__` jobs) is skipped for local repos
5106
5644
  - Environments must have `allowLocalExecution: true` to be accessible from local runs (default is `false`)
5107
5645
 
5646
+ Destination routing is unchanged for fresh repos: the run still goes to your active org through the Platform.
5647
+
5108
5648
  For a detailed guide on writing fixtures, configuring secrets, and understanding the upload flow, see [Testing guide](testing-guide.md).
5109
5649
 
5650
+ #### kici orchestrators
5651
+
5652
+ List the orchestrator clusters connected to an organization, and set the per-org default cluster used by `kici run remote`. Requires `kici login` and an active org (or pass `--org`).
5653
+
5654
+ ```bash
5655
+ kici orchestrators list [--org <id>]
5656
+ kici orchestrators use <clusterName> [--org <id>]
5657
+ ```
5658
+
5659
+ **`kici orchestrators list`** prints the org's connected orchestrator clusters, so you know what to pass to `--orchestrator` (or to `kici orchestrators use`).
5660
+
5661
+ **`kici orchestrators use <clusterName>`** sets the default orchestrator cluster for the org, stored per-org in your global config. Subsequent `kici run remote` invocations target that cluster unless overridden with `--orchestrator`.
5662
+
5663
+ **Examples:**
5664
+
5665
+ ```bash
5666
+ # List the active org's connected clusters
5667
+ kici orchestrators list
5668
+
5669
+ # List a specific org's clusters
5670
+ kici orchestrators list --org xyz789ghi012
5671
+
5672
+ # Set the default cluster for the active org
5673
+ kici orchestrators use us-east
5674
+
5675
+ # Set the default cluster for a specific org
5676
+ kici orchestrators use us-east --org xyz789ghi012
5677
+ ```
5678
+
5110
5679
  ### kici test
5111
5680
 
5112
5681
  Preview which workflows match a trigger event (dry-run, no execution). Useful for verifying trigger configurations during development.
@@ -5181,9 +5750,7 @@ kici login [options]
5181
5750
  | --------------------------- | ------- | ---------------------------------------------- |
5182
5751
  | `--token <key>` | none | API key for direct authentication (legacy) |
5183
5752
  | `--device` | false | Force device authorization flow (headless/SSH) |
5184
- | `--endpoint <url>` | none | Orchestrator URL for direct connection |
5185
- | `--platform-endpoint <url>` | none | Platform relay URL |
5186
- | `--routing-key <key>` | none | Routing key for webhook source identification |
5753
+ | `--platform-endpoint <url>` | none | Platform API base URL |
5187
5754
 
5188
5755
  **Environment variables:**
5189
5756
 
@@ -5208,11 +5775,8 @@ kici login --device
5208
5775
  # Legacy API key login
5209
5776
  kici login --token kici_sk_abc123...
5210
5777
 
5211
- # Direct connection to orchestrator with API key
5212
- kici login --token kici_sk_abc123... --endpoint https://my-orchestrator.example.com
5213
-
5214
- # Platform relay connection with routing key
5215
- kici login --token kici_sk_abc123... --platform-endpoint https://platform.kici.dev --routing-key github:42
5778
+ # Log in against a self-hosted Platform
5779
+ kici login --platform-endpoint https://platform.example.com
5216
5780
 
5217
5781
  # Suppress browser opening (print authorize URL to stdout)
5218
5782
  KICI_BROWSER_CMD=none kici login
@@ -5230,7 +5794,7 @@ KICI_CALLBACK_PORT=19876 KICI_CONFIG_DIR=/tmp/kici-test kici login
5230
5794
 
5231
5795
  Revoke your personal access token on the server and clear local credentials.
5232
5796
 
5233
- If the server is unreachable, local credentials are still cleared (the PAT will expire automatically). Non-auth config fields (endpoint, routing key, etc.) are preserved.
5797
+ If the server is unreachable, local credentials are still cleared (the PAT will expire automatically). Non-auth config fields (active org, default clusters, etc.) are preserved.
5234
5798
 
5235
5799
  ```bash
5236
5800
  kici logout
@@ -5296,87 +5860,169 @@ Show the current active organization.
5296
5860
  kici org current
5297
5861
  ```
5298
5862
 
5299
- ### kici status
5863
+ ### kici diagnostics
5300
5864
 
5301
- Show details for a specific test run. Fetches from the orchestrator with fallback to local history.
5865
+ Show the orchestrators, scalers, and agents serving your organization the
5866
+ terminal equivalent of the dashboard Diagnostics page. Reads the same
5867
+ org-scoped data the dashboard does, so it needs `kici login` and an active org
5868
+ (`kici org use <name>`).
5302
5869
 
5303
- The status output includes an **auth section** showing login state, active organization, and PAT expiry. A warning appears when the PAT expires within 7 days.
5304
-
5305
- For a failed run, the output prints a `Reason:` line with the run's failure reason and shows the failed job's error inline, so you can see why a run failed without opening the dashboard. When provisioning an agent failed before any step ran, this reason is the captured scaler error (for example a missing binary or an unpullable image) rather than a generic "no agents available" message.
5870
+ The output has three parts: a one-line header (runs in the last 24h, success
5871
+ rate, average duration, queued/running job counts), any infrastructure alerts
5872
+ (only shown when present), and a tree of each orchestrator with its scalers and
5873
+ agents. Each agent line shows its labels, platform/architecture, active/maximum
5874
+ concurrency, and heartbeat age.
5306
5875
 
5307
5876
  ```bash
5308
- kici status <run-id> [options]
5877
+ kici diagnostics [options]
5309
5878
  ```
5310
5879
 
5311
- **Arguments:**
5312
-
5313
- | Argument | Required | Description |
5314
- | -------- | -------- | -------------- |
5315
- | `run-id` | yes | Run identifier |
5316
-
5317
5880
  **Options:**
5318
5881
 
5319
- | Option | Default | Description |
5320
- | -------------- | ------- | ------------------------------------------------------------ |
5321
- | `--logs` | `false` | Stream logs (live for active runs, historical for completed) |
5322
- | `--job <name>` | all | Filter logs to a specific job |
5323
- | `--json` | `false` | Machine-readable JSON output |
5882
+ | Option | Default | Description |
5883
+ | --------------------- | ------- | --------------------------------------------------- |
5884
+ | `--json` | `false` | Machine-readable JSON output |
5885
+ | `--verbose` | `false` | Show extended per-agent fields (host, node, memory) |
5886
+ | `--orchestrator <id>` | all | Scope the tree to one orchestrator connection id |
5324
5887
 
5325
5888
  **Examples:**
5326
5889
 
5327
5890
  ```bash
5328
- # Show run summary
5329
- kici status abc123
5891
+ # Show the full infrastructure tree
5892
+ kici diagnostics
5330
5893
 
5331
- # Show full logs (historical for completed runs, live streaming for active runs)
5332
- kici status abc123 --logs
5894
+ # Extended per-agent detail
5895
+ kici diagnostics --verbose
5333
5896
 
5334
- # Show logs for a specific job
5335
- kici status abc123 --logs --job build
5897
+ # Only one orchestrator's scalers and agents
5898
+ kici diagnostics --orchestrator conn-abc123
5336
5899
 
5337
5900
  # Machine-readable output
5338
- kici status abc123 --json
5901
+ kici diagnostics --json
5339
5902
  ```
5340
5903
 
5341
- When `--json` is set, `kici` emits only the JSON document on stdout — the
5342
- `kici v<version>` banner is suppressed — so the output is safe to pipe into
5343
- `jq` or `JSON.parse`. The same holds for the other `--json` commands (`kici run
5344
- remote --json`, `kici workflows list --json`) and for `--quiet`.
5904
+ ### kici runs
5905
+
5906
+ Inspect and manage execution runs from the terminal the equivalent of the
5907
+ dashboard Runs page. All `kici runs` subcommands read/write the same org-scoped
5908
+ data as the dashboard, so they require `kici login` and an active org
5909
+ (`kici org use <name>`).
5345
5910
 
5346
- ### kici cancel
5911
+ #### kici runs list
5347
5912
 
5348
- Cancel a running workflow or all runs on a branch.
5913
+ List runs with optional filters. Output is a table (run id, workflow, status,
5914
+ branch, trigger, started, duration); pagination is reported at the bottom.
5349
5915
 
5350
5916
  ```bash
5351
- kici cancel [run-id] [options]
5917
+ kici runs list [options]
5352
5918
  ```
5353
5919
 
5354
- **Arguments:**
5920
+ **Options:**
5921
+
5922
+ | Option | Default | Description |
5923
+ | ----------------------- | ------- | --------------------------------------------- |
5924
+ | `--status <s>` | all | Filter by run status |
5925
+ | `--workflow <w>` | all | Filter by workflow name |
5926
+ | `--branch <b>` | all | Filter by branch/ref |
5927
+ | `--repo <r>` | all | Filter by repository |
5928
+ | `--trigger <t>` | all | Filter by trigger type |
5929
+ | `--source <routingKey>` | all | Filter by source routing key |
5930
+ | `--since <ts>` | none | Only runs since this ISO-8601 or epoch ms |
5931
+ | `--page <n>` | `1` | Page number (server page size is fixed at 20) |
5932
+ | `--json` | `false` | Machine-readable JSON output |
5933
+
5934
+ ```bash
5935
+ kici runs list
5936
+ kici runs list --status running
5937
+ kici runs list --workflow ci --branch main
5938
+ kici runs list --json | jq '.runs[].runId'
5939
+ ```
5940
+
5941
+ #### kici runs show
5942
+
5943
+ Show a run's summary header plus its jobs-and-steps tree (name, status,
5944
+ duration, exit code). If the run id is not on the Platform but exists in your
5945
+ local run history (from `kici run local`), the local record is shown instead.
5946
+
5947
+ ```bash
5948
+ kici runs show <run-id> [options]
5949
+ ```
5950
+
5951
+ | Option | Default | Description |
5952
+ | -------- | ------- | ---------------------------- |
5953
+ | `--json` | `false` | Machine-readable JSON output |
5954
+
5955
+ ```bash
5956
+ kici runs show abc123
5957
+ kici runs show abc123 --json
5958
+ ```
5959
+
5960
+ #### kici runs logs
5961
+
5962
+ Print each job/step's log lines in order, with headers.
5963
+
5964
+ ```bash
5965
+ kici runs logs <run-id> [options]
5966
+ ```
5967
+
5968
+ | Option | Default | Description |
5969
+ | -------------- | ------- | -------------------------------------- |
5970
+ | `--job <name>` | all | Only print logs for this job |
5971
+ | `-f, --follow` | `false` | Tail logs for a live run until it ends |
5972
+ | `--json` | `false` | Machine-readable JSON output |
5973
+
5974
+ ```bash
5975
+ kici runs logs abc123
5976
+ kici runs logs abc123 --job build
5977
+ kici runs logs abc123 --follow
5978
+ ```
5979
+
5980
+ #### kici runs rerun
5981
+
5982
+ Re-trigger a completed run. Prints the new run id. The server enforces a short
5983
+ cooldown between reruns of the same run.
5984
+
5985
+ ```bash
5986
+ kici runs rerun <run-id> [options]
5987
+ ```
5988
+
5989
+ | Option | Default | Description |
5990
+ | -------- | ------- | ---------------------------- |
5991
+ | `--json` | `false` | Machine-readable JSON output |
5992
+
5993
+ ```bash
5994
+ kici runs rerun abc123
5995
+ ```
5996
+
5997
+ #### kici runs cancel
5998
+
5999
+ Cancel a single run, or all in-progress runs on a branch.
6000
+
6001
+ ```bash
6002
+ kici runs cancel [run-id] [options]
6003
+ ```
5355
6004
 
5356
6005
  | Argument | Required | Description |
5357
6006
  | -------- | -------- | ---------------- |
5358
6007
  | `run-id` | no | Run ID to cancel |
5359
6008
 
5360
- **Options:**
5361
-
5362
6009
  | Option | Default | Description |
5363
6010
  | ----------------- | ------- | ------------------------------------------- |
5364
6011
  | `--force` | `false` | Force cancel (kill immediately, skip hooks) |
5365
6012
  | `--branch <name>` | none | Cancel all in-progress runs on this branch |
5366
6013
 
5367
- **Examples:**
5368
-
5369
6014
  ```bash
5370
- # Cancel a specific run
5371
- kici cancel abc123
5372
-
5373
- # Force cancel (kill immediately)
5374
- kici cancel abc123 --force
5375
-
5376
- # Cancel all runs on a branch
5377
- kici cancel --branch feature/wip
6015
+ kici runs cancel abc123
6016
+ kici runs cancel abc123 --force
6017
+ kici runs cancel --branch feature/wip
5378
6018
  ```
5379
6019
 
6020
+ When `--json` is set on any of these commands, `kici` emits only the JSON
6021
+ document on stdout — the `kici v<version>` banner is suppressed — so the output
6022
+ is safe to pipe into `jq` or `JSON.parse`. The same holds for the other
6023
+ `--json` commands (`kici run remote --json`, `kici workflows list --json`) and
6024
+ for `--quiet`.
6025
+
5380
6026
  ### kici approve
5381
6027
 
5382
6028
  Approve a held [approval gate](approvals.md) so the run resumes. Identify the held element by run ID, optionally narrowed to a job and step.
@@ -5779,6 +6425,70 @@ kici admin drain-worker --url http://worker-2.internal:10143
5779
6425
  | 0 | Drain request accepted |
5780
6426
  | 1 | Error (unreachable or request fail) |
5781
6427
 
6428
+ ### kici verify-attestation
6429
+
6430
+ Verify a KiCI build-provenance attestation bundle offline. A bundle is the signed package a workflow step produces via `ctx.attestProvenance(...)`: a DSSE-wrapped SLSA in-toto statement, the ephemeral public key that signed it, and the KiCI identity token that anchors the build context. For the end-to-end attest → verify → view journey, see the [build provenance guide](./provenance.md). Verification establishes the full chain — the identity token verifies against the trusted issuer's JWKS, the DSSE signature verifies against the bundled key, and the statement's build context must match the token's claims (a mismatch is a hard failure). When an `[artifact]` is given, its SHA-256 digest is also matched against the attestation subject.
6431
+
6432
+ ```bash
6433
+ kici verify-attestation [artifact] --bundle <path-or-url> --trust-root <url-or-file> [options]
6434
+ ```
6435
+
6436
+ **Arguments:**
6437
+
6438
+ | Argument | Required | Description |
6439
+ | ------------ | -------- | --------------------------------------------------------------------- |
6440
+ | `[artifact]` | no | Artifact path to digest-check against the attestation subject digest. |
6441
+
6442
+ **Options:**
6443
+
6444
+ | Option | Required | Description |
6445
+ | ---------------------------- | -------- | ----------------------------------------------------------------------------------------- |
6446
+ | `--bundle <path-or-url>` | yes | Path or `http(s)` URL to the attestation bundle JSON. |
6447
+ | `--trust-root <url-or-file>` | yes | Trusted issuer (see below). The token issuer is pinned to it, never taken from the token. |
6448
+ | `--audience <aud>` | no | Expected token audience (defaults to the KiCI provenance audience). |
6449
+ | `--json` | no | Print the structured verification result as JSON instead of human-readable output. |
6450
+
6451
+ **Trust root:** the verifier never trusts the issuer named inside the token — you supply the trusted issuer out-of-band via `--trust-root`, in one of two forms:
6452
+
6453
+ - **Online — an HTTPS issuer URL.** The verifier fetches `<url>/.well-known/openid-configuration`, reads its `issuer` and `jwks_uri`, and fetches the JWKS. The token's `iss` is pinned to the discovery document's `issuer`.
6454
+ - **Offline — a self-contained trust-root file.** A local JSON file with the issuer and JWKS inlined, so no network access is needed (air-gapped verification):
6455
+
6456
+ ```json
6457
+ {
6458
+ "issuer": "https://platform.example/issuer",
6459
+ "jwks": {
6460
+ "keys": [
6461
+ { "kty": "EC", "crv": "P-256", "x": "...", "y": "...", "alg": "ES256", "kid": "..." }
6462
+ ]
6463
+ }
6464
+ }
6465
+ ```
6466
+
6467
+ **Examples:**
6468
+
6469
+ ```bash
6470
+ # Online: verify a bundle against a deployed issuer, digest-checking the artifact
6471
+ kici verify-attestation ./dist/app.tgz \
6472
+ --bundle ./app.tgz.kici.json \
6473
+ --trust-root https://platform.example/issuer
6474
+
6475
+ # Offline / air-gapped: verify against a self-contained trust-root file
6476
+ kici verify-attestation ./dist/app.tgz \
6477
+ --bundle ./app.tgz.kici.json \
6478
+ --trust-root ./kici-trust-root.json
6479
+
6480
+ # Machine-readable result for scripting
6481
+ kici verify-attestation --bundle ./app.tgz.kici.json \
6482
+ --trust-root https://platform.example/issuer --json
6483
+ ```
6484
+
6485
+ **Exit codes:**
6486
+
6487
+ | Code | Meaning |
6488
+ | ---- | ----------------------------------------------------------------------------------- |
6489
+ | 0 | Verified — signature, identity, build context (and digest, if checked) all pass |
6490
+ | 1 | Not verified, or an error (missing flag, unreadable bundle, unreachable trust root) |
6491
+
5782
6492
  ## Workflow discovery
5783
6493
 
5784
6494
  The CLI discovers workflows by scanning `.kici/workflows/*.ts` (or `.mjs` in MJS mode). Each file should `export default` a single workflow:
@@ -6110,7 +6820,7 @@ The lock file (`kici.lock.json`) is a JSON file with the following top-level fie
6110
6820
 
6111
6821
  | Field | Description |
6112
6822
  | --------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
6113
- | `schemaVersion` | Lock file schema version (currently 15). Incremented on breaking format changes. |
6823
+ | `schemaVersion` | Lock file schema version (currently 17). Incremented on breaking format changes. |
6114
6824
  | `source` | Reference to the source file and export (e.g., `{ file: “.kici/workflows/ci.ts”, export: “#default” }`). |
6115
6825
  | `contentHash` | SHA-256 of the serialized lock file content (excluding itself). Changes when any workflow, trigger, or job changes. |
6116
6826
  | `lockfileHash` | SHA-256 of the detected package manager's lockfile, used as the dependency cache key. The lockfile is `.kici/package-lock.json` for npm, or the repo-root `pnpm-lock.yaml` / `yarn.lock` for a pnpm/yarn workspace; the hash input is prefixed with the manager name so a manager change is a guaranteed cache miss. Omitted when no lockfile exists. |
@@ -6238,8 +6948,8 @@ The command is remote-only -- all execution happens on the orchestrator and agen
6238
6948
  :::note[Orchestrator prerequisite: cache storage]
6239
6949
  `kici run remote` uploads your working-tree overlay to the orchestrator's **cache storage** via a pre-signed URL, and the agent fetches it from there (see [Repo state transfer](#repo-state-transfer)). The target orchestrator must therefore have cache storage enabled (`KICI_STORAGE_TYPE` = `s3` or `filesystem`).
6240
6950
 
6241
- - **The [Docker / Podman quickstart](quickstart/compose.md) wires this up for you** it ships a SeaweedFS service, so `kici run remote` works there out of the box (see its "run a workflow without pushing" step).
6242
- - **The [bare-metal quickstart](quickstart/bare-metal.md) does not configure storage by default** — enable a backend before using `kici run remote`:
6951
+ - **Both quickstarts wire this up for you** — the [Docker / Podman quickstart](quickstart/compose.md) and the [bare-metal quickstart](quickstart/bare-metal.md) each ship a SeaweedFS object store and pre-fill the orchestrator's `KICI_STORAGE_*` block, so `kici run remote` works out of the box (see each guide's "run a workflow without pushing" step).
6952
+ - **A hand-rolled orchestrator deploy does not configure storage by default** — enable a backend before using `kici run remote`:
6243
6953
  - **`filesystem`** — simplest for a single-host orchestrator: set `KICI_STORAGE_TYPE=filesystem` and `KICI_STORAGE_FS_PATH=/var/lib/kici/cache`. No external service needed; blobs are served through the orchestrator's own HMAC-signed HTTP route.
6244
6954
  - **`s3`** — any S3-compatible bucket. **A non-public / self-hosted endpoint works**: set `KICI_STORAGE_TYPE=s3`, `KICI_STORAGE_BUCKET`, `KICI_STORAGE_ENDPOINT=https://your-endpoint` and (for most self-hosted services) `KICI_STORAGE_FORCE_PATH_STYLE=true`. If the developer machine running `kici run remote` reaches the bucket at a different address than the orchestrator, set `KICI_STORAGE_UPLOAD_ENDPOINT` to the developer-reachable address; if agents reach it at yet another address (e.g. agents in containers), set `KICI_STORAGE_EXTERNAL_ENDPOINT` to the agent-routable URL.
6245
6955
 
@@ -6428,7 +7138,7 @@ kici run remote push-main --junit results.xml
6428
7138
  kici run remote push-main --no-wait
6429
7139
 
6430
7140
  # Check status later
6431
- kici status <run-id>
7141
+ kici runs show <run-id>
6432
7142
  ```
6433
7143
 
6434
7144
  ### Cancellation
@@ -6596,17 +7306,17 @@ kici run remote --history
6596
7306
  ### Run details
6597
7307
 
6598
7308
  ```bash
6599
- # Show run summary (tries orchestrator first, falls back to local history)
6600
- kici status <run-id>
7309
+ # Show run summary (reads the Platform, falls back to local history)
7310
+ kici runs show <run-id>
6601
7311
 
6602
7312
  # Show full logs
6603
- kici status <run-id> --logs
7313
+ kici runs logs <run-id>
6604
7314
 
6605
7315
  # Show logs for a specific job
6606
- kici status <run-id> --logs --job build
7316
+ kici runs logs <run-id> --job build
6607
7317
 
6608
7318
  # Machine-readable output
6609
- kici status <run-id> --json
7319
+ kici runs show <run-id> --json
6610
7320
  ```
6611
7321
 
6612
7322
  ## Scaffolding with kici init
@@ -6723,6 +7433,178 @@ To actually change how you sign in, use your account console as described above.
6723
7433
 
6724
7434
  ---
6725
7435
 
7436
+ ## Approval gates
7437
+
7438
+ Source: https://docs.kici.dev/user/approvals/
7439
+
7440
+ An **approval gate** pauses execution until an authorized person approves it. Execution resumes from exactly where it paused; a rejection (or an expired hold) fails the run.
7441
+
7442
+ You declare a gate in your workflow with `requireApproval`. It is available at three levels of granularity:
7443
+
7444
+ - **Step** — pause mid-job, before a specific step runs. The agent holds the live workspace (with all prior-step state intact) for the duration of the wait.
7445
+ - **Job** — hold the job before any of its steps run.
7446
+ - **Workflow** — hold the whole run before any job is dispatched.
7447
+
7448
+ Approvers are named as **teams** and **users**. A team is an operator-defined group of org members; your workflow code may name a team but can never change its membership, which is what makes a team clause a real gate rather than a suggestion. See [Approval gates (operator guide)](../operator/approvals.md) for how operators define teams, the approval queue, and expiry; see [the architecture overview](../architecture/approvals.md) for how a hold is evaluated and resumed.
7449
+
7450
+ ## Quick start
7451
+
7452
+ Hold a deploy job until a member of the `leads` team approves:
7453
+
7454
+ ```typescript
7455
+ import { workflow, job, step, push } from '@kici-dev/sdk';
7456
+
7457
+ export default workflow('deploy', {
7458
+ on: [push({ branches: ['main'] })],
7459
+ jobs: [
7460
+ job('deploy-production', {
7461
+ runsOn: 'default',
7462
+ requireApproval: [{ team: 'leads' }],
7463
+ steps: [step('deploy', async (ctx) => ctx.$`deploy --prod`)],
7464
+ }),
7465
+ ],
7466
+ });
7467
+ ```
7468
+
7469
+ When the run reaches this job, it is held instead of dispatched. The held run appears in the dashboard approval queue and can be released from there or with the [`kici approve`](#approving-from-the-cli) command. Once a member of `leads` approves, the job dispatches normally.
7470
+
7471
+ ## The `requireApproval` field
7472
+
7473
+ `requireApproval` accepts three forms.
7474
+
7475
+ ### Shorthand: `true`
7476
+
7477
+ ```typescript
7478
+ job('deploy', {
7479
+ runsOn: 'default',
7480
+ requireApproval: true,
7481
+ steps: [
7482
+ /* ... */
7483
+ ],
7484
+ });
7485
+ ```
7486
+
7487
+ `requireApproval: true` holds the element until **any** org member who can act on approvals signs off — anyone with the `environments:write` or `ci_trust:write` permission. Use it when you want a manual gate without restricting who may release it.
7488
+
7489
+ ### Approver list (AND)
7490
+
7491
+ ```typescript
7492
+ requireApproval: [{ team: 'leads' }, { user: 'cto' }],
7493
+ ```
7494
+
7495
+ A list of approver clauses is an **AND** list: every clause must be satisfied before the element is released.
7496
+
7497
+ - `{ team: 'leads' }` is satisfied once **any** member of the `leads` team approves.
7498
+ - `{ user: 'cto' }` is satisfied once the user `cto` approves.
7499
+
7500
+ A single approver may satisfy more than one clause. If `cto` is also a member of `leads`, one approval from `cto` satisfies both `{ team: 'leads' }` and `{ user: 'cto' }`, releasing the element. A user is named by their KiCI user identifier (their linked identity), and a team by its name as defined by your operator.
7501
+
7502
+ There is no OR or nested logic — clauses are always a flat AND list.
7503
+
7504
+ ### Object form: reason and timeout
7505
+
7506
+ ```typescript
7507
+ requireApproval: {
7508
+ approvers: [{ team: 'security' }, { team: 'leads' }],
7509
+ reason: 'Production deploy requires security + leads sign-off',
7510
+ timeout: 7200, // seconds
7511
+ },
7512
+ ```
7513
+
7514
+ | Field | Type | Description |
7515
+ | ----------- | ------------------ | --------------------------------------------------------------------------------------------------- |
7516
+ | `approvers` | `ApproverClause[]` | The AND list of `{ team }` / `{ user }` clauses. An empty list means "any approval-capable member". |
7517
+ | `reason` | `string` | A human-readable label shown in the dashboard queue and the held-for-approval status check. |
7518
+ | `timeout` | `number` | Per-gate expiry in **seconds**, overriding the org default. On expiry the element is rejected. |
7519
+
7520
+ When `timeout` is omitted, the gate uses the org's default approval expiry (set by the operator). On expiry, the held element is rejected and the run fails — see [expiry](../operator/approvals.md#expiry).
7521
+
7522
+ ## Granularity
7523
+
7524
+ The same `requireApproval` field is accepted on a workflow, a job, and a step.
7525
+
7526
+ ### Workflow-level
7527
+
7528
+ A workflow-level gate holds the entire run before any job is dispatched:
7529
+
7530
+ ```typescript
7531
+ export default workflow('release', {
7532
+ on: [push({ branches: ['main'] })],
7533
+ requireApproval: [{ team: 'release-managers' }],
7534
+ jobs: [buildJob, publishJob],
7535
+ });
7536
+ ```
7537
+
7538
+ ### Job-level
7539
+
7540
+ A job-level gate holds just that job; other jobs in the run proceed normally:
7541
+
7542
+ ```typescript
7543
+ job('publish', {
7544
+ runsOn: 'default',
7545
+ requireApproval: [{ team: 'leads' }],
7546
+ steps: [
7547
+ /* ... */
7548
+ ],
7549
+ });
7550
+ ```
7551
+
7552
+ ### Step-level
7553
+
7554
+ A step-level gate pauses mid-job, immediately before the named step. Earlier steps in the job have already run and their workspace state is preserved across the wait:
7555
+
7556
+ ```typescript
7557
+ job('migrate-and-deploy', {
7558
+ runsOn: 'default',
7559
+ steps: [
7560
+ step('build-plan', async (ctx) => ctx.$`./gen-migration-plan.sh`),
7561
+ step('apply-migration', {
7562
+ requireApproval: [{ team: 'dba' }],
7563
+ run: async (ctx) => ctx.$`./apply-migration.sh`,
7564
+ }),
7565
+ step('deploy', async (ctx) => ctx.$`deploy --prod`),
7566
+ ],
7567
+ });
7568
+ ```
7569
+
7570
+ Here `build-plan` runs, then the job pauses for a `dba` approval. On approval, `apply-migration` runs against the exact workspace `build-plan` produced, followed by `deploy`. A rejection or expiry fails the job.
7571
+
7572
+ Because a step-level hold keeps an agent and its workspace occupied for the whole human wait, prefer job- or workflow-level gates when you do not need prior-step state, and keep step-level timeouts short. See the [operator note on agent occupancy](../operator/approvals.md#agent-occupancy-during-step-level-holds).
7573
+
7574
+ ## Mandatory vs. explicit gates
7575
+
7576
+ `requireApproval` is the **explicit** gate — a deliberate "pause for a human here" written by the workflow author. It composes with the **mandatory** gate an operator can attach to a protected environment via required reviewers (see [Environments](environments.md#required-reviewers)). When both apply to the same job, all clauses from both sources must be satisfied before the job is released. The two funnel into one held-element mechanism, so the dashboard queue and `kici approve` work the same way regardless of which source held the element.
7577
+
7578
+ ## Approving from the CLI
7579
+
7580
+ Approve or reject a held element with the `kici` CLI:
7581
+
7582
+ ```bash
7583
+ # Approve a workflow-level hold
7584
+ kici approve <run-id>
7585
+
7586
+ # Approve a held job
7587
+ kici approve <run-id> --job deploy-production
7588
+
7589
+ # Approve a held step
7590
+ kici approve <run-id> --job migrate-and-deploy --step apply-migration
7591
+
7592
+ # Reject (a reason is required)
7593
+ kici reject <run-id> --job deploy-production --reason "Wrong release branch"
7594
+ ```
7595
+
7596
+ You must be eligible for at least one unsatisfied clause — being a member of a named team or being a named user. The orchestrator verifies eligibility against the operator-defined teams, so naming a team in your workflow can never let an ineligible person release the gate. The command reports whether the element was released, how many clauses remain, or that it was rejected. See [`kici approve`](cli-reference.md#kici-approve) for the full command reference.
7597
+
7598
+ You can also approve from the dashboard approval queue. See [Dashboard](dashboard/environments-and-secrets.md#approval-queue).
7599
+
7600
+ ## See also
7601
+
7602
+ - [Environments](environments.md) — operator-required reviewers on protected environments.
7603
+ - [Approval gates (operator guide)](../operator/approvals.md) — teams, the approval queue, expiry, and self-approval.
7604
+ - [Approval gates (architecture)](../architecture/approvals.md) — the unified hold model and the step-level round-trip.
7605
+
7606
+ ---
7607
+
6726
7608
  ## Concurrency groups
6727
7609
 
6728
7610
  Source: https://docs.kici.dev/user/concurrency/
@@ -6820,1164 +7702,152 @@ Run #1 starts deploying -> running
6820
7702
  Run #2 arrives for same group -> queued ("Waiting for deploy-main (1 ahead)")
6821
7703
  Run #1 completes -> success
6822
7704
  Run #2 starts -> running
6823
- ```
6824
-
6825
- In queue mode, the agent that picked up the queued run **stays connected** to the orchestrator and parks on a long-poll wait. When the holder finishes (success, failure, or cancel), the orchestrator dequeues the FIFO-next entry and pushes a `proceed` notification over the same WebSocket; the queued agent then continues with normal step execution against the workspace it already has. The agent's slot is therefore held for the duration of the queue wait — bound by `KICI_CONCURRENCY_WAIT_TIMEOUT_MS` (default 1 hour).
6826
-
6827
- ```typescript
6828
- workflow('migrate-db', {
6829
- concurrency: {
6830
- group: () => 'migrations',
6831
- cancelInProgress: false,
6832
- max: 1,
6833
- },
6834
- jobs: [
6835
- /* ... */
6836
- ],
6837
- });
6838
- ```
6839
-
6840
- The dashboard will show a "Queued" badge with the reason: "Waiting for deploy-main (1 ahead)".
6841
-
6842
- ## Max concurrent runs
6843
-
6844
- The `max` field controls how many runs can execute simultaneously in the same group:
6845
-
6846
- ```typescript
6847
- // Allow up to 3 parallel test runs per branch
6848
- workflow('test', {
6849
- concurrency: {
6850
- group: (ctx) => `test-${ctx.branch}`,
6851
- cancelInProgress: false,
6852
- max: 3,
6853
- },
6854
- jobs: [
6855
- /* ... */
6856
- ],
6857
- });
6858
- ```
6859
-
6860
- When `max: 1` (default), runs are fully serialized within the group.
6861
-
6862
- ## Group key examples
6863
-
6864
- ### Deploy per environment
6865
-
6866
- ```typescript
6867
- workflow('deploy', {
6868
- concurrency: {
6869
- group: (ctx) => `deploy-${ctx.branch}`,
6870
- cancelInProgress: true,
6871
- },
6872
- jobs: [
6873
- job('deploy-staging', {
6874
- runsOn: 'linux',
6875
- environment: 'staging',
6876
- steps: [
6877
- /* ... */
6878
- ],
6879
- }),
6880
- ],
6881
- });
6882
- ```
6883
-
6884
- ### Global singleton
6885
-
6886
- ```typescript
6887
- // Only one migration can run at a time, regardless of branch
6888
- workflow('migrate', {
6889
- concurrency: {
6890
- group: () => 'db-migration',
6891
- cancelInProgress: false,
6892
- },
6893
- jobs: [
6894
- /* ... */
6895
- ],
6896
- });
6897
- ```
6898
-
6899
- ### Environment-aware groups
6900
-
6901
- ```typescript
6902
- // Serialize deploys per environment
6903
- workflow('deploy', {
6904
- concurrency: {
6905
- group: (ctx) => {
6906
- const env = ctx.branch === 'main' ? 'production' : 'staging';
6907
- return `deploy-${env}`;
6908
- },
6909
- cancelInProgress: true,
6910
- },
6911
- jobs: [
6912
- /* ... */
6913
- ],
6914
- });
6915
- ```
6916
-
6917
- ## Interaction with environment protection
6918
-
6919
- When a workflow has both `concurrency` and `environment` protection rules:
6920
-
6921
- 1. Environment protection gates (required reviewers, wait timer) apply first
6922
- 2. Concurrency group check happens after protection gates pass
6923
- 3. If the run is queued by concurrency, it keeps its protection approval
6924
-
6925
- This means a run that passed approval won't need re-approval if it gets queued by concurrency.
6926
-
6927
- ## Cancelling queued runs
6928
-
6929
- Queued runs can be cancelled before they start executing. The cancel request removes them from the queue immediately -- they don't go through the grace period since no step is running.
6930
-
6931
- ## Job-level concurrency groups
6932
-
6933
- In addition to workflow-level concurrency, individual jobs can define their own concurrency group via the `concurrencyGroup` property. This controls concurrent execution at the job level rather than the workflow level. See [Environments — concurrency groups](environments.md#concurrency-groups) for details.
6934
-
6935
- ## Local execution
6936
-
6937
- `kici run local` honors workflow-level `concurrency` per-machine, per-user. The `group` callback is evaluated against the simulated event identically to the remote orchestrator path; `cancelInProgress` carries the same semantics — `true` interrupts the holder via `SIGTERM` (escalating to `SIGKILL` after a grace window) and proceeds with the new run, while `false` queues the new invocation in FIFO order until the holder finishes.
6938
-
6939
- Coordination is local only. Running the same workflow on two different machines does not serialize across them — that requires the orchestrator. For full cross-host enforcement (queueing across agents, dashboard visibility, `max > 1`), use `kici run remote` against a deployed orchestrator.
6940
-
6941
- Lock files live under `$XDG_RUNTIME_DIR/kici-local-locks/` on Linux, falling back to `os.tmpdir()/kici-local-locks-<uid>/`. A workflow whose `group` callback throws aborts the run with a clear error rather than running unprotected. See [`kici run local` — Concurrency enforcement](cli-reference.md#concurrency-enforcement) for the `KICI_LOCAL_LOCK_KILL_GRACE_MS` override and the diagnostic output emitted while contending on a busy lock.
6942
-
6943
- The `kici run local --concurrency <n>` flag is a separate concept — it caps **job-level** parallelism within a single run (how many jobs from one workflow run at once), not cross-run serialization.
6944
-
6945
- ---
6946
-
6947
- _Source: `packages/sdk/src/types.ts` (WorkflowOptions.concurrency, JobOptions.concurrencyGroup)_
6948
-
6949
- ---
6950
-
6951
- ## Dashboard
6952
-
6953
- Source: https://docs.kici.dev/user/dashboard/
6954
-
6955
- The KiCI dashboard is a web-based interface for monitoring workflow runs, inspecting job and step details, and reading log output. It is a browser single-page application that authenticates via OIDC and communicates with the Platform tier through REST API endpoints.
6956
-
6957
- ## Getting started
6958
-
6959
- <!-- help:getting-started-overview#getting-started -->
6960
-
6961
- The getting-started page is a six-step checklist that takes you from zero to your first workflow run.
6962
-
6963
- - **Self-checked steps** -- install the CLI, scaffold a workflow, and run it locally. These run on your own machine, so you tick them off yourself; the dashboard remembers your choices in the browser.
6964
- - **Auto-detected steps** -- connect an orchestrator, add a webhook source, and trigger your first run. These tick automatically as the dashboard observes the matching activity in your organization.
6965
-
6966
- Each step links to the relevant settings page or documentation. A progress bar tracks overall completion, and the sidebar entry shows a `done/total` badge until you finish or dismiss reminders.
6967
-
6968
- <!-- /help:getting-started-overview -->
6969
-
6970
- When you first sign in to a brand-new organization with no orchestrator, no webhook source, and no runs, the dashboard opens this page automatically. Once your organization has any activity, the run list becomes your landing page instead. The **Getting started** sidebar entry stays available so you can return to the checklist at any time.
6971
-
6972
- The six steps are:
6973
-
6974
- 1. **Install the kici CLI** -- `npm install -g kici`.
6975
- 2. **Create a workflow** -- `kici init` scaffolds a `.kici/` directory in your repository.
6976
- 3. **Run a workflow locally** -- `kici run local pr:open` executes a workflow on your machine with no orchestrator required.
6977
- 4. **Connect an orchestrator** -- deploy an orchestrator and connect it with a join token from **Settings → Orchestrator keys**.
6978
- 5. **Add a webhook source** -- register a source under **Settings → Sources** so pushes and pull requests trigger runs.
6979
- 6. **Trigger your first run** -- push to your repository to produce your first run through the relay.
6980
-
6981
- ## Navigation
6982
-
6983
- ### Sidebar
6984
-
6985
- The left sidebar provides persistent navigation across all org-scoped pages:
6986
-
6987
- - **Org switcher** -- dropdown at the top to switch between organizations
6988
- - **Getting started** -- onboarding checklist (shows a `done/total` badge until complete or dismissed)
6989
- - **Runs** -- the default landing page, showing your workflow run history
6990
- - **Workflows** -- permanently registered workflows listening for events
6991
- - **Diagnostics** -- infrastructure health, execution metrics, and recent errors
6992
- - **Metrics** -- time-series charts of orchestrator health (dispatch & agents, execution, webhooks, caching, logs, errors), scoped to this org
6993
- - **Environments** -- deployment environments with protection rules
6994
- - **Secrets** -- secret scope management with environment bindings
6995
- - **Approval queue** -- held runs pending approval (shows a badge with pending count)
6996
- - **Activity** -- federated forensic log merging upstream tenant-plane mutations and orchestrator reads (`access_log`) into one chronological stream
6997
- - **DLQ** -- dead-letter queue of internal events whose dispatch retries were exhausted (shows a badge with the current depth)
6998
- - **Settings** -- organization settings with tabbed sub-pages
6999
-
7000
- The sidebar footer shows the WebSocket connection indicator, your user profile, UTC/local time toggle, theme toggle, and a collapse button.
7001
-
7002
- <!-- help:sidebar-build-info#sidebar -->
7003
-
7004
- Below the KiCI logo, the sidebar shows build information for both the dashboard UI and the Platform API backend:
7005
-
7006
- - Git commit hash.
7007
- - Relative build timestamp (e.g. "2h ago").
7008
-
7009
- This makes it easy to confirm which version is currently deployed.
7010
-
7011
- <!-- /help:sidebar-build-info -->
7012
-
7013
- ### Mobile navigation
7014
-
7015
- On screens narrower than 768px (the `sm` breakpoint), the sidebar collapses and is replaced by a bottom tab bar with six navigation items: Runs, Workflows, Envs (environments), Secrets, Health (diagnostics), and Settings. Note that the mobile tab bar shows a subset of the full sidebar navigation -- activity and approval queue are only available in the full desktop sidebar.
7016
-
7017
- <!-- help:run-list-overview#run-list -->
7018
-
7019
- The run list is your organization's default landing page, showing all workflow runs with status, trigger, branch, and timing. Use filters and sorting to find specific runs, or enable commit grouping to see all runs triggered by a single push.
7020
-
7021
- <!-- /help:run-list-overview -->
7022
-
7023
- <!-- help:run-list-commit-grouping#commit-grouped-view -->
7024
-
7025
- Commit grouping collapses runs that share the same commit SHA under a single header. This is useful when a push triggers multiple workflows -- you can see their aggregate status at a glance instead of scanning individual rows.
7026
-
7027
- <!-- /help:run-list-commit-grouping -->
7028
-
7029
- ## Run list
7030
-
7031
- The run list is the default page when entering an organization (`/orgs/:customerId/runs`).
7032
-
7033
- ### Columns
7034
-
7035
- Each run is displayed in a table row (desktop) or card (mobile) with:
7036
-
7037
- - **Status** -- colored badge (green = success, red = failed/error/timed out, amber = running/cancelling, yellow = queued/pending, gray = cancelled/skipped)
7038
- - **Trigger** -- icon indicating the event type (push, pull request, tag, dispatch, etc.)
7039
- - **Workflow** -- the workflow name from your `.kici/workflows/` directory
7040
- - **Branch** -- the git ref that triggered the run
7041
- - **Commit** -- the first 7 characters of the commit SHA, linked to the provider (GitHub)
7042
- - **Duration** -- how long the run took (e.g. "2m 30s")
7043
- - **Time** -- relative timestamp (e.g. "5 minutes ago")
7044
-
7045
- ### Filters
7046
-
7047
- Dropdown filters appear above the table:
7048
-
7049
- - **Status** -- filter by success, failed, running, or cancelled
7050
- - **Workflow** -- filter by workflow name
7051
- - **Branch** -- filter by git branch
7052
- - **Repository** -- filter by repository
7053
-
7054
- A "More filters" button reveals additional filters:
7055
-
7056
- - **Trigger type** -- filter by push, pull_request, tag, dispatch, etc.
7057
-
7058
- Filters persist in URL query parameters (e.g. `/runs?status=failed&branch=main`), making filtered views shareable and bookmark-friendly. A "Clear filters" button appears when any filter is active.
7059
-
7060
- ### Sorting
7061
-
7062
- Click any column header to sort the table by that column. Clicking the same header toggles between ascending and descending order. The current sort is reflected in the URL (e.g. `?sort=workflowName&dir=desc`), so sorted views are shareable.
7063
-
7064
- Sorting is server-side -- the API returns results in the requested order.
7065
-
7066
- ### Column visibility
7067
-
7068
- A gear icon button (labeled "Toggle columns") next to the filter bar opens a menu of toggleable columns. Uncheck a column to hide it from the table. Column visibility preferences are saved per organization in `localStorage`.
7069
-
7070
- ### Commit grouped view
7071
-
7072
- A "Group by commit" toggle switch groups runs by their commit SHA. When enabled, runs sharing the same commit are collapsed under a group header showing the commit SHA (first 7 characters), commit message, and aggregate status dots. This is useful for seeing all workflow runs triggered by a single push.
7073
-
7074
- ### Compile indicator
7075
-
7076
- Runs where the lock file was recompiled during execution show a hammer icon next to the workflow name. Hover over the icon to see the tooltip "Lock file recompiled".
7077
-
7078
- ### Pagination
7079
-
7080
- The run list shows 20 runs per page with numbered pagination controls. A footer displays the current range and total count (e.g. "Showing 1-20 of 237 runs").
7081
-
7082
- ### Empty states
7083
-
7084
- - **No runs, WS disconnected** -- "No orchestrator connected" with guidance to check orchestrator configuration and a link to settings.
7085
- - **No runs, WS connected** -- "No runs yet" with guidance to push code to trigger a workflow run.
7086
- - **No filter matches** -- "No matching runs" with guidance to adjust filters.
7087
-
7088
- <!-- help:run-detail-job-tree#job-tree -->
7089
-
7090
- The job tree shows the hierarchical structure of your run's jobs and steps. Click a job to see combined logs from all its steps, or expand a job to select an individual step.
7091
-
7092
- Failed runs auto-expand the first failed job for quick diagnosis.
7093
-
7094
- <!-- /help:run-detail-job-tree -->
7095
-
7096
- <!-- help:run-detail-metadata#metadata -->
7097
-
7098
- The metadata panel displays detailed context about the selected run, job, or step:
7099
-
7100
- - IDs, status, duration.
7101
- - Orchestrator and agent assignment.
7102
- - Matrix values (when present).
7103
- - Provider links — commit SHA, trigger event, workflow source file on GitHub.
7104
-
7105
- Use it as the quick-reference card when you need to jump from the dashboard to the underlying VCS or infrastructure.
7106
-
7107
- <!-- /help:run-detail-metadata -->
7108
-
7109
- <!-- help:run-detail-source#metadata -->
7110
-
7111
- The source row identifies which webhook source produced this run. Orchestrators register sources at startup with a friendly name and a fine-grained subtype — GitHub App, generic webhook, universal Git, or internal.
7112
-
7113
- The dimmed routing key under the name (e.g. `github:12345` or `generic:org:src-id`) is the unique identifier Platform uses to route the webhook back to your orchestrator.
7114
-
7115
- Two repos with the same path served by different sources are distinguished here, so you can tell at a glance which deployment a run came from.
7116
-
7117
- <!-- /help:run-detail-source -->
7118
-
7119
- <!-- help:run-detail-job-labels#metadata -->
7120
-
7121
- Labels show the routing constraints used to match this job to an agent.
7122
-
7123
- Label categories:
7124
-
7125
- - **Platform labels:** `kici:os:linux`, `kici:arch:x64`, etc.
7126
- - **Scaler labels:** added by the scaler that provisioned the agent.
7127
- - **Role labels:** e.g. `kici:role:builder`.
7128
- - **Custom labels:** anything you set via `runsOn` in your workflow definition.
7129
-
7130
- <!-- /help:run-detail-job-labels -->
7131
-
7132
- <!-- help:run-detail-trust-context#metadata -->
7133
-
7134
- The trust context shows the security evaluation for PR-triggered runs:
7135
-
7136
- - **Trust tier:** trusted, known, or unknown contributor.
7137
- - **Lock file source:** head branch or base branch.
7138
- - **Secrets access level:** what the run was permitted to read.
7139
-
7140
- Use this to understand why a run was held for approval or ran with restricted permissions.
7141
-
7142
- <!-- /help:run-detail-trust-context -->
7143
-
7144
- <!-- help:summary-job-contexts#tabs -->
7145
-
7146
- The job contexts section in the run summary shows execution context per job:
7147
-
7148
- - **Sandbox type** — container, Firecracker, bare-metal.
7149
- - **Runtime environment** — image, OS, arch.
7150
- - **Dependency cache status** — hit, miss, or skipped.
7151
- - **Available secret keys** — which scopes the job could read.
7152
-
7153
- This gives a quick overview of every job's execution environment without needing to select each job individually.
7154
-
7155
- <!-- /help:summary-job-contexts -->
7156
-
7157
- <!-- help:summary-scaler-context#tabs -->
7158
-
7159
- The scaler configuration section shows the execution mode and backend-specific settings used to provision the agent that ran this job. The execution mode describes how steps run — for example, "bare-metal" means direct processes, even inside containers where the container is the isolation layer.
7160
-
7161
- Backend-specific fields:
7162
-
7163
- - **Container backends:** image name, runtime (Docker/Podman), resource limits, and network isolation settings.
7164
- - **Firecracker backends:** rootfs and kernel paths, vCPU/memory allocation, and the VM's IP address.
7165
- - **Bare-metal backends:** binary path and resource hints.
7166
-
7167
- Hover over the execution mode badge for a contextual explanation.
7168
-
7169
- <!-- /help:summary-scaler-context -->
7170
-
7171
- <!-- help:summary-job-outputs#tabs -->
7172
-
7173
- Shows plain outputs and secret output keys produced by this job.
7174
-
7175
- - **Plain outputs:** values returned by step functions, grouped by step name.
7176
- - **Secret outputs:** set via `ctx.setSecretOutput()`, shown here as masked key names — the values are encrypted and never sent to the dashboard.
7177
-
7178
- Downstream jobs that declare this job in their `needs` array can read these outputs.
7179
-
7180
- <!-- /help:summary-job-outputs -->
7181
-
7182
- <!-- help:summary-step-secrets#tabs -->
7183
-
7184
- When a step is selected, the secrets-accessed section shows which secret keys the step read via `ctx.secrets.get()` or `ctx.secrets.expose()` during execution.
7185
-
7186
- Only key names are shown, never values — use this to audit which steps access sensitive credentials.
7187
-
7188
- Available for runs executed after this feature was deployed; older runs show no data.
7189
-
7190
- <!-- /help:summary-step-secrets -->
7191
-
7192
- ## Run detail
7193
-
7194
- Click any run in the list to open its detail page (`/orgs/:customerId/runs/:runId`).
7195
-
7196
- ### Layout
7197
-
7198
- The page uses a responsive multi-panel layout that adapts to screen width:
7199
-
7200
- - **Wide desktop (>= 1200px)** -- three-panel layout with a resizable job tree (left), content area (center), and metadata sidebar (right). Two draggable dividers between the panels let you resize them. Panel sizes persist to `localStorage`.
7201
- - **Medium desktop (< 1200px)** -- two-panel layout with the job tree and content area. Metadata is accessible via a "Show metadata" drawer button.
7202
- - **Mobile (< 768px)** -- stacked layout with the job tree at the top and content below. Metadata is available as a tab alongside Logs, Payload, Timeline, and Summary.
7203
-
7204
- ### Run header
7205
-
7206
- A summary bar above the two panels shows:
7207
-
7208
- - **Breadcrumbs** -- Runs > github > owner/repo > commit SHA > #runId > workflow name (each segment is clickable and filters the run list by that dimension)
7209
- - **Status badge** -- the current run status
7210
- - **Trigger icon** -- visual indicator of the event type
7211
- - **Branch** -- the git ref with a branch icon
7212
- - **Commit SHA** -- linked to the provider's commit page
7213
- - **Duration** -- total run time
7214
- - **Timestamp** -- relative time since the run started (hover for absolute time)
7215
- - **Re-run button** -- available for terminal-state runs (success, failed, cancelled, error, timed out) triggered by webhooks. Opens a confirmation dialog before re-running on the same commit. After confirmation, navigates to the new run.
7216
- - **Cancel button** -- available for pending, running, cancelling, or queued runs. For running runs, sends a graceful cancel; for already-cancelling runs, a "Force cancel" button appears to immediately kill without cleanup.
7217
- - **Lineage badge** -- if the run is a re-run, a badge shows the parent/child relationship with a link to the original run.
7218
-
7219
- ### Job tree
7220
-
7221
- The left panel shows a tree of jobs and their steps:
7222
-
7223
- - Each job shows a **status dot**, **name**, and **duration** (live timer while running)
7224
- - Click a job row to select the job and view its combined logs (all steps merged with sticky headers)
7225
- - Click the expand chevron on a job to expand/collapse its steps
7226
- - Each step shows a **status dot**, **name**, and **duration**
7227
- - Click a step to select it and view its individual logs
7228
-
7229
- **Job-level selection** -- clicking a job row selects it and shows combined logs from all of its steps, with sticky step headers separating each step's output. This provides a unified view of the entire job's execution without needing to click through steps individually.
7230
-
7231
- **Matrix jobs** are grouped under a parent node. For example, a matrix with 3 Node.js versions appears as "Test (3 variants)" with expandable sub-entries like "Test (node:18)", "Test (node:20)", "Test (node:22)".
7232
-
7233
- **Hook steps** -- lifecycle hook steps (e.g. `onCancel`, `cleanup`, `onSuccess`) are displayed with a distinct badge to differentiate them from regular steps.
7234
-
7235
- <!-- help:run-detail-setup-jobs#job-tree -->
7236
-
7237
- **Setup jobs** are rows prefixed with `__init__`, `__build__`, or `__dynamic__`. They run before (or alongside) your workflow jobs.
7238
-
7239
- In the tree they appear with:
7240
-
7241
- - A pretty display name — `Init: foo`, `Build: foo`, `Evaluate: foo`.
7242
- - A muted "setup" visual variant that distinguishes them from regular jobs without hiding them.
7243
- - A single synthetic step-0 log that captures everything the workflow source and dynamic functions write — explicit `log.*` calls, `console.*` output, and subprocess stdout from `await $` inside a `DynamicJobFn`.
7244
-
7245
- Their elapsed time is intentionally visible: clone, dependency install, and dynamic evaluation can consume real user-observable time, and hiding it would obscure where the run is actually spending itself.
7246
-
7247
- <!-- /help:run-detail-setup-jobs -->
7248
-
7249
- **Auto-expand on failure** -- when viewing a failed run, the first failed job is automatically expanded and the failed step is selected.
7250
-
7251
- **URL sync** -- selecting a job updates the URL to `/runs/:runId/jobs/:jobId`, and selecting a step updates it to `/runs/:runId/jobs/:jobId/steps/:stepIndex`, making selections bookmarkable and shareable.
7252
-
7253
- ### Keyboard navigation
7254
-
7255
- The job tree supports keyboard navigation:
7256
-
7257
- - **Arrow Up/Down** -- move focus through tree items
7258
- - **Enter** -- select a job (show combined logs) or select a step (show step logs)
7259
- - **Escape** -- deselect the current selection and navigate to the first job
7260
-
7261
- ### Tabs
7262
-
7263
- The content area has the following tabs:
7264
-
7265
- - **Logs** (default) -- shows log output for the selected job or step
7266
- - **Payload** -- webhook payload viewer showing the raw event payload that triggered the run. This tab appears only for runs triggered by a webhook event (and re-runs of those, which copy the original payload); runs started by a schedule, manual schedule, lifecycle event, or another run carry no payload, so the tab is hidden for them
7267
- - **Timeline** -- CSS Gantt chart showing the execution timeline of all jobs, with percentage-based bars and striped animation for running jobs. A **Provisioning** milestones section between the dispatch and execution phases plots scaler lifecycle events for the run — including a **Provisioning failed** marker when the scaler could not bring an agent up
7268
- - **Summary** -- contextual overview scoped to the current selection (run-level trigger/repo/timing info, or job-level execution context with environment variables, runtime info, and sandbox details)
7269
-
7270
- On wide desktop (>= 1200px), Metadata is shown in a dedicated sidebar panel instead of as a tab.
7271
-
7272
- ### Metadata
7273
-
7274
- The metadata panel shows detailed information organized into sections:
7275
-
7276
- - **Run metadata** -- run ID, status, trigger event, branch, commit SHA (linked to provider), workflow name (linked to source file on provider), duration, and timestamps
7277
- - **Job metadata** -- job name, status, agent ID, matrix values (if present), duration
7278
- - **Step metadata** -- step name, step index, status, duration
7279
- - **Trust context** (PR-triggered runs only) -- shows the contributor's trust tier (trusted, known, or unknown), lock file source (head or base branch), and secrets access level
7280
-
7281
- Provider-specific links (e.g., GitHub commit URL, branch URL, PR link, workflow source file link) are automatically generated based on the repository context. The workflow name in the metadata panel is a clickable link to the `.kici/workflows/<name>.ts` source file on the provider (e.g. GitHub blob view).
7282
-
7283
- ### WebSocket connection indicator
7284
-
7285
- A small indicator in the sidebar footer shows the real-time WebSocket connection status:
7286
-
7287
- - **Green dot** -- connected and receiving live updates
7288
- - **Red dot (pulsing)** -- disconnected
7289
-
7290
- ## Log viewer
7291
-
7292
- The log viewer renders step output with full terminal color support.
7293
-
7294
- ### ANSI color rendering
7295
-
7296
- Log lines containing ANSI escape codes are rendered with color. Supported sequences include:
7297
-
7298
- - Standard 16 colors (red, green, blue, etc.) and bright variants
7299
- - 256-color palette
7300
- - Truecolor (24-bit RGB)
7301
- - Bold, faint, italic, underline, and inverse text
7302
-
7303
- Colors use CSS classes with a dark background (similar to a terminal), regardless of the dashboard's light/dark theme setting.
7304
-
7305
- ### Timestamps
7306
-
7307
- A clock icon button next to the search bar toggles per-line timestamps in the log viewer. When enabled, each log line shows the timestamp in the gutter alongside the line number. The timestamp format respects the UTC/local time preference. The setting persists to `localStorage`.
7308
-
7309
- ### Search
7310
-
7311
- A search bar at the top of the log viewer provides:
7312
-
7313
- - **Debounced search** -- type a query and matches are highlighted after 300ms
7314
- - **Match count** -- shows "N of M" with the current and total match count
7315
- - **Navigation** -- up/down arrows to jump between matches (also Enter/Shift+Enter)
7316
- - **Clear** -- press Escape or click the X button to clear the search
7317
- - **Wraparound** -- navigation wraps from the last match back to the first
7318
-
7319
- ### Permalink
7320
-
7321
- Click any line number in the gutter to:
7322
-
7323
- 1. Highlight that line with a blue tint
7324
- 2. Update the URL hash to `#L42` (for line 42)
7325
-
7326
- Sharing the URL scrolls the recipient directly to the highlighted line.
7327
-
7328
- ### Copy to clipboard
7329
-
7330
- Hover over any line to reveal a copy button on the right. Clicking it copies the line's **plain text** (ANSI escape codes are stripped) to the clipboard. A "Copied!" tooltip confirms the action.
7331
-
7332
- ### Live log streaming
7333
-
7334
- When viewing a running job, logs appear in real time as the agent executes steps. The dashboard maintains a WebSocket connection to the Platform tier and subscribes to log updates for the currently selected step.
7335
-
7336
- **Auto-scroll** -- new lines automatically scroll into view as they arrive. If you scroll up to review earlier output, auto-scroll pauses and a **"Jump to bottom"** button appears. Clicking it resumes auto-scroll.
7337
-
7338
- **Streaming indicator** -- a pulsing "Streaming" badge appears next to the Logs tab header while a step is actively running.
7339
-
7340
- **Completion banner** -- when a step finishes, a banner appears at the bottom of the log viewer showing the final status (success or failed) and total line count.
7341
-
7342
- **Status updates** -- the run list and run detail pages update live as jobs and steps change state. You do not need to refresh the page to see a run complete.
7343
-
7344
- **Known limitations**:
7345
-
7346
- - Live streaming requires an active WebSocket connection. Some corporate proxies may block WebSocket upgrades.
7347
- - If the WS connection drops, the dashboard reconnects automatically and refetches all cached data to catch up on missed updates.
7348
- - Log lines received during streaming are held in memory. For very long-running steps with massive output, the REST endpoint is the authoritative source for complete logs.
7349
-
7350
- ### Provisioning logs
7351
-
7352
- Above the step logs, a collapsible **Provisioning logs** section shows the orchestrator-side lifecycle of the agent that ran the job — the scaler lifecycle events emitted while bringing an agent up. It starts expanded while provisioning is in progress (no step logs yet) and collapses once steps begin producing output.
7353
-
7354
- When the scaler **fails** to provision an agent (for example a missing binary, an unpullable container image, or a microVM that fails to boot), the failure appears here along with a bounded tail of the agent process's own stdout/stderr captured by the scaler. This is the surface to check for a run that fails with no step logs at all — the agent never started, so the cause lives in the provisioning lifecycle rather than in any step's output.
7355
-
7356
- ### Performance
7357
-
7358
- The log viewer uses virtualized scrolling to handle large outputs. Only the visible lines plus a small buffer are rendered in the DOM, keeping performance smooth even for logs with 10,000+ lines.
7359
-
7360
- <!-- help:settings-general#settings -->
7361
-
7362
- General settings show your organization's basic information, including the org name (editable by owners) and the unique organization ID. Use this to rename your org or reference the ID for API calls and configuration.
7363
-
7364
- <!-- /help:settings-general -->
7365
-
7366
- <!-- help:settings-members#settings -->
7367
-
7368
- The members tab lets you manage your team:
7369
-
7370
- - Invite new members by email.
7371
- - Assign roles.
7372
- - Suspend or remove members.
7373
- - Configure per-user CI trust levels.
7374
-
7375
- Each member's linked provider accounts (e.g. GitHub) are also visible here.
7376
-
7377
- <!-- /help:settings-members -->
7378
-
7379
- <!-- help:settings-roles#settings -->
7380
-
7381
- Roles define granular permissions across 15 resource categories (runs, secrets, members, etc.) with 5 access levels: `none`, `read`, `read_payload`, `write`, `admin`.
7382
-
7383
- Create custom roles to restrict what team members can do, or use the built-in **Owner** role for full access.
7384
-
7385
- <!-- /help:settings-roles -->
7386
-
7387
- <!-- help:settings-teams#settings -->
7388
-
7389
- Teams are named groups of organization members. A role granted to a team is inherited by every member, so you can manage permissions for a whole group in one place.
7390
-
7391
- Team names can also be referenced in workflow approval gates (`requireApproval: [{ team: 'leads' }]`) — any member of the named team can satisfy that gate.
7392
-
7393
- Managing teams (create / rename / delete, membership, role grants) requires the **Teams** permission at `admin`; `read` shows a view-only list.
7394
-
7395
- <!-- /help:settings-teams -->
7396
-
7397
- <!-- help:settings-api-keys#settings -->
7398
-
7399
- API keys allow programmatic access to the KiCI API for automation, scripts, and CI integrations.
7400
-
7401
- Each key is scoped to this organization with a custom permission matrix and an optional expiry date. Keys can be revoked individually.
7402
-
7403
- Use a key's clone button to open the creation modal prefilled with that key's name, expiry, and permissions — handy for recreating an expired key or deriving a new key from an existing one.
7404
-
7405
- <!-- /help:settings-api-keys -->
7406
-
7407
- <!-- help:settings-orchestrator-keys#orchestrator-keys -->
7408
-
7409
- Orchestrator keys authenticate the WebSocket connection between your orchestrator and the KiCI Platform relay.
7410
-
7411
- Create a key here and set it as the `KICI_PLATFORM_TOKEN` environment variable in your orchestrator configuration. Keys can optionally be restricted to specific routing patterns.
7412
-
7413
- Use a key's clone button to open the creation modal prefilled with that key's name and description.
7414
-
7415
- <!-- /help:settings-orchestrator-keys -->
7416
-
7417
- <!-- help:settings-sources#sources -->
7418
-
7419
- Webhook sources are registered automatically when an orchestrator connects to the Platform and sends a `source.register` message.
7420
-
7421
- Each source shows its routing key and full webhook URL — configure this URL in your provider's webhook settings (e.g. GitHub App).
7422
-
7423
- To retrieve the webhook secret for signature verification, use the `kici-admin source get-webhook-secret <routingKey>` command shown below each source.
7424
-
7425
- <!-- /help:settings-sources -->
7426
-
7427
- <!-- help:settings-billing#settings -->
7428
-
7429
- The billing tab shows your current plan (Free, Pro, or Team), resource usage meters, and lets you upgrade to a paid tier.
7430
-
7431
- Choose Monthly or Annual billing, click "Upgrade to Pro" or "Upgrade to Team" to start a Stripe Checkout, or use "Manage payment" to switch tiers and update payment methods via the Stripe Billing Portal.
7432
-
7433
- The usage meters track:
7434
-
7435
- - **Members:** invited users in this org.
7436
- - **Orchestrator connections:** direct WebSocket connections from your orchestrators to the Platform. Only coordinators (and standalone orchestrators) open a connection; peer/worker nodes in a Raft cluster share their coordinator's connection and don't count separately.
7437
- - **Relayed webhooks (this month):** webhooks delivered through the Platform relay during the current billing window.
7438
- - **Live log minutes (today):** log streaming time consumed in the current UTC day.
7439
- - **Retention period:** how long execution history is kept.
7440
-
7441
- The diagnostics page may show a higher orchestrator count than this tab — diagnostics counts cluster nodes, billing counts billable connections.
7442
-
7443
- <!-- /help:settings-billing -->
7444
-
7445
- <!-- help:settings-billing-orch-connections#settings -->
7446
-
7447
- The orchestrator-connections counter measures the number of **direct WebSocket connections** that your orchestrator processes hold open against the KiCI Platform — one count per live connection.
7448
-
7449
- **What counts as one connection:**
7450
-
7451
- - One standalone orchestrator (single process, no cluster) → **1 connection**.
7452
- - One Raft cluster (1 coordinator + N peers) → **1 connection** — only the coordinator opens a Platform WebSocket. The peers gossip through the coordinator and never connect to Platform directly, so they do **not** count toward your billing limit.
7453
- - N independent orchestrator deployments (e.g., one per environment, one per region) → **N connections**.
7454
-
7455
- This is why the diagnostics page can show more orchestrator **nodes** than the billing page shows **connections**: diagnostics counts every node in your topology (coordinator + peers), while billing only counts the WebSocket connections you pay for. A 4-connection org running two 3-node clusters and two standalones will show 4 on the billing meter and 8 on the diagnostics page — both numbers are correct, they measure different things.
7456
-
7457
- When you hit the cap, the next coordinator that tries to connect is rejected with WebSocket close code 4020 (`WS_CLOSE_PLAN_LIMIT`). Existing connections are never disconnected. Upgrade your plan to lift the cap; the meter updates immediately.
7458
-
7459
- <!-- /help:settings-billing-orch-connections -->
7460
-
7461
- <!-- help:settings-billing-relayed-webhooks#settings -->
7462
-
7463
- The relayed-webhooks counter only includes webhooks delivered through the KiCI Platform relay — the route at `kici.dev` that signature-verifies an inbound webhook and forwards it over WebSocket to your orchestrator.
7464
-
7465
- Webhooks pointed directly at your orchestrator's public ingest endpoint never reach the Platform, so they're invisible to this counter and uncapped on every Hosted tier. If you have a public orchestrator ingress, you can mix-and-match: use the relay for sources you can't expose publicly, and point GitHub (or any provider / generic webhook) straight at your orchestrator for the rest.
7466
-
7467
- Every webhook the relay forwards counts — **including ones your workflows ultimately ignore**. Trigger matching runs on your orchestrator, not on the Platform, so the relay forwards each signature-verified webhook before any trigger is evaluated. A source that sends many events you filter down to a handful of runs still consumes one relayed webhook per event. If a high-volume source mostly produces no run, point it directly at your orchestrator (see above) to keep it off this counter entirely.
7468
-
7469
- When you hit the cap, new relayed webhooks are rejected with `429 Plan limit reached`. Upgrade in the Stripe Billing Portal to lift the cap immediately; usage resets monthly on your billing anniversary.
7470
-
7471
- <!-- /help:settings-billing-relayed-webhooks -->
7472
-
7473
- <!-- help:settings-billing-currency#settings -->
7474
-
7475
- Switch the prices shown on the tier cards between US dollars and euros. The choice you pick here is also the currency Stripe charges in when you click "Upgrade".
7476
-
7477
- The default is detected from your browser language. EU, EFTA, and UK locales default to euros; everywhere else defaults to dollars.
7478
-
7479
- Your choice persists in a 90-day cookie (`kici_pricing_currency`), so it survives across reloads and applies on every billing page.
7480
-
7481
- <!-- /help:settings-billing-currency -->
7482
-
7483
- <!-- help:billing-payment-failure#settings -->
7484
-
7485
- This banner appears when your organization's latest payment to Stripe has failed. Your subscription remains active during the retry period, but you should update your payment method promptly to avoid service interruption.
7486
-
7487
- <!-- /help:billing-payment-failure -->
7488
-
7489
- <!-- help:activity-overview#activity -->
7490
-
7491
- Activity is your forensic log — every Platform mutation (invites, role changes, sources, plans) and every orchestrator action (reads, run cancels, secret reveals, environment edits) merged into one chronological stream.
7492
-
7493
- Each row shows the actor, the action, the target, and the outcome.
7494
-
7495
- - **Audit rows:** expand for field-level change tracking.
7496
- - **Access rows:** expand for the request ID, origin, and any error message.
7497
-
7498
- <!-- /help:activity-overview -->
7499
-
7500
- <!-- help:activity-filters#activity -->
7501
-
7502
- Filters live entirely in the URL — bookmark or share a filtered view to replay it.
7503
-
7504
- - **Search:** full-text match against access-log error messages and the JSON body of audit entries.
7505
- - **Run ID:** combine with another filter to scope all activity touching a specific run.
7506
-
7507
- Click a row's run target to jump straight to the run detail page.
7508
-
7509
- <!-- /help:activity-filters -->
7510
-
7511
- ## DLQ
7512
-
7513
- The DLQ (dead-letter queue) page lists internal events whose dispatch attempts were exhausted (or that hit a non-retryable error). Each row shows when the event landed in the DLQ, the event name, the attempt count, the failure reason, and the last error message.
7514
-
7515
- <!-- help:dlq#dlq -->
7516
-
7517
- The DLQ holds events your org emitted that could not be dispatched within the retry budget. The sidebar badge shows the current depth so you can spot a building backlog without opening the page.
7518
-
7519
- Per-row actions (visible when you have `event_dlq:write`):
7520
-
7521
- - **Retry:** clears the DLQ flag and re-publishes the event. A healthy orchestrator picks it up immediately.
7522
- - **Discard:** permanently deletes the row. Use when the payload is corrupt or the routing target no longer exists.
7523
-
7524
- Members with only `event_dlq:read` see the list but cannot retry or discard. Org owners have both actions by default.
7525
-
7526
- <!-- /help:dlq -->
7527
-
7528
- <!-- help:settings-ci-trust#settings -->
7529
-
7530
- CI trust policy controls how your organization handles PR-triggered runs from different contributor types. Configure the default trust level for unknown contributors and set per-member overrides to control who can run workflows with full secrets access.
7531
-
7532
- <!-- /help:settings-ci-trust -->
7533
-
7534
- <!-- help:settings-global-workflows#settings -->
7535
-
7536
- Global workflows let a single "workflow repo" define jobs that run when events happen in other repos in the same org.
7537
-
7538
- This tab exposes the security knobs as independent axes:
7539
-
7540
- - **Master enable toggle:** turn the whole feature on or off.
7541
- - **Authoring allow-list:** which repos may **define** global workflows.
7542
- - **Source deny-list:** **source** repos whose events never trigger globals (forks, public-contrib).
7543
- - **Elevated-access list:** authoring repos that need source-repo secrets during execution.
7544
-
7545
- See the [user guide](global-workflows.md) and the [architecture reference](../architecture/global-workflows.md) for the full model.
7546
-
7547
- <!-- /help:settings-global-workflows -->
7548
-
7549
- <!-- help:settings-global-workflows-enable#settings -->
7550
-
7551
- Master kill-switch for global workflows in this org.
7552
-
7553
- - **OFF:** the orchestrator will **not register** any workflow that declares `repos:` patterns, and will **not dispatch** cross-repo triggers — effectively rolling the org back to per-repo-only semantics. All other settings on this page are ignored.
7554
- - **ON:** the other toggles become your safety rails. Turn ON to opt in.
7555
-
7556
- <!-- /help:settings-global-workflows-enable -->
7557
-
7558
- <!-- help:settings-global-workflows-authors#settings -->
7559
-
7560
- Restricts which repos in this org may **define** global workflows (the "authoring axis").
7561
-
7562
- - **OFF:** any repo in the org may declare a workflow with `repos:` patterns and have it registered.
7563
- - **ON:** only repos whose identifier matches one of the entries below may author globals. Non-matching repos have their global workflows dropped at registration time, with a warning in the orchestrator log.
7564
- - **ON + empty list:** **no repo** may author globals — use as a temporary lock-down.
7565
-
7566
- Each entry has two parts:
7567
-
7568
- - **Source:** pick a configured source (a specific GitHub App or universal-git source) to pin the entry to that source only, or leave it as **Any source** to match across every source in the org.
7569
- - **Pattern:** a glob matched against the authoring repo identifier (e.g. `myorg/ci-*`, `myorg/platform-*`).
7570
-
7571
- Pinning by source is useful when the same `owner/repo` could legitimately exist on more than one configured source and you only want to trust one of them as an author.
7572
-
7573
- <!-- /help:settings-global-workflows-authors -->
7574
-
7575
- <!-- help:settings-global-workflows-blocked-sources#settings -->
7576
-
7577
- Deny-list for **source** repos whose events must never trigger a global workflow (the "source axis").
7578
-
7579
- Use this for untrusted territory — forks, public-contrib mirrors, sandboxes — where a single push shouldn't be able to fan out org-wide automation.
7580
-
7581
- Evaluated at dispatch time against the repo that emitted the event, independently of the authoring allow-list: a global workflow whose author is allowed will still be skipped if the _source_ repo is denied. Both lists can be active simultaneously.
7582
-
7583
- Each entry has two parts:
7584
-
7585
- - **Source:** pick a configured source to deny only events delivered on that source, or leave it as **Any source** to deny across the org.
7586
- - **Pattern:** a glob matched against the source repo identifier (e.g. `myorg/fork-*`, `myorg/public-*`).
7587
-
7588
- Pinning by source is the right move when the same `owner/repo` is reachable through more than one configured source (e.g. a public forge and a trusted mirror) and you want to drop deliveries from only one of them.
7589
-
7590
- <!-- /help:settings-global-workflows-blocked-sources -->
7591
-
7592
- <!-- help:settings-global-workflows-elevated#settings -->
7593
-
7594
- Authoring repos listed here receive **elevated access to source-repo secrets** during global workflow execution.
7595
-
7596
- - **Without elevation:** a global workflow job runs with only the workflow repo's own credentials — it can clone both repos but can't read the source repo's scoped secrets.
7597
- - **With elevation:** the job gets the source repo's secret context injected, so deploy / release / cross-repo automation flows work.
7598
-
7599
- Treat elevated repos as effective owners of every source repo's CI secrets — only add repos you fully trust.
7600
-
7601
- Each entry has two parts:
7602
-
7603
- - **Source:** pick a configured source to elevate only when the authoring repo lives on that source, or leave it as **Any source** to elevate across the org.
7604
- - **Pattern:** a glob matched against the **workflow-authoring** repo, not the source repo (e.g. `myorg/ci-deploy`, `myorg/release-automation`).
7605
-
7606
- Pinning by source narrows the trust window: if the same `owner/repo` is configured on more than one source, only the source you pick will grant elevation.
7607
-
7608
- <!-- /help:settings-global-workflows-elevated -->
7609
-
7610
- <!-- help:settings-webhooks#settings -->
7611
-
7612
- Configure outbound webhook endpoints to receive notifications when runs and jobs change status. Each endpoint receives HMAC-SHA256 signed payloads with event details.
7613
-
7614
- For each endpoint you can:
7615
-
7616
- - **Subscribe to event types:** `run.started`, `run.completed`, `run.failed`, `job.started`, `job.completed`, `job.failed`.
7617
- - **View delivery logs:** HTTP response codes and retry counts.
7618
- - **Send a test ping:** verify connectivity before going live.
7619
-
7620
- <!-- /help:settings-webhooks -->
7621
-
7622
- <!-- help:settings-security-dashboard-policy#settings -->
7623
-
7624
- Read-only view of the orchestrator's dashboard-write policy.
7625
-
7626
- Each row toggles one mutating dashboard action — setting a secret, approving a held run, retrying a dead-lettered webhook, and so on. The orchestrator operator decides which actions stay on the dashboard and which become **CLI-only**. The dashboard cannot change the policy itself — that's the point: disabled actions stay out of the SaaS Platform's trust path.
7627
-
7628
- Manage the policy with:
7629
-
7630
- - **Show the full policy:** `kici-admin org-settings dashboard-writes show`
7631
- - **Disable an operation:** `kici-admin org-settings dashboard-writes set --op <name>=false`
7632
- - **Reset to permissive defaults:** `kici-admin org-settings dashboard-writes reset`
7633
-
7634
- The summary strip at the top shows total / enabled / disabled counts plus whether your orchestrator is currently connected. A disconnected orchestrator means the page falls back to the cached policy from the most recent connection.
7635
-
7636
- <!-- /help:settings-security-dashboard-policy -->
7637
-
7638
- <!-- help:settings-support-access#settings -->
7639
-
7640
- Controls whether KiCI support staff may open read-only support sessions against your organization. Sessions are **off by default** — nobody outside your org can read your data until you opt in here.
7641
-
7642
- When enabled:
7643
-
7644
- - KiCI staff can open time-boxed, read-only sessions to investigate an issue.
7645
- - Every read they perform is recorded in your audit trail with the support reason.
7646
- - No writes are ever possible during a session.
7647
-
7648
- Disabling the toggle immediately ends any in-progress support session. Only users with the `support:admin` permission (owners by default) can change this setting.
7649
-
7650
- <!-- /help:settings-support-access -->
7651
-
7652
- <!-- help:settings-webhooks-delivery-log#settings -->
7653
-
7654
- The delivery log shows recent webhook deliveries for an endpoint, including the HTTP status code, number of retry attempts, and the event payload.
7655
-
7656
- Retry behavior:
7657
-
7658
- - Deliveries are retried up to 3 times with exponential backoff.
7659
- - After 10 consecutive failures, the endpoint is automatically disabled — you can re-enable it from this view.
7660
-
7661
- <!-- /help:settings-webhooks-delivery-log -->
7662
-
7663
- <!-- help:settings-event-log#event-log -->
7664
-
7665
- The event log shows every inbound webhook this organization has received, regardless of whether it came in via the Platform relay or directly to an orchestrator.
7666
-
7667
- Each row joins two records:
7668
-
7669
- - **Platform side:** event metadata and a SHA-256 hash of the body (no payload stored).
7670
- - **Orchestrator side:** full payload and processing outcome.
7671
-
7672
- Filter by routing key, event type, status, or delivery ID. Click a row for the full per-tier breakdown.
7673
-
7674
- <!-- /help:settings-event-log -->
7675
-
7676
- <!-- help:settings-event-log-detail#event-log -->
7677
-
7678
- The detail panel shows both tiers' projections side-by-side.
7679
-
7680
- - **Platform record:** answers "did the delivery arrive at the relay and where was it routed".
7681
- - **Orchestrator record:** answers "what was the body and what happened next" — including the matched workflow count and spawned run links.
7682
- - **Payload:** the raw webhook body. Streams over the dashboard's existing WebSocket connection in 64 KiB chunks so Platform never buffers the full body and you see progress for large deliveries. Requires `event_log:read_payload`.
7683
-
7684
- Oversized or storage-failed payloads show an "omitted" badge, with the hash preserved for correlation against raw logs.
7685
-
7686
- <!-- /help:settings-event-log-detail -->
7687
-
7688
- ## Activity
7689
-
7690
- The activity page (`/orgs/:customerId/activity`) is the org-level forensic log. It federates two streams into one chronological view: the upstream tenant-plane audit log (every tenant-plane mutation -- invites, role changes, source registrations, plan changes) and orchestrator `access_log` rows (every read and admin action -- run cancels, secret reveals, environment edits, dashboard data fetches via the Platform proxy). Filters live in the URL via search params so a filtered view is bookmarkable and shareable. The page uses cursor-based pagination and supports filtering by source (audit / access_log / all), free-text search, run ID, and other dimensions. Requires `audit:read` permission. The legacy `/orgs/:customerId/audit-log` URL redirects here to preserve bookmarks.
7691
-
7692
- ## Settings
7693
-
7694
- The settings page (`/orgs/:customerId/settings`) uses a tabbed layout:
7695
-
7696
- 1. **General** -- displays the organization name (editable by owners via inline click-to-edit) and the organization ID
7697
- 2. **Members** -- team management with invite, role assignment, and member removal
7698
- 3. **Roles** -- custom role management with granular permission matrix
7699
- 4. **API keys** -- API key creation and revocation for dashboard/programmatic access
7700
- 5. **Orchestrator keys** -- orchestrator API key management for Platform WebSocket connections
7701
- 6. **Sources** -- read-only list of registered webhook sources (see below)
7702
- 7. **Billing** -- plan and payment management (hidden in the `kici-admin` org)
7703
- 8. **CI trust** -- trust policy configuration for CI runs (visible with `ci_trust:read` permission)
7704
- 9. **Global workflows** -- org-level security knobs for cross-repo workflows (visible with `org_settings:read` permission)
7705
- 10. **Webhooks** -- outbound webhook endpoint management with delivery logs and test ping
7706
- 11. **Event log** -- inbound webhook delivery log (visible with `event_log:read` permission)
7707
- 12. **Security** -- read-only view of the orchestrator's dashboard-write policy matrix (visible with `org_settings:read` permission)
7708
- 13. **Support access** -- opt-in switch that controls whether KiCI support staff may open read-only support sessions against your org (visible with `support:read`; toggled with `support:admin`)
7709
-
7710
- Audit-log-style entries are no longer a settings tab; they live on the dedicated **Activity** page accessible from the sidebar.
7711
-
7712
- Tab selection syncs with the URL path (`/settings/members`, `/settings/api-keys`, etc.), making tabs bookmarkable.
7713
-
7714
- ### Support access
7715
-
7716
- The Support access tab controls whether KiCI support staff may open a read-only **support session** against your organization to help diagnose an issue. The setting is **off by default** -- until you opt in here, no one outside your org can read your data.
7717
-
7718
- When support access is enabled:
7719
-
7720
- - A KiCI operator can open a time-boxed (30-minute, renewable), read-only support session scoped to a stated reason.
7721
- - A support session is **runs-only**: the operator can browse your run list and, by confirming each run individually, view that run's detail and step logs. Nothing else is visible, and no write is ever possible.
7722
- - Every run an operator opens is recorded in your [Activity](#activity) audit trail, attributed to the operator with the support reason -- so you can see exactly what was looked at and why.
7723
-
7724
- **Disabling immediately ends any active session.** Toggling the switch off closes every in-progress support session for your org at once. Enabling and disabling the setting is itself audited, attributed to the user who changed it.
7725
-
7726
- Viewing the setting requires the `support:read` permission; changing it requires `support:admin` (granted to owners by default).
7727
-
7728
- ### Orchestrator keys
7729
-
7730
- The orchestrator keys tab manages API keys used to authenticate orchestrator-to-Platform WebSocket connections. These are separate from user API keys (which grant dashboard/API access).
7731
-
7732
- **List view** -- shows all active orchestrator keys with name, description, key prefix, creation date, and last used date.
7733
-
7734
- **Create** -- opens a modal to enter a name and optional description. After creation, the raw key is shown once in a copyable box. Set this key as the `KICI_PLATFORM_TOKEN` environment variable in your orchestrator configuration.
7735
-
7736
- **Revoke** -- opens a confirmation modal before soft-deleting the key. Any orchestrators using the revoked key will be disconnected.
7737
-
7738
- ### Sources
7739
-
7740
- The sources tab shows webhook sources registered by connected orchestrators. Sources appear here **automatically** when an orchestrator connects to the Platform via WebSocket and sends a `source.register` message -- there is no manual "add source" action in the UI.
7741
-
7742
- **What causes a source to appear:**
7743
-
7744
- 1. An orchestrator is configured with one or more providers (e.g., a GitHub App with `appId: 12345`)
7745
- 2. The orchestrator connects to the Platform using an orchestrator API key for your organization
7746
- 3. On connection, the orchestrator sends `source.register` with its provider sources (e.g., `github:12345`)
7747
- 4. The Platform records the source against your organization
7748
- 5. The source immediately appears in the dashboard
7749
-
7750
- **Each source displays:**
7751
-
7752
- - **Routing key** -- the source identifier (e.g., `github:12345` for a GitHub App, `generic:my-source` for a generic webhook)
7753
- - **Webhook URL** -- the URL to configure in your provider's webhook settings (constructed by the Platform based on the provider type and org ID)
7754
- - **Registered at** -- when the orchestrator first registered this source
7755
- - **Copy button** -- copies the webhook URL to the clipboard
7756
-
7757
- **Read-only** -- sources cannot be created, edited, or deleted from the dashboard. They are managed entirely by orchestrator connections. When an orchestrator disconnects, its sources remain visible (they are not automatically removed).
7758
-
7759
- **Empty state** -- if no orchestrator has connected yet, the tab shows "No webhook sources registered" with a link to the operator setup guide.
7760
-
7761
- **Webhook secrets** -- webhook HMAC secrets are not visible in the dashboard. They are stored in the orchestrator's database (`webhook_secrets` table) and pushed to the Platform via the `source.secrets` WebSocket message after registration. The Platform uses these secrets to verify incoming webhook signatures. Secrets are configured in the orchestrator's database, not through the UI.
7762
-
7763
- **Adding a new source** requires:
7764
-
7765
- 1. Configure a new provider in the orchestrator (e.g., add a GitHub App to the orchestrator's provider config)
7766
- 2. Seed the webhook secret in the orchestrator's `webhook_secrets` database table
7767
- 3. Restart the orchestrator -- it will register the new source with the Platform on connection
7768
- 4. Configure the webhook URL (shown in the sources tab) in the provider's settings (e.g., GitHub App webhook URL)
7769
-
7770
- ### Event log
7771
-
7772
- The event log tab (`/orgs/:customerId/settings/event-log`) shows every inbound webhook this organization has received. Each row joins two tiers of records:
7773
-
7774
- 1. **Platform record** -- written by the Platform relay on every delivery: routing key, event, action, repo, routing target, status, SHA-256 payload hash. The Platform never persists the payload (trust boundary).
7775
- 2. **Orchestrator record** -- written by the destination orchestrator when it processes the delivery: full payload (in object storage), processing outcome (`processed` / `duplicate` / `lockfile_missing` / `failed`), matched workflow count, first run spawned (if any), and a payload hash that matches the Platform record for cross-tier correlation.
7776
-
7777
- The list view supports filters for routing key, event type, status, and free-text delivery ID search. Click a row to open a detail panel with both tiers' projections side-by-side, plus the payload viewer.
7778
-
7779
- **Permissions:**
7780
-
7781
- - `event_log:read` -- list rows and view metadata in the detail panel.
7782
- - `event_log:read_payload` -- additionally view the raw webhook payload body. (Owners and admins inherit this. Lower-tier roles see "Payload not available" with a hint to ask for an elevated role.)
7783
-
7784
- **Edge cases the UI surfaces:**
7785
-
7786
- - **Payload omitted** -- when the inbound payload exceeded the orchestrator's `eventLog.maxPayloadBytes` soft cap (default 5 MB) or the object-storage write failed, the row is still recorded with `payload_omitted=true`. The hash is preserved so operators can correlate against `KICI_WEBHOOK_PAYLOAD_DIR` or raw logs.
7787
- - **Orchestrator unavailable** -- when the orchestrator does not respond within 2 seconds of the merge fan-out, the list still loads with Platform-side metadata only, marked with an `orchestrator_unavailable` banner.
7788
- - **Orchestrator-only deliveries** -- direct-ingress deliveries (independent / hybrid mode) that never crossed the Platform appear with `platform.status = orchestrator_only`.
7789
-
7790
- Retention is 30 days on both tiers, matching the Platform `event_log` audit window.
7791
-
7792
- <!-- help:personal-profile#account -->
7793
-
7794
- Account settings let you view your profile information (name, email) and manage your KiCI account. Changes here apply across all organizations you belong to.
7795
-
7796
- <!-- /help:personal-profile -->
7797
-
7798
- <!-- help:personal-pats#account -->
7799
-
7800
- Personal access tokens (PATs) are long-lived credentials for programmatic API access.
7801
-
7802
- Create a PAT to authenticate CLI tools or scripts without going through the OIDC login flow. Tokens can be revoked at any time.
7803
-
7804
- Use a token's clone button to open the creation modal prefilled with that token's name, expiry, and permissions.
7805
-
7806
- <!-- /help:personal-pats -->
7807
-
7808
- <!-- help:personal-linked-accounts#account -->
7809
-
7810
- Linked accounts connect your external provider identities (like GitHub) to your KiCI account. Linking shows your provider username in run metadata and sets your contributor trust level.
7811
-
7812
- **Unlinking here removes the display link only** — it does not remove a sign-in method. To change how you sign in (add a password, remove GitHub login), use the **Account console** link, or see [Account and sign-in](./account-and-login.md).
7813
-
7814
- <!-- /help:personal-linked-accounts -->
7815
-
7816
- <!-- help:orgs-list#organizations -->
7817
-
7818
- Organizations are the top-level container for your CI/CD resources. Each org has its own runs, settings, environments, secrets, and team members. Select an organization to manage its workflows and configuration.
7819
-
7820
- <!-- /help:orgs-list -->
7821
-
7822
- <!-- help:orchestrators-list#orchestrators -->
7823
-
7824
- The Orchestrators page lists every orchestrator currently connected to this org, keyed by **cluster name**. Each row shows:
7825
-
7826
- - **Cluster** — the human-friendly cluster name set on the orch via `kici-admin cluster-name set <name>`, or an auto-generated `cluster-<6hex>` if no operator has renamed it.
7827
- - **Role** — `coordinator` (talks to Platform directly) or `worker` (relays through a coordinator).
7828
- - **Version**, **mode**, **routing keys**, and **last heartbeat**.
7829
-
7830
- Click a cluster to drill into its per-orch surfaces (security policy, environments, secrets, DLQ, registrations, global workflows). Different clusters in the same org can have different settings — this page is the entry point that lets you pick which cluster you're configuring.
7831
-
7832
- <!-- /help:orchestrators-list -->
7833
-
7834
- <!-- help:orchestrators-scope#orchestrators -->
7835
-
7836
- Every panel inside this view scopes to the named cluster. Settings shown here come from that orchestrator's own database — a sibling orchestrator in the same org may have a different security policy, different environments, and different secrets.
7837
-
7838
- When the cluster shows **disconnected**, the orch is offline and its current state can't be queried. Most child pages will return 404 in that state; return to the orchestrator list to find a connected cluster.
7839
-
7840
- To rename a cluster, run `kici-admin cluster-name set <new>` on the orchestrator host and restart the orch service so the new name reaches Platform on the next `source.register`.
7841
-
7842
- <!-- /help:orchestrators-scope -->
7843
-
7844
- ## Workflows
7845
-
7846
- The workflows page (`/orgs/:customerId/workflows`) shows permanently registered workflows listening for events. It displays a filterable table with columns for workflow name, repository, trigger types, last triggered time, next fire time (for scheduled workflows), source repos, and actions.
7847
-
7848
- Each row is expandable to show trigger configuration details. Rows include action controls: a "Run now" button for manual triggering, a toggle switch to enable/disable the workflow, and a delete button with a confirmation modal (optionally cancelling active runs). Stale workflows (no triggers in the last 30 days) show a yellow "Stale" badge. Registry health indicators (version, sync status, last updated) appear above the table.
7849
-
7850
- Filters include trigger type, repository, and workflow name.
7851
-
7852
- ## Diagnostics
7853
-
7854
- The diagnostics page (`/orgs/:customerId/diagnostics`) provides infrastructure health monitoring. It has four sections:
7855
-
7856
- 1. **Execution metrics** -- cards showing total runs (24h), success rate, average duration, and active jobs (queued + running). Refreshes every 30 seconds.
7857
- 2. **Infrastructure alerts** -- banner summarizing any critical or warning alerts from connected orchestrators
7858
- 3. **Infrastructure tree** -- hierarchical view of orchestrators, their scalers, and agents. Refreshes every 10 seconds. Each orchestrator row shows:
7859
- - **`orchestrator:`** (bold monospace, left group) -- the orchestrator's cluster instance ID, set via `KICI_CLUSTER_INSTANCE_ID` env var or auto-generated as a UUID. If no instance ID is set, the first 8 characters of the connection ID are shown here instead.
7860
- - **`conn:`** (dimmed monospace, left group) -- first 8 characters of the WebSocket connection ID assigned by the Platform relay. Only shown when an explicit instance ID is present.
7861
- - Connection status badge, role badge (coordinator or worker), version badge (left group, after the ID labels)
7862
- - **`host:`** badge (right side) -- the system hostname of the machine running the orchestrator process
7863
- - Additional badges on the right side: running-as user, CPU count, memory usage, uptime
7864
-
7865
- Each orchestrator lists its **scalers** (indented at level 1) showing scaler name, type badge (container/firecracker/bare-metal), active/max agent count, and a config info popover. Below each scaler, its **agents** (indented at level 2) display agent ID, platform/arch, heartbeat age, hostname, running-as user, CPU count, memory, uptime, and version. Labels (both user-defined and auto-generated `kici:` prefixed) are shown on a separate row beneath scalers and stateful agents, with a tooltip distinguishing user labels from auto labels.
7866
-
7867
- 4. **Secret backends** -- health cards for each configured secret backend (e.g. OpenBao), showing connection status with sync and test actions. Allows triggering a manual sync or connectivity test per backend.
7868
-
7869
- ## Environments
7870
-
7871
- The environments page (`/orgs/:customerId/environments`) lists all deployment environments for the organization. Each environment shows its name, type (fixed or glob pattern), protection status (branch restrictions, concurrency limits, required reviewers, wait timers), and enabled/disabled state.
7872
-
7873
- Users with `environments:admin` permission can create new environments via a modal dialog, choosing between fixed and glob (pattern-matching) types. Clicking an environment row navigates to the environment detail page.
7874
-
7875
- ### Environment detail
7876
-
7877
- The environment detail page (`/orgs/:customerId/environments/:environmentId`) shows a header with the environment name, type badge, enabled/disabled toggle, and a delete button. Below the header, a tabbed layout provides four sections:
7878
-
7879
- 1. **Variables** (default) -- environment-scoped variables
7880
- 2. **Secrets** -- secrets bound to this environment
7881
- 3. **Protection** -- protection rules (branch restrictions, concurrency limits, required reviewers, wait timers)
7882
- 4. **History** -- audit history of changes to this environment
7883
-
7884
- Tab selection syncs with the URL path (`/orgs/:customerId/environments/:environmentId/variables`, `/orgs/:customerId/environments/:environmentId/protection`, etc.).
7885
-
7886
- ## Secrets
7705
+ ```
7887
7706
 
7888
- The secrets page (`/orgs/:customerId/secrets`) provides a scope-centric view of all secrets in the organization. Secrets are organized into a scope tree with environment binding checkboxes, allowing you to control which secret scopes are available in which environments.
7707
+ In queue mode, the agent that picked up the queued run **stays connected** to the orchestrator and parks on a long-poll wait. When the holder finishes (success, failure, or cancel), the orchestrator dequeues the FIFO-next entry and pushes a `proceed` notification over the same WebSocket; the queued agent then continues with normal step execution against the workspace it already has. The agent's slot is therefore held for the duration of the queue wait — bound by `KICI_CONCURRENCY_WAIT_TIMEOUT_MS` (default 1 hour).
7889
7708
 
7890
- Permission-gated: `secrets:read` to view scopes, `secrets:write` to add or delete secrets, `environments:write` to modify environment bindings.
7709
+ ```typescript
7710
+ workflow('migrate-db', {
7711
+ concurrency: {
7712
+ group: () => 'migrations',
7713
+ cancelInProgress: false,
7714
+ max: 1,
7715
+ },
7716
+ jobs: [
7717
+ /* ... */
7718
+ ],
7719
+ });
7720
+ ```
7891
7721
 
7892
- ### Where secrets live
7722
+ The dashboard will show a "Queued" badge with the reason: "Waiting for deploy-main (1 ahead)".
7893
7723
 
7894
- Secret values are stored in the orchestrator's secret store and authorized through the orchestrator's RBAC. The dashboard surfaces secret **names** and scope membership for every secret regardless of where the value was entered.
7724
+ ## Max concurrent runs
7895
7725
 
7896
- Whether secret **values** can be set from the dashboard depends on the orchestrator's [dashboard-write policy](/operator/security/dashboard-write-policy):
7726
+ The `max` field controls how many runs can execute simultaneously in the same group:
7897
7727
 
7898
- - **Permissive (default):** the "Add secret" and "Edit value" controls accept plaintext directly in the dashboard. This is how a typical SaaS CI tool works and is the right default for small teams.
7899
- - **`secrets.set` disabled by policy:** the controls render with a lock icon, grayed out. Hovering shows the exact `kici-admin secret set` invocation needed; a copy button puts it on the clipboard. The control is inert — the dashboard issues no mutating request. Use the CLI to enter values; the dashboard refreshes within ~30 seconds and shows the new secret name.
7728
+ ```typescript
7729
+ // Allow up to 3 parallel test runs per branch
7730
+ workflow('test', {
7731
+ concurrency: {
7732
+ group: (ctx) => `test-${ctx.branch}`,
7733
+ cancelInProgress: false,
7734
+ max: 3,
7735
+ },
7736
+ jobs: [
7737
+ /* ... */
7738
+ ],
7739
+ });
7740
+ ```
7900
7741
 
7901
- The policy state is visible at three layers in the UI:
7742
+ When `max: 1` (default), runs are fully serialized within the group.
7902
7743
 
7903
- - A **lock-icon prefix** on every disabled control, with a per-control CLI hint.
7904
- - A **per-page banner** on any page containing at least one disabled operation, listing every disabled op on that page and its CLI equivalent.
7744
+ ## Group key examples
7905
7745
 
7906
- The Security policy page (Settings → Security → Dashboard policy) renders the full 24-row read-only matrix with the current state and the `kici-admin` command for each row. The policy itself cannot be changed from the dashboard — the orchestrator operator manages it via `kici-admin org-settings dashboard-writes`. See [Dashboard-write policy](/operator/security/dashboard-write-policy) for the operator-side details.
7746
+ ### Deploy per environment
7907
7747
 
7908
- ## Approval queue
7748
+ ```typescript
7749
+ workflow('deploy', {
7750
+ concurrency: {
7751
+ group: (ctx) => `deploy-${ctx.branch}`,
7752
+ cancelInProgress: true,
7753
+ },
7754
+ jobs: [
7755
+ job('deploy-staging', {
7756
+ runsOn: 'linux',
7757
+ environment: 'staging',
7758
+ steps: [
7759
+ /* ... */
7760
+ ],
7761
+ }),
7762
+ ],
7763
+ });
7764
+ ```
7909
7765
 
7910
- The approval queue page (`/orgs/:customerId/approval-queue`) shows held runs that are pending approval. Runs can be held due to environment protection rules (required reviewers, wait timers). The page supports filtering by status (pending, approved, rejected, expired) and provides approve/reject actions for users with `environments:write` permission. Users with `environments:admin` permission can skip wait timers.
7766
+ ### Global singleton
7911
7767
 
7912
- ## Account
7768
+ ```typescript
7769
+ // Only one migration can run at a time, regardless of branch
7770
+ workflow('migrate', {
7771
+ concurrency: {
7772
+ group: () => 'db-migration',
7773
+ cancelInProgress: false,
7774
+ },
7775
+ jobs: [
7776
+ /* ... */
7777
+ ],
7778
+ });
7779
+ ```
7913
7780
 
7914
- The standalone account page (`/account`) provides access to personal settings outside of any organization context. It has three tabs:
7781
+ ### Environment-aware groups
7915
7782
 
7916
- - **Profile** -- view your name and email
7917
- - **Personal access tokens** -- create and revoke PATs for programmatic API access
7918
- - **Linked accounts** -- connect external provider identities (e.g. GitHub) to your KiCI account
7783
+ ```typescript
7784
+ // Serialize deploys per environment
7785
+ workflow('deploy', {
7786
+ concurrency: {
7787
+ group: (ctx) => {
7788
+ const env = ctx.branch === 'main' ? 'production' : 'staging';
7789
+ return `deploy-${env}`;
7790
+ },
7791
+ cancelInProgress: true,
7792
+ },
7793
+ jobs: [
7794
+ /* ... */
7795
+ ],
7796
+ });
7797
+ ```
7919
7798
 
7920
- Linked accounts control run-attribution metadata only — unlinking a provider here does not remove it as a way to sign in. To change how you sign in, see [Account and sign-in](./account-and-login.md).
7799
+ ## Interaction with environment protection
7921
7800
 
7922
- This page is also accessible within an org context via the user menu in the sidebar (`/orgs/:customerId/account`).
7801
+ When a workflow has both `concurrency` and `environment` protection rules:
7923
7802
 
7924
- ## Admin section
7803
+ 1. Environment protection gates (required reviewers, wait timer) apply first
7804
+ 2. Concurrency group check happens after protection gates pass
7805
+ 3. If the run is queued by concurrency, it keeps its protection approval
7925
7806
 
7926
- When viewing the `kici-admin` organization, the dashboard switches to an admin-mode interface for platform-wide management. The admin pages are:
7807
+ This means a run that passed approval won't need re-approval if it gets queued by concurrency.
7927
7808
 
7928
- - **Overview** (`/orgs/kici-admin/admin`) -- embedded Grafana dashboards with three tabs: System, Orgs, and Execution
7929
- - **Organizations** (`/orgs/kici-admin/admin/orgs`) -- table of all organizations with plan type, member count, Stripe status, and creation date; rows link to org detail pages
7930
- - **Org detail** (`/orgs/kici-admin/admin/orgs/:orgId`) -- org info summary, plan limit controls, current usage stats with over-limit warnings, quick actions, and a tabbed section with audit log
7931
- - **Connections** (`/orgs/kici-admin/admin/connections`) -- table of connected orchestrators showing org, routing keys, heartbeat age, running jobs, and force-disconnect action
7932
- - **Scheduled jobs** (`/orgs/kici-admin/admin/jobs`) -- table of Platform scheduled background jobs with cron schedule, last run status, consecutive failure count, estimated next run time, and a "Run now" action to trigger immediate execution
7933
- - **Audit log** (`/orgs/kici-admin/admin/audit-log`) -- paginated table of platform-level admin actions with expandable JSON details
7934
- - **Metrics** -- external link to the Grafana instance
7809
+ ## Cancelling queued runs
7935
7810
 
7936
- ## Organizations
7811
+ Queued runs can be cancelled before they start executing. The cancel request removes them from the queue immediately -- they don't go through the grace period since no step is running.
7937
7812
 
7938
- The organizations page (`/orgs`) lists all organizations your account has access to.
7813
+ ## Job-level concurrency groups
7939
7814
 
7940
- Organizations are sorted alphabetically by display name. Each entry shows your role (owner or member). A "Create organization" button opens an inline form to create a new org by name.
7815
+ In addition to workflow-level concurrency, individual jobs can define their own concurrency group via the `concurrencyGroup` property. This controls concurrent execution at the job level rather than the workflow level. See [Environments — concurrency groups](environments.md#concurrency-groups) for details.
7941
7816
 
7942
- ## Theme
7817
+ ## Local execution
7943
7818
 
7944
- The dashboard supports three theme modes:
7819
+ `kici run local` honors workflow-level `concurrency` per-machine, per-user. The `group` callback is evaluated against the simulated event identically to the remote orchestrator path; `cancelInProgress` carries the same semantics — `true` interrupts the holder via `SIGTERM` (escalating to `SIGKILL` after a grace window) and proceeds with the new run, while `false` queues the new invocation in FIFO order until the holder finishes.
7945
7820
 
7946
- - **System** (default) -- follows your operating system's dark/light preference
7947
- - **Dark** -- forced dark mode
7948
- - **Light** -- forced light mode
7821
+ Coordination is local only. Running the same workflow on two different machines does not serialize across them — that requires the orchestrator. For full cross-host enforcement (queueing across agents, dashboard visibility, `max > 1`), use `kici run remote` against a deployed orchestrator.
7949
7822
 
7950
- Toggle between modes using the sun/moon icon in the sidebar footer. The selection persists to `localStorage`.
7823
+ Lock files live under `$XDG_RUNTIME_DIR/kici-local-locks/` on Linux, falling back to `os.tmpdir()/kici-local-locks-<uid>/`. A workflow whose `group` callback throws aborts the run with a clear error rather than running unprotected. See [`kici run local` — Concurrency enforcement](cli-reference.md#concurrency-enforcement) for the `KICI_LOCAL_LOCK_KILL_GRACE_MS` override and the diagnostic output emitted while contending on a busy lock.
7951
7824
 
7952
- ## Date and time preferences
7825
+ The `kici run local --concurrency <n>` flag is a separate concept — it caps **job-level** parallelism within a single run (how many jobs from one workflow run at once), not cross-run serialization.
7953
7826
 
7954
- A toggle button in the sidebar lets you switch between **local time** and **UTC time** display. When UTC mode is enabled:
7827
+ ---
7955
7828
 
7956
- - All timestamps in the run list, run detail header, metadata panel, and log viewer show UTC times
7957
- - Tooltips on relative timestamps (e.g. "5 minutes ago") show the absolute time in UTC
7958
- - The timeline Gantt chart uses UTC for time labels
7829
+ _Source: `packages/sdk/src/types.ts` (WorkflowOptions.concurrency, JobOptions.concurrencyGroup)_
7959
7830
 
7960
- The preference persists to `localStorage`.
7831
+ ---
7961
7832
 
7962
- ## Keyboard shortcuts
7833
+ ## Dashboard
7963
7834
 
7964
- | Key | Context | Action |
7965
- | ------------- | ---------- | ---------------------- |
7966
- | Arrow Up/Down | Job tree | Move focus |
7967
- | Enter | Job tree | Select job or step |
7968
- | Escape | Job tree | Navigate to first job |
7969
- | Enter | Log search | Jump to next match |
7970
- | Shift+Enter | Log search | Jump to previous match |
7971
- | Escape | Log search | Clear search |
7835
+ Source: https://docs.kici.dev/user/dashboard/
7972
7836
 
7973
- ## Error pages
7837
+ The KiCI dashboard is the browser interface for monitoring workflow runs, inspecting jobs and logs, and managing your organization. It signs in via OIDC and talks to KiCI over its API.
7974
7838
 
7975
- The dashboard shows informative error pages instead of blank screens:
7839
+ This guide is split across the following pages:
7976
7840
 
7977
- - **404** -- "Page not found" with a "Go home" button linking to the organizations page
7978
- - **500** -- "Failed to load" with an error message, a trace ID for support, and a "Go home" button (shown when API requests fail)
7979
- - **Client-side rendering errors** -- caught by the error boundary, showing "Something went wrong" with a trace ID and a "Reload page" button
7980
- - **Auth errors** -- authentication failures on the OIDC callback page show the error message with a retry mechanism and a "Back to login" link
7841
+ | Page | Covers |
7842
+ | ------------------------------------------------------------------------------------- | ----------------------------------------------------------------------- |
7843
+ | [Getting started](./dashboard/getting-started.md) | Onboarding checklist and your organizations list |
7844
+ | [Navigation and layout](./dashboard/navigation.md) | Sidebar, mobile nav, theme, time display, shortcuts, error pages |
7845
+ | [Runs and logs](./dashboard/runs.md) | Run list, run detail, the log viewer |
7846
+ | [Settings](./dashboard/settings.md) | Members, roles, teams, keys, sources, billing, security, support access |
7847
+ | [Workflows, diagnostics, and orchestrators](./dashboard/workflows-and-diagnostics.md) | Registered workflows, infra health, per-cluster views |
7848
+ | [Environments, secrets, and approvals](./dashboard/environments-and-secrets.md) | Environments, secret scopes, approval queue |
7849
+ | [Activity and DLQ](./dashboard/activity-and-dlq.md) | Forensic activity log and dead-letter queue |
7850
+ | [Account](./dashboard/account.md) | Personal account settings |
7981
7851
 
7982
7852
  ---
7983
7853
 
@@ -9184,29 +9054,37 @@ registries: [
9184
9054
 
9185
9055
  ## Security model
9186
9056
 
9187
- - **Per-environment scoping.** Every `tokenSecret` and `installEnv` entry is qualified with an environment name. The orchestrator runs the same protection-rule pipeline (branch / trust / concurrency / reviewer / wait-timer) against each named environment **before** resolving any secret, so a workflow that wants a `production` token from a feature branch is rejected exactly like a job that tries to deploy to `production` from a feature branch.
9057
+ - **Per-environment scoping.** Every `tokenSecret` and `installEnv` entry is qualified with an environment name. The orchestrator runs the same protection-rule pipeline (branch / trust / concurrency / reviewer / wait-timer) against each named environment **before** resolving any secret, so a workflow that wants a `production` token from a feature branch is rejected exactly like a job that tries to deploy to `production` from a feature branch. A reviewer-gated install environment **pauses** the whole workflow dispatch as a workflow-scoped held run instead of resolving the token — see [Reviewer-gated installs](#reviewer-gated-installs) below.
9188
9058
  - **Untrusted contributors get no tokens.** When a fork PR is dispatched and the contributor-trust resolution returns anything other than `trusted`, the orchestrator strips both `npmRegistries` and `installEnvSecrets` out of the dispatch. The install runs without auth and fails naturally on the first private dep — fork PRs cannot ever observe a registry token, even if a misconfigured environment lacks an explicit `requiredTrustTier`.
9189
9059
  - **Lifecycle scripts disabled.** Whenever a private registry is in scope, the agent runs the install with `--ignore-scripts` (npm or pnpm alike). A malicious `preinstall` / `postinstall` hook in committed `package.json` cannot read the synthesized token env vars, even though they exist in the install subprocess. For a pnpm workspace, the agent builds your in-repo dependency closure as a separate step **after** the install's auth is torn down, so build scripts never see the tokens either.
9190
9060
  - **Stderr is redacted.** If the install fails, the agent masks every token literal out of the surfaced stderr / stdout chunks before logging.
9191
9061
  - **Job-scoped env-var names.** The synthesized auth env var is `KICI_NPM_TOKEN_<jobIdShort>_<i>` where `jobIdShort` is the first 8 chars of the dispatched job id. The name is unguessable from outside the install subprocess and not reused across jobs.
9192
9062
  - **`.npmrc` restored.** Whatever the agent appended for one install is stripped (or the file unlinked) on cleanup, so the workspace is never permanently modified.
9193
9063
 
9064
+ ## Reviewer-gated installs
9065
+
9066
+ When the named install environment carries a protection rule that holds — a required reviewer (`hold`) or a wait timer (`wait`) — the install gate **pauses the whole workflow dispatch** instead of rejecting it. The run is created in the `held` state, no jobs are queued, and a workflow-scoped row appears on the held-runs page with a `Workflow` scope badge.
9067
+
9068
+ - **Reviewer hold:** the run waits for an approver. On approval the dispatch resumes from the install gate, resolves the token, and dispatches its jobs as a normal run. On rejection the run transitions to `cancelled` — no jobs ever run.
9069
+ - **Wait timer:** the run waits out the timer and resumes automatically when it elapses.
9070
+
9071
+ A `reject` protection outcome (for example a disabled environment or a branch the environment forbids) still fails the dispatch loudly with a clear reason, exactly as before — the orchestrator never dispatches a run with an unresolved install token.
9072
+
9194
9073
  ## Limitations
9195
9074
 
9196
9075
  - **`registries:` is workflow-level only in v1.** Per-job overrides aren't supported — there is one shared `.kici/` per workspace, so a per-job `registries:` would be physically nonsensical.
9197
- - **A non-pass protection-rule outcome rejects the whole workflow dispatch.** Today, if the named environment requires reviewer approval (`hold` action) or hits concurrency (`queue` / `wait`) for the install gate, the entire workflow dispatch is rejected with a clear reason. Workflow-scoped held-runs (which would let the install wait for an approver instead of failing outright) are tracked as a follow-up — until then, choose an environment whose protection rules `pass` for the branches that need to install private deps.
9198
9076
  - **Container registries (Docker Hub, ECR, GHCR) are out of scope.** This feature covers **npm** registry auth only. Container image pulls travel through the executor backend's own credential paths.
9199
9077
 
9200
9078
  ## Observability
9201
9079
 
9202
9080
  The orchestrator exposes Prometheus counters and a histogram under the `kici_orch_install_secrets_*` prefix on its `/metrics` endpoint. They populate the **Install secrets resolution** Grafana dashboard and let operators graph install-secrets activity without digging through Loki.
9203
9081
 
9204
- | Metric | Type | Labels | What it tells you |
9205
- | ------------------------------------------------------------- | --------- | ------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
9206
- | `kici_orch_install_secrets_decisions_total` | Counter | `decision`, `reason` | Pass / reject volume. Reject reasons enumerate the failure mode: `malformed_ref`, `invalid_url_scheme`, `env_not_found`, `protection_rule_block`, `missing_token`, `missing_install_env`, etc. |
9207
- | `kici_orch_install_secrets_npm_registry_used_total` | Counter | `channel`, `provider`, `scope` | Per-channel + per-scope usage. `channel=registries` is Option A, `channel=install_env` is Option C. `scope=default` marks a no-scope default registry; `scope=-` marks Option C entries. |
9208
- | `kici_orch_install_secrets_contributor_stripped_total` | Counter | `trust_tier` | Number of dispatches where registry tokens were stripped because the contributor tier wasn't `trusted` (fork PRs from unknown / known contributors). Expected to be 0 in single-tenant orgs. |
9209
- | `kici_orch_install_secrets_token_resolution_duration_seconds` | Histogram | `environment` | Latency of per-environment secret resolution. Pathological tails (>500ms) usually mean a Vault timeout or a slow Postgres replica. |
9082
+ | Metric | Type | Labels | What it tells you |
9083
+ | ------------------------------------------------------------- | --------- | ------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
9084
+ | `kici_orch_install_secrets_decisions_total` | Counter | `decision`, `reason` | Pass / reject / hold volume. `decision=hold` (reason `held`) counts dispatches paused at a reviewer-gated install environment. Reject reasons enumerate the failure mode: `malformed_ref`, `invalid_url_scheme`, `env_not_found`, `protection_rule_block`, `missing_token`, `missing_install_env`, etc. |
9085
+ | `kici_orch_install_secrets_npm_registry_used_total` | Counter | `channel`, `provider`, `scope` | Per-channel + per-scope usage. `channel=registries` is Option A, `channel=install_env` is Option C. `scope=default` marks a no-scope default registry; `scope=-` marks Option C entries. |
9086
+ | `kici_orch_install_secrets_contributor_stripped_total` | Counter | `trust_tier` | Number of dispatches where registry tokens were stripped because the contributor tier wasn't `trusted` (fork PRs from unknown / known contributors). Expected to be 0 in single-tenant orgs. |
9087
+ | `kici_orch_install_secrets_token_resolution_duration_seconds` | Histogram | `environment` | Latency of per-environment secret resolution. Pathological tails (>500ms) usually mean a Vault timeout or a slow Postgres replica. |
9210
9088
 
9211
9089
  The dashboard JSON lives at `infra/terraform/modules/grafana/dashboards/install-secrets.json`; if you maintain your own monitoring stack, you can import it directly.
9212
9090
 
@@ -9218,6 +9096,197 @@ The dashboard JSON lives at `infra/terraform/modules/grafana/dashboards/install-
9218
9096
 
9219
9097
  ---
9220
9098
 
9099
+ ## Build provenance and attestations
9100
+
9101
+ Source: https://docs.kici.dev/user/provenance/
9102
+
9103
+ Build provenance is a signed, verifiable statement of **what produced an
9104
+ artifact** — the source repository, commit, ref, workflow path, and builder that
9105
+ ran. When a workflow step attests an artifact, KiCI records that statement,
9106
+ signs it, and makes it retrievable so anyone can later prove the artifact came
9107
+ from a specific KiCI run and was not swapped along the way.
9108
+
9109
+ This is the same idea behind supply-chain attestation systems like
9110
+ [SLSA](https://slsa.dev/spec/v1.0/provenance): a downstream consumer (a release
9111
+ gate, a security audit, a `"show me the provenance"` request) can verify the
9112
+ artifact's origin without trusting the person who handed it over.
9113
+
9114
+ ## What an attestation contains
9115
+
9116
+ An attestation is a self-contained bundle holding three things:
9117
+
9118
+ - An **in-toto SLSA v1.0 statement** describing the build: the subject artifact
9119
+ (name + content digest) and the provenance predicate (source repository,
9120
+ commit, ref, workflow, run/job identifiers, timestamps).
9121
+ - A **[DSSE](https://github.com/secure-systems-lab/dsse) signature** over that
9122
+ statement, made with an ephemeral signing key generated for the run.
9123
+ - A short-lived **OIDC identity token** issued by the KiCI platform that binds
9124
+ the signature to the build identity. The token's identity claims
9125
+ (`repository`, `ref`, `sha`, run/job ids) are derived by the platform from the
9126
+ run itself — a step cannot forge them.
9127
+
9128
+ Because the bundle carries the identity token and the public signing key, it is
9129
+ **offline-verifiable**: a verifier checks it against the platform's published
9130
+ signing keys with no per-attestation online lookup.
9131
+
9132
+ ## Attesting an artifact in a workflow
9133
+
9134
+ Call `ctx.attestProvenance({ subject })` from a step after you have produced the
9135
+ artifact:
9136
+
9137
+ ```typescript
9138
+ import { workflow, job, step } from '@kici-dev/sdk';
9139
+
9140
+ export default workflow('release', {
9141
+ on: { push: { branches: ['main'] } },
9142
+ jobs: [
9143
+ job('publish', {
9144
+ steps: [
9145
+ step('build', async (ctx) => {
9146
+ await ctx.$`npm pack`;
9147
+ }),
9148
+ step('attest', async (ctx) => {
9149
+ const result = await ctx.attestProvenance({
9150
+ subject: { name: 'my-pkg-1.2.3.tgz', path: 'my-pkg-1.2.3.tgz' },
9151
+ });
9152
+ ctx.log.info(`Attestation stored at ${result.storageKey}`);
9153
+ }),
9154
+ ],
9155
+ }),
9156
+ ],
9157
+ });
9158
+ ```
9159
+
9160
+ The **subject is caller-supplied** — you name the artifact and give KiCI either a
9161
+ path or a precomputed digest:
9162
+
9163
+ - `{ name, path }` — a path relative to the step working directory. KiCI reads
9164
+ the file and computes its SHA-256 digest.
9165
+ - `{ name, digest }` — a precomputed digest. For a container image, pass the OCI
9166
+ manifest digest your build tool emitted:
9167
+
9168
+ ```typescript
9169
+ await ctx.attestProvenance({
9170
+ subject: { name: 'ghcr.io/acme/app', digest: { sha256: '<manifest-digest>' } },
9171
+ });
9172
+ ```
9173
+
9174
+ The identity token is fetched and masked in logs automatically — you never
9175
+ handle it. The call returns `{ storageKey, subjectDigest, bundleMediaType }`
9176
+ identifying the stored bundle.
9177
+
9178
+ `ctx.attestProvenance` is only available inside a running job step. Calling it
9179
+ during local execution rejects with a clear error.
9180
+
9181
+ ### Requesting a raw identity token
9182
+
9183
+ `ctx.attestProvenance` builds on a lower-level primitive you can call directly
9184
+ when you need the identity token for a different tool:
9185
+
9186
+ ```typescript
9187
+ step('mint', async (ctx) => {
9188
+ const { token, expiresIn } = await ctx.kici.oidc.token({ audience: 'sigstore' });
9189
+ ctx.log.info(`Got an ID token valid for ${expiresIn}s`);
9190
+ // Hand `token` to a tool that exchanges it with a service trusting the issuer.
9191
+ });
9192
+ ```
9193
+
9194
+ The token is a short-lived (about 10 minutes) signed JWT scoped to the current
9195
+ run and job. Its identity claims (`repository`, `ref`, `sha`, `kici_run_id`,
9196
+ `kici_job_id`) are derived by the platform from the run context, so a step cannot
9197
+ spoof them. The returned token value is automatically masked in step logs, and
9198
+ the step never holds platform credentials — the request is relayed through the
9199
+ orchestrator, which mints the token on the step's behalf. Like
9200
+ `attestProvenance`, it is only available inside a running job step.
9201
+
9202
+ ## Verifying an attestation
9203
+
9204
+ Verify a bundle with the `kici verify-attestation` command. It establishes the
9205
+ full chain offline: the identity token verifies against the trusted issuer's
9206
+ JWKS, the DSSE signature verifies against the bundled signing key, and the
9207
+ statement's build context must match the token's identity claims (a mismatch is
9208
+ a hard failure).
9209
+
9210
+ ```bash
9211
+ kici verify-attestation [artifact] --bundle <path-or-url> --trust-root <url-or-file>
9212
+ ```
9213
+
9214
+ You supply the trusted issuer out-of-band via `--trust-root` — the verifier
9215
+ never trusts the issuer named inside the token. There are two forms:
9216
+
9217
+ - **Online — an HTTPS issuer URL.** The verifier fetches
9218
+ `<url>/.well-known/openid-configuration`, reads its `issuer` and `jwks_uri`,
9219
+ and fetches the JWKS. The token's `iss` is pinned to the discovery document's
9220
+ `issuer`.
9221
+ - **Offline — a self-contained trust-root file.** A local JSON file with the
9222
+ issuer and JWKS inlined, for air-gapped verification:
9223
+
9224
+ ```json
9225
+ {
9226
+ "issuer": "https://platform.example/issuer",
9227
+ "jwks": {
9228
+ "keys": [
9229
+ { "kty": "EC", "crv": "P-256", "x": "...", "y": "...", "alg": "ES256", "kid": "..." }
9230
+ ]
9231
+ }
9232
+ }
9233
+ ```
9234
+
9235
+ Pass an optional `[artifact]` to also digest-check the file against the
9236
+ attestation subject — this is what binds the attestation to a specific set of
9237
+ bytes. Omit it to verify the signatures and identity only. Use `--json` for a
9238
+ machine-readable result. The command exits `0` when everything verifies and `1`
9239
+ when it does not (or on an error such as a missing flag or unreachable trust
9240
+ root).
9241
+
9242
+ ```bash
9243
+ # Verify a bundle against a deployed issuer, digest-checking the artifact:
9244
+ kici verify-attestation ./dist/app.tgz \
9245
+ --bundle ./app.tgz.kici.json \
9246
+ --trust-root https://platform.example/issuer
9247
+
9248
+ # Air-gapped: verify against a self-contained trust-root file:
9249
+ kici verify-attestation ./dist/app.tgz \
9250
+ --bundle ./app.tgz.kici.json \
9251
+ --trust-root ./kici-trust-root.json
9252
+ ```
9253
+
9254
+ The full flag reference is in the [CLI reference](./cli-reference.md#kici-verify-attestation).
9255
+
9256
+ ## Viewing attestations in the dashboard
9257
+
9258
+ <!-- help:run-attestations#viewing-attestations-in-the-dashboard -->
9259
+
9260
+ The run detail page has an **Attestations** tab listing each artifact a run's
9261
+ steps attested (via `ctx.attestProvenance`), one row per artifact.
9262
+
9263
+ Each row shows:
9264
+
9265
+ - **Status** — a **verified** badge computed in your browser. It checks the
9266
+ attestation's signature, the build identity, and the build context against the
9267
+ trusted provenance issuer. **verified** (green) means all of those pass;
9268
+ **failed** (red) shows why in a tooltip; **unverifiable** means the provenance
9269
+ issuer is not configured.
9270
+ - **Job / Artifact / Digest / Created** — the producing job, the artifact name,
9271
+ its content digest, and when it was recorded.
9272
+ - **Download** — saves the signed bundle as a `.sigstore.json` file.
9273
+
9274
+ The badge does **not** re-hash the artifact bytes — the dashboard does not have
9275
+ the artifact. To bind the attestation to a specific file, run
9276
+ `kici verify-attestation <artifact> --bundle <bundle>`. A run with no
9277
+ attestations shows an empty state.
9278
+
9279
+ <!-- /help:run-attestations -->
9280
+
9281
+ ## See also
9282
+
9283
+ - [SDK runtime reference](./sdk/runtime.md) — the `ctx.attestProvenance` and
9284
+ `ctx.kici.oidc.token` step APIs in full.
9285
+ - [CLI reference](./cli-reference.md#kici-verify-attestation) — every
9286
+ `kici verify-attestation` flag and exit code.
9287
+
9288
+ ---
9289
+
9221
9290
  ## Secrets
9222
9291
 
9223
9292
  Source: https://docs.kici.dev/user/secrets/
@@ -9438,6 +9507,45 @@ export default workflow('deploy', {
9438
9507
  });
9439
9508
  ```
9440
9509
 
9510
+ ### Injecting decrypted sops values into the environment
9511
+
9512
+ KiCI does **not** scan your repository for `*.enc.yaml` files and auto-decrypt them into the environment at job init — nothing in a job runs `sops` on your behalf, and resolved secrets are never auto-injected as environment variables (see [Security notes](./sdk/runtime.md#security-notes)). Decryption is always something your workflow does explicitly: provision the age (or other) decryption key as a KiCI secret, expose it for the step, run `sops -d`, and decide what to do with the output.
9513
+
9514
+ When you want the decrypted values available as environment variables — not just written to a file — decrypt early and export the values through `$KICI_ENV` (or `ctx.setEnv`). Anything appended to `$KICI_ENV` becomes an environment variable for every later step in the same job, so a single decrypt step can populate the environment for the whole job:
9515
+
9516
+ ```typescript
9517
+ import { workflow, job, step, push } from '@kici-dev/sdk';
9518
+
9519
+ export default workflow('deploy', {
9520
+ on: push({ branches: ['main'] }),
9521
+ jobs: [
9522
+ job('decrypt-and-deploy', {
9523
+ runsOn: 'default',
9524
+ environment: 'production',
9525
+ steps: [
9526
+ step('decrypt-to-env', async (ctx) => {
9527
+ await ctx.secrets.exposeFile('SOPS_AGE_KEY_FILE', {
9528
+ sources: ctx.secrets.list().filter((k) => k.startsWith('AGE_KEY_')),
9529
+ divider: '\n',
9530
+ });
9531
+ // Decrypt to dotenv format, then append every KEY=value line to
9532
+ // $KICI_ENV so subsequent steps see them as environment variables.
9533
+ await ctx.$`sops -d --output-type dotenv secrets.enc.yaml >> "$KICI_ENV"`;
9534
+ }),
9535
+ step('deploy', async (ctx) => {
9536
+ // Values decrypted above are now ordinary env vars here.
9537
+ await ctx.$`./deploy.sh`;
9538
+ }),
9539
+ ],
9540
+ }),
9541
+ ],
9542
+ });
9543
+ ```
9544
+
9545
+ Decrypted values exported this way follow the same rules as any other `$KICI_ENV` / `ctx.setEnv` export: last-write-wins on a repeated key, and a key that collides with an operator-injected secret is ignored (the operator value wins). See [Exporting env from shell commands](./sdk/runtime.md#exporting-env-from-shell-commands-kici_env--kici_path) for the full `$KICI_ENV` contract.
9546
+
9547
+ If you only need the decrypted material as a file on disk (the common `kubectl` / `gcloud` case), skip the env hop and redirect to a file instead — see the [canonical sops example](#canonical-sops-example) above.
9548
+
9441
9549
  ## API reference
9442
9550
 
9443
9551
  | Method | Signature | Description |
@@ -9800,6 +9908,123 @@ and confirm the App is installed on that repo.
9800
9908
 
9801
9909
  ---
9802
9910
 
9911
+ ## Local filesystem (file://) source
9912
+
9913
+ Source: https://docs.kici.dev/user/providers/local-file/
9914
+
9915
+ The **local** provider lets the orchestrator run workflows from a git
9916
+ repository that already lives on the agent's filesystem — cloned via a
9917
+ `file://` URL rather than fetched from a remote forge. There is no webhook
9918
+ from GitHub or GitLab; the operator drives runs explicitly with the
9919
+ `kici-admin` CLI or a generated `post-receive` hook.
9920
+
9921
+ > **This is an operator-curated source, not daily developer CI.** Use it for
9922
+ > globally-registered or policy workflows the operator controls — a vendored
9923
+ > repo baked into an agent image, a repo synced onto a host out-of-band, a
9924
+ > golden internal pipeline. For ordinary per-developer CI driven by pull
9925
+ > requests and pushes, use the [GitHub App provider](./github.md) or the
9926
+ > [universal-git provider](./universal-git.md) against a real forge.
9927
+
9928
+ ## Trust caveat (read first)
9929
+
9930
+ A local source uses signature verification `none` — there is no remote forge
9931
+ to sign the webhook payload, so the orchestrator cannot authenticate the
9932
+ trigger. **Only register repos you trust.** Anyone who can reach the
9933
+ orchestrator's webhook route for this source, or push to the repo on disk,
9934
+ can drive a run. Treat the repo path as a trusted operator input, the same
9935
+ way you treat the orchestrator's own configuration.
9936
+
9937
+ ## Register a source
9938
+
9939
+ ```bash
9940
+ kici-admin source add local \
9941
+ --org <orgId> \
9942
+ --path /abs/path/to/repo \
9943
+ --name my-local-repo
9944
+ ```
9945
+
9946
+ - `--path` must be an **absolute** directory on the agent filesystem. It is
9947
+ the base path the orchestrator's lock-file fetcher reads
9948
+ (`<path>/.kici/kici.lock.json`) and the base for the `file://` clone the
9949
+ agent performs.
9950
+ - `--clone-url-base <url>` is optional. By default the agent clones via
9951
+ `file://<path>`. Supply a `git://` or `http://` base when the agent does
9952
+ **not** share the orchestrator's filesystem and must fetch the repo over a
9953
+ git server instead (see "Per-scaler reachability" below).
9954
+
9955
+ Update the path or name later:
9956
+
9957
+ ```bash
9958
+ kici-admin source update-local <id> --path /new/abs/path
9959
+ kici-admin source update-local <id> --name new-name
9960
+ ```
9961
+
9962
+ Remove it:
9963
+
9964
+ ```bash
9965
+ kici-admin source remove <routingKey> --local
9966
+ ```
9967
+
9968
+ List and inspect (local sources render their `repoBasePath`):
9969
+
9970
+ ```bash
9971
+ kici-admin source list --org <orgId>
9972
+ kici-admin source get <id>
9973
+ ```
9974
+
9975
+ ## Trigger runs
9976
+
9977
+ A local repo has no forge to send webhooks, so you trigger runs yourself.
9978
+
9979
+ **One-shot, by hand:**
9980
+
9981
+ ```bash
9982
+ kici-admin source trigger-local <id>
9983
+ ```
9984
+
9985
+ The command reads the repo's current HEAD ref and commit SHA, builds a
9986
+ GitHub-shaped `push` payload, and POSTs it to the orchestrator's generic
9987
+ webhook route. Override the ref/sha/event explicitly when needed:
9988
+
9989
+ ```bash
9990
+ kici-admin source trigger-local <id> --event push --ref refs/heads/main --sha <sha>
9991
+ ```
9992
+
9993
+ **On every push, via a hook:**
9994
+
9995
+ ```bash
9996
+ kici-admin source install-hook <id>
9997
+ ```
9998
+
9999
+ This writes a `post-receive` hook into the repo so that every push to it
10000
+ triggers a run automatically — the local equivalent of a forge webhook.
10001
+
10002
+ ## Per-scaler reachability (operator's responsibility)
10003
+
10004
+ The orchestrator accepts a local source on **any** scaler backend and does
10005
+ **not** verify that the repo is actually reachable inside the agent. Making
10006
+ the path reachable is the operator's job. On a container or Firecracker
10007
+ scaler the orchestrator logs a reachability warning when it registers the
10008
+ source, but it does not reject it.
10009
+
10010
+ | Scaler | How the repo must be reachable in the agent |
10011
+ | ----------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
10012
+ | bare-metal | The agent runs on the host, so the `--path` host directory is used directly. |
10013
+ | container | Bake the repo into the agent image, or bind-mount it at the **same absolute path** the source was registered with. Alternatively register `--clone-url-base` pointing at a git server the container can reach. |
10014
+ | Firecracker | The repo must be present on the microVM rootfs at the registered path, or reachable via a `--clone-url-base` git server. |
10015
+
10016
+ If the path is not reachable inside the agent, the clone fails at run time —
10017
+ the run is created and then fails, rather than being silently dropped.
10018
+
10019
+ ## See also
10020
+
10021
+ - [Universal-git provider](./universal-git.md) — for a remote forge (or any
10022
+ `http://` git server) when there is no shared filesystem.
10023
+ - [GitHub App provider](./github.md) — the flagship source for pull-request
10024
+ CI with Checks.
10025
+
10026
+ ---
10027
+
9803
10028
  ## Universal-git provider
9804
10029
 
9805
10030
  Source: https://docs.kici.dev/user/providers/universal-git/
@@ -9826,6 +10051,13 @@ No mirror, no GitHub App, no `checkout: false` escape hatch. The same
9826
10051
  trigger matching, global-workflow policy, and agent execution pipeline
9827
10052
  that back the GitHub App source also serve universal-git sources.
9828
10053
 
10054
+ > **No shared filesystem between orchestrator and agent?** Universal-git is
10055
+ > the right choice for the **remote-agent** case — point it at an `http://`
10056
+ > git server and the agent clones over the network. When the repo instead
10057
+ > lives on the agent's own filesystem (a vendored / operator-curated repo),
10058
+ > use a [local `file://` source](./local-file.md) and drive it with the
10059
+ > `kici-admin` CLI.
10060
+
9829
10061
  ## Which preset do I need?
9830
10062
 
9831
10063
  KiCI ships canonical presets so you don't have to spell out JSONPath for
@@ -10045,9 +10277,9 @@ with the right PEM.
10045
10277
 
10046
10278
  Source: https://docs.kici.dev/architecture/data-flows/
10047
10279
 
10048
- This document describes the key data flows through the KiCI architecture: webhook delivery, job execution, dependency caching, re-run and cancel, trace ID propagation, internal event routing, and generic webhook ingestion.
10280
+ This document describes the key data flows through the KiCI architecture: webhook delivery, job execution, developer-initiated remote runs, dependency caching, re-run and cancel, trace ID propagation, internal event routing, and generic webhook ingestion.
10049
10281
 
10050
- > **Lock file schema version:** The lock file uses schema version 15, which adds per-job init config on top of v14's declarative cache specs, v11's `LockInlineValue` for pure function inline evaluation, v10's simplified negative patterns (! prefix in repos/paths arrays), v9's global workflow repos matching, and v8's runsOn polymorphic type support.
10282
+ > **Lock file schema version:** The lock file uses schema version 17, which widens per-job init to typed presets (`mise` / `{ mise }`) and `auto` detection on top of v16's normalized approval config, v15's per-job init config, v14's declarative cache specs, v11's `LockInlineValue` for pure function inline evaluation, v10's simplified negative patterns (! prefix in repos/paths arrays), v9's global workflow repos matching, and v8's runsOn polymorphic type support.
10051
10283
 
10052
10284
  ## Webhook delivery flow
10053
10285
 
@@ -10117,6 +10349,41 @@ The agent delegates job execution to an `ExecutionSandbox` (container, bare-meta
10117
10349
  7. **Report** -- Send final `job.status` back to orchestrator with step results and timing
10118
10350
  8. **Cleanup** -- Tear down sandbox and remove work directory
10119
10351
 
10352
+ ## Remote run flow (`kici run remote`)
10353
+
10354
+ A developer running `kici run remote` from a working tree initiates a run through the three-tier relay without a provider webhook. The flow splits into two independent planes: a **control plane** through the Platform relay, and a **data plane** that uploads the working-tree overlay straight to object storage.
10355
+
10356
+ ```
10357
+ Developer machine Platform relay Orchestrator Object store
10358
+ | | | |
10359
+ |-- upload-init (control) ----->|--- WS relay ----------->| |
10360
+ | (org, cluster, overlay | |-- mint presigned |
10361
+ | metadata, inline lock) | | PUT URL ---------->|
10362
+ |<-- presigned PUT URL ---------|<--- WS relay -----------| |
10363
+ | | | |
10364
+ |== overlay tarball PUT (data plane) ===========================================>|
10365
+ | | | |
10366
+ |-- trigger (control) --------->|--- WS relay ----------->|-- dispatch jobs |
10367
+ | | | (agents fetch |
10368
+ | | | overlay) |
10369
+ |-- poll logs + status -------->|--- WS relay ----------->| |
10370
+ |<-- log chunks + status -------|<--- WS relay -----------| |
10371
+ ```
10372
+
10373
+ ### Control plane
10374
+
10375
+ Run initiation (`upload-init`), the trigger, status polling, log retrieval, and cancellation all flow from the developer machine to the Platform, which relays them over its WebSocket connection to the org's orchestrator. The developer machine never talks to the orchestrator's HTTP API directly. Logs are delivered by the CLI polling the Platform for log chunks — tracked by a monotonic line cursor — and run status until the run reaches a terminal state; there is no direct streaming socket between the developer machine and the orchestrator.
10376
+
10377
+ ### Data plane
10378
+
10379
+ The working-tree overlay tarball uploads **directly** from the developer machine to the orchestrator's object store via a pre-signed PUT URL minted during `upload-init`. The overlay bytes never pass through the Platform. Because of this split, only the object store needs to be reachable from the developer machine — the orchestrator can sit behind a private network. See [Storage layout](../operator/orchestrator/storage-layout.md) for the upload-endpoint configuration.
10380
+
10381
+ ### Org anchor
10382
+
10383
+ The run is dispatched to the developer's active organization (selected with `kici org use`, or overridden per-run). The orchestrator anchors its bound organization with a system-managed **remote source** (routing key `remote:<orgId>`) that it auto-provisions — no manual webhook source is required, so a zero-source org is immediately routable for remote runs. The Platform forces the run's routing key to `remote:<orgId>` server-side; the developer never sets a routing key. When an org has more than one connected orchestrator cluster, the CLI selects the target cluster explicitly (or relies on the per-org default), and a single connected cluster is auto-selected.
10384
+
10385
+ Remote runs are offered by the Platform; an orchestrator with no Platform connection cannot serve them. Executing workflow steps on the developer machine with no orchestrator is the separate `kici run local` path.
10386
+
10120
10387
  ## Source and dependency caching flow
10121
10388
 
10122
10389
  KiCI runs two orchestrator-side caches — the **source tarball cache** (raw `.kici/` directory minus `node_modules/`) and the **dependency tarball cache** (packed `node_modules/`). Both use a build-then-execute pattern: the orchestrator checks the caches before dispatching execution jobs, and if the source cache is cold a build agent populates both in one pass.
@@ -10218,7 +10485,7 @@ Dep cache misses alone do **not** trigger a build job. Deps are platform-specifi
10218
10485
 
10219
10486
  ### Cross-source / no-contentHash workflows
10220
10487
 
10221
- - **Lock files without `contentHash`** (schema v1) skip the source cache entirely; agents compile from source. Regenerate lock files with `kici compile` to enable caching. The current lock file schema version is 15.
10488
+ - **Lock files without `contentHash`** (schema v1) skip the source cache entirely; agents compile from source. Regenerate lock files with `kici compile` to enable caching. The current lock file schema version is 17.
10222
10489
  - **Cross-source / global-workflow dispatch** (a workflow registered against source A fired by a webhook on source B) bypasses both caches. The registration's lock file entry still carries `contentHash`, but the cross-source path always clone-and-installs — the eval temp dir doesn't ship `@kici-dev/sdk`. The execution agent still verifies `contentHash` against the cloned source for drift detection.
10223
10490
 
10224
10491
  ### Build deduplication
@@ -10935,7 +11202,7 @@ The agent is the execution worker. It runs on customer infrastructure and has fu
10935
11202
 
10936
11203
  ### `@kici-dev/engine`
10937
11204
 
10938
- Shared business logic used by all three tiers. Single source of truth for cross-tier concerns. Has no internal `@kici-dev/*` dependencies (only zod, picomatch, and jsonpath-plus).
11205
+ Shared business logic used by all three tiers. Single source of truth for cross-tier concerns. Has no internal `@kici-dev/*` dependencies -- only a handful of third-party libraries.
10939
11206
 
10940
11207
  - Protocol message schemas (Zod-based, direction-specific unions including dashboard REST-over-WS, browser live streaming, test run lifecycle, observer channel, log pull, run events, peer-to-peer, cluster join, and source registration)
10941
11208
  - Provider interfaces (WebhookNormalizer, LockFileFetcher, ChangedFilesFetcher, CloneTokenProvider, RepoUrlBuilder, ContributorResolver, CheckStatusPoster)
@@ -10947,7 +11214,7 @@ Shared business logic used by all three tiers. Single source of truth for cross-
10947
11214
  - Environment allowlist (safe env var filtering)
10948
11215
  - Secrets management (secret context resolution)
10949
11216
  - Environment model (scoped secrets, env merge, protection gates)
10950
- - Label utilities (platform label derivation, runsOn normalization, `kici:*` reserved namespace, role labels)
11217
+ - Label utilities (platform label derivation, runsOn normalization, `kici:*` set-only reserved namespace, role labels)
10951
11218
  - Audit policy and retention (per-action access-log sampling, warm-retention windows for cold-store eligibility, federated activity row schema)
10952
11219
  - Scaler backend type enum (`container`, `bare-metal`, `firecracker`, `kubernetes`)
10953
11220
  - Registration trigger type enum (registerable trigger discriminator)