@kici-dev/orchestrator 0.4.0 → 0.5.0

package/dist/reporting/check-run-reporter.d.ts

@@ -87,6 +87,15 @@ interface SetPendingOptions {
     repo: string;
     sha: string;
     workflowName: string;
+    /**
+     * The repository that DEFINES the workflow, when that is not the repository
+     * the run acted on — an organization-wide workflow dispatched against
+     * another repository. Qualifies the check-run name so the run cannot share a
+     * check run with a same-named workflow of the acted-on repository; see
+     * `workflowLabel`. Passing the acted-on repository here is a no-op, so a
+     * caller cannot change a per-repository run's name by accident.
+     */
+    workflowRepoIdentifier?: string;
     jobNames: string[];
     installationId?: number;
     /** Routing key for per-app credential lookup (e.g., "github:12345"). */
@@ -105,6 +114,8 @@ interface SetBuildPendingOptions {
     repo: string;
     sha: string;
     workflowName: string;
+    /** See {@link SetPendingOptions.workflowRepoIdentifier}. */
+    workflowRepoIdentifier?: string;
     installationId?: number;
     /** Routing key for per-app credential lookup (e.g., "github:12345"). */
     routingKey?: string;
@@ -122,6 +133,8 @@ interface SetBuildCompleteOptions {
     repo: string;
     sha: string;
     workflowName: string;
+    /** See {@link SetPendingOptions.workflowRepoIdentifier}. */
+    workflowRepoIdentifier?: string;
     status: TerminalJobStatus;
     installationId?: number;
     /** Routing key for per-app credential lookup (e.g., "github:12345"). */
@@ -141,6 +154,8 @@ interface UpdateJobStatusOptions {
     repo: string;
     sha: string;
     workflowName: string;
+    /** See {@link SetPendingOptions.workflowRepoIdentifier}. */
+    workflowRepoIdentifier?: string;
     jobName: string;
     state: TerminalJobStatus;
     installationId?: number;
@@ -167,6 +182,8 @@ interface UpdateWorkflowStatusOptions {
     repo: string;
     sha: string;
     workflowName: string;
+    /** See {@link SetPendingOptions.workflowRepoIdentifier}. */
+    workflowRepoIdentifier?: string;
     overallStatus: TerminalJobStatus;
     installationId?: number;
     /** Routing key for per-app credential lookup (e.g., "github:12345"). */
@@ -186,6 +203,8 @@ interface UpdateStepProgressOptions {
     repo: string;
     sha: string;
     workflowName: string;
+    /** See {@link SetPendingOptions.workflowRepoIdentifier}. */
+    workflowRepoIdentifier?: string;
     jobName: string;
     stepIndex: number;
     stepName: string;
@@ -321,6 +340,8 @@ export declare class CheckRunReporter {
         repo: string;
         sha: string;
         workflowName: string;
+        /** See {@link SetPendingOptions.workflowRepoIdentifier}. */
+        workflowRepoIdentifier?: string;
         jobNames: string[];
     }): void;
     /**
@@ -337,7 +358,8 @@ export declare class CheckRunReporter {
      * Separate from execution check runs so users see build progress independently.
      * Fire-and-forget: errors are logged but don't block the pipeline.
      *
-     * Check run name format: kici/{workflowName}/setup
+     * Check run name format: kici/{workflowName}/setup — prefixed with the
+     * defining repository for a cross-repository global run, see `workflowLabel`.
      */
     setBuildPending(opts: SetBuildPendingOptions): void;
     /**
@@ -390,6 +412,13 @@ export declare class CheckRunReporter {
      * Read-through helper: look up a check-run ID. Checks L1 first, falls
      * through to the store on miss, caches the result on hit. Returns
      * undefined when neither layer has the ID — the caller logs + skips.
+     *
+     * The same row also rehydrates the `terminalSent` latch. Both L1 entries are
+     * dropped together by `cleanupRun`, and only one of them used to come back:
+     * the id reloaded from here while the latch did not, which is exactly the
+     * pair that lets a late step-progress update resolve a check run and PATCH
+     * `status: in_progress` over its completion. Reading `terminal_sent_at` off
+     * the row this query already selects costs nothing and closes that gap.
      */
     private resolveCheckRunId;
     /**
@@ -425,6 +454,28 @@ export declare class CheckRunReporter {
      */
     private appendTraceIds;
     private checkRunKey;
+    /**
+     * The workflow label every check-run name and title is built from.
+     *
+     * A check run's identity is `(owner, repo, sha, check name)` — on the
+     * provider, in `check_run_tracking`'s primary key, and in this class's L1
+     * keys. There is no run id anywhere in it. Two per-repository runs cannot
+     * collide on that identity, because one lock file cannot define a workflow
+     * name twice. An organization-wide workflow can: it is defined in ANOTHER
+     * repository, so its name is free to equal a workflow name of the repository
+     * it was dispatched against, and on the same commit the two runs then resolve
+     * to one check run. The global run's conclusion would complete the acted-on
+     * repository's check — the signal branch protection reads — and point its
+     * `details_url` at the wrong run, while `cleanupRun` would evict the other
+     * run's check-run id and terminal latch on prune.
+     *
+     * Qualifying the label with the defining repository keeps them apart. The
+     * "differs from the acted-on repository" narrowing lives here rather than at
+     * the call sites, so a per-repository run's name — which is customer-visible
+     * and may sit in a branch-protection required-check list — cannot move
+     * because a caller passed the field where it did not apply.
+     */
+    private workflowLabel;
     /**
      * Build the `details_url` for a check run pointing at the dashboard's
      * public-alias resolver (`/r/orgs/<oal_xxx>/runs/<runId>`). Returns

package/dist/reporting/execution-tracker.d.ts

@@ -28,6 +28,14 @@ export interface ExecutionContext {
     installationId?: number;
     requestId?: string;
     routingKey?: string;
+    /**
+     * The repository that DEFINES the workflow, when that is not
+     * `repoIdentifier` — a global workflow authored in one repository and
+     * dispatched against another. Set only when the two differ, so "present"
+     * marks a cross-repository global run; forwarded to the Platform on
+     * `execution.status`.
+     */
+    workflowRepoIdentifier?: string;
     /** Git branch or tag (e.g. "main", "feature/foo"). */
     ref?: string;
     /** Trigger event type (e.g. "push", "pr:open"). */
@@ -302,7 +310,14 @@ export declare class ExecutionTracker {
     /** Agent provenance label when triggered through an agent credential. */
     triggeredByAgentLabel?: string | null, 
     /** Pull-request number for PR-triggered runs; null/omitted for non-PR runs. */
-    prNumber?: number | null): Promise<void>;
+    prNumber?: number | null, 
+    /**
+     * The repository that DEFINES the workflow, when that is not
+     * `repoIdentifier` — an organization-wide workflow authored in one
+     * repository and dispatched against another. Omitted/null for every
+     * per-repository run, where the two are the same repository.
+     */
+    workflowRepoIdentifier?: string | null): Promise<void>;
     /**
      * Upsert one execution_jobs row per dispatched job (idempotent on
      * (run_id, job_id) to tolerate a race with an early `onJobStatus`). The
@@ -515,6 +530,58 @@ export declare class ExecutionTracker {
         initFailure: InitFailure;
         triggerEvent?: string;
         commitMessage?: string;
+        /**
+         * The repository that DEFINES the workflow this run was about to execute.
+         * REQUIRED: every run has one, and a caller that does not state it must not
+         * compile. Left optional, a global dispatch path added later would record a
+         * null marker — and a null marker does not mean "unknown", it means "the
+         * workflow lives in this run's own repository", which the whole
+         * defining-repository predicate then reads as fact.
+         *
+         * Narrowed to the differing case here, exactly as `onExecutionStarted`
+         * does, so passing `repoIdentifier` (the per-repository case) is correct
+         * and records nothing.
+         */
+        workflowRepoIdentifier: string;
+    }): Promise<void>;
+    /**
+     * Write one `failed` execution_runs row for a global eval round that never
+     * produced verdicts.
+     *
+     * The round decides which organization-wide workflows apply to an event, so a
+     * round that fails suppresses every workflow it was deciding on. Without a row
+     * that outcome is invisible: no run was ever created for those workflows, so
+     * there is nothing on the dashboard to explain why they did not appear.
+     *
+     * **One row for the whole round, not one per workflow.** The round exists to
+     * collapse N candidate workflows into a single pre-run job; fanning its
+     * failure back out into N rows would undo exactly that. The reason names every
+     * affected workflow instead.
+     *
+     * Its `runId` is the round job's own — the last attempt's — so the row, the
+     * `dispatch_queue` row, and the attempt's logs all carry one id. No conflict
+     * guard is needed beyond `doNothing`: the id was minted for this round and
+     * belongs to no other run.
+     */
+    recordGlobalEvalRoundFailureRun(args: {
+        runId: string;
+        workflowName: string;
+        provider: string;
+        repoIdentifier: string;
+        ref: string;
+        sha: string;
+        deliveryId: string | null;
+        providerContext: Record<string, unknown>;
+        routingKey: string;
+        failureReason: string;
+        triggerEvent?: string;
+        /**
+         * The repository whose global workflows the round was deciding. A round is
+         * definitionally the cross-repository case, so this is normally a different
+         * repository from `repoIdentifier` — recorded, like everywhere else, only
+         * when the two actually differ.
+         */
+        workflowRepoIdentifier: string;
     }): Promise<void>;
     /**
      * Record a run paused at the workflow install gate (a `registries:` /
@@ -549,6 +616,13 @@ export declare class ExecutionTracker {
          * security hold fail-closed unreachable by the comment path.
          */
         prNumber?: number | null;
+        /**
+         * The repository that DEFINES the held workflow. REQUIRED for the same
+         * reason as on {@link ExecutionTracker.recordInitFailureRun}: a hold is a
+         * live, resumable run, so a null marker here would misattribute it for the
+         * whole time it sits in the queue and for the run it resumes into.
+         */
+        workflowRepoIdentifier: string;
     }): Promise<void>;
     /**
      * Flip a `held` run back to `pending` so the resumed dispatch can proceed
@@ -593,11 +667,13 @@ export declare class ExecutionTracker {
      * registered. Returns true when a token was taken, false for an unknown or
      * already-completed run (nothing to hold).
      *
-     * Three registration windows need this. A run whose source-pack `__build__`
+     * Four registration windows need this. A run whose source-pack `__build__`
      * job is dispatched first is registered with that job ALONE, and its real
-     * jobs are only dispatched once the build finishes. A deferred init job and a
-     * deferred dynamic entry each register their jobs from a fire-and-forget task
-     * that outlives the dispatch call. In every case, without a token the already
+     * jobs are only dispatched once the build finishes; a run without a build is
+     * registered with NO jobs before its dispatch loop, so its jobs land once
+     * every dispatch has returned. A deferred init job and a deferred dynamic
+     * entry each register their jobs from a fire-and-forget task that outlives
+     * the dispatch call. In every case, without a token the already
      * -registered jobs reaching a terminal state satisfies {@link isRunComplete},
      * so the run is finalized early: a terminal run status is written, the
      * provider check is posted, and the status is forwarded to the Platform — all
@@ -612,9 +688,13 @@ export declare class ExecutionTracker {
      * and an off-by-one job count for the length of the window. A counter holds
      * the completion check open without being visible to any of them, and it
      * writes no `execution_jobs` row, so it never surfaces as a phantom job.
-     * A token whose holder never settles keeps the run `running` until the stale
+     * A token whose holder never settles keeps the run open until the stale
      * detector reaps it — the same backstop that covers a job that never
-     * reports.
+     * reports. That backstop only reaches a run that has at least one job: every
+     * sub-scan of the stale detector starts from `execution_jobs` or
+     * `dispatch_queue`. So a holder that registers NO jobs must terminalize the
+     * run itself rather than rely on the sweep — which is what
+     * `dispatchMatchedWorkflow` does when its dispatch loop throws.
      *
      * Each token must be paired with exactly one {@link releasePendingJobsHold}.
      */
@@ -645,6 +725,35 @@ export declare class ExecutionTracker {
      * Used by LogWriter to construct log paths.
      */
     getJobName(runId: string, jobId: string): string | undefined;
+    /**
+     * Resolve the job name that names a step-log storage path, with a durable
+     * fallback. The step-log path is `executions/{runId}/job-{jobName}/…` and it
+     * is written from two sites — the chunk writer (`log-chunk-sink`) and the
+     * `execution_steps.log_path` upsert in `onStepStatus`. A purely in-memory
+     * lookup (`getJobName`) returns `undefined` during the brief window between
+     * dispatch and `addJobsToRun` populating `run.jobs`, so an early chunk would
+     * land under `job-{jobId}` while the reader later keys on `job-{realName}` —
+     * silently losing those lines. Both write sites go through this resolver so
+     * they cannot disagree: in-memory first, then the authoritative
+     * `dispatch_queue.job_name` (the same source `recoverJobFromDispatchQueue`
+     * trusts), and only `jobId` when the job is genuinely unknown.
+     */
+    resolveJobName(runId: string, jobId: string): Promise<string>;
+    /**
+     * Seed the in-memory job name for a job the coordinator did not itself
+     * dispatch — a worker-rerouted job. Such a job is tracked in the cluster
+     * coordinator's own map, never enters this tracker's `run.jobs` via
+     * `addJobsToRun`, and has no coordinator-owned `dispatch_queue` row (the
+     * worker owns it under a fresh id), so `resolveJobName` would fall back to
+     * the bare `jobId` for its early log chunks while the reader keys on the real
+     * name — the same split-brain that loses lines. Seeding the name here, at
+     * reroute time, makes `resolveJobName`'s fast path correct from the first
+     * relayed chunk, independent of when the worker's `dispatch_queue` row becomes
+     * visible. In-memory only: the `execution_jobs` row is still created lazily by
+     * the worker's first status update. Guarded against clobbering an existing
+     * entry so a later terminal status is never overwritten.
+     */
+    registerJobName(runId: string, jobId: string, jobName: string): void;
     /**
      * Find the runId of the in-memory run that owns a given dispatched jobId.
      *
@@ -697,6 +806,7 @@ export declare class ExecutionTracker {
         status: ExecutionRunStatus;
         routingKey?: string;
         repoIdentifier?: string;
+        workflowRepoIdentifier?: string;
         sha?: string;
         ref?: string;
         triggerEvent?: string;

package/dist/reporting/log-chunk-sink.d.ts

@@ -7,9 +7,12 @@
  * behave identically — a worker-dispatched job's logs are persisted, counted
  * and forwarded exactly like a locally-dispatched one's.
  *
- * The storage key is derived from `executionTracker.getJobName`, the same call
- * `ExecutionTracker.onStepStatus` uses to fill `execution_steps.log_path`, so
- * the reader and the writer cannot disagree about the naming rule.
+ * The storage key is derived from `executionTracker.resolveJobName`, the same
+ * call `ExecutionTracker.onStepStatus` uses to fill `execution_steps.log_path`,
+ * so the reader and the writer cannot disagree about the naming rule — the
+ * resolver falls back to `dispatch_queue.job_name` during the dispatch window
+ * before in-memory job state is populated, which is what stops an early chunk
+ * from being persisted under an unreadable `job-{jobId}` path.
  */
 import type { LogStream } from '@kici-dev/engine';
 import type { LogWriter } from './log-writer.js';
@@ -32,9 +35,9 @@ export interface LogChunkSinkDeps {
     stepLogBuffer?: StepLogBuffer;
     /** Durable step-log persistence. Absent when the orchestrator has no database. */
     logWriter?: LogWriter;
-    /** Resolves the job name that names the storage path. */
+    /** Resolves the job name that names the storage path (durable fallback). */
     executionTracker?: {
-        getJobName(runId: string, jobId: string): string | undefined;
+        resolveJobName(runId: string, jobId: string): Promise<string>;
     };
     /**
      * Forward to the Platform for browser fan-out. Absent in independent mode.

package/dist/routes/admin-org-settings.d.ts

@@ -18,6 +18,11 @@ import type { AccessLogWriter } from '../audit/access-log.js';
 interface OrgSettingsRouteDeps {
     db: Kysely<Database>;
     rbac: RbacEnforcer;
+    /**
+     * Applies when `cluster_settings.global_workflows_enabled` is NULL —
+     * `config.globalWorkflowsEnabled`.
+     */
+    globalWorkflowsEnabledDefault: boolean;
     /**
      * Optional — when wired, each `dashboard_write_policy` flip emits one
      * `access_log` row (`org_settings.dashboard_write_policy.update`)

package/dist/routes/admin.d.ts

@@ -40,6 +40,12 @@ export interface AdminRouteDeps {
     rbac: RbacEnforcer;
     secretStore: PgSecretStore;
     auditLogger: AuditLogger;
+    /**
+     * Fleet-wide default for the global-workflows master switch, applied when the
+     * cluster column is NULL. Optional so WS-only / test admins can omit it;
+     * omitting it means the secure default (disabled).
+     */
+    globalWorkflowsEnabledDefault?: boolean;
     /**
      * Optional -- the coordinator drain controller. When provided, the
      * `POST`/`GET /api/v1/admin/orchestrator/drain` routes are mounted (backing

package/dist/scaler/manager.d.ts

@@ -422,16 +422,18 @@ export declare class ScalerManager {
      */
     onAgentDisconnected(agentId: string): void;
     /**
-     * Look up the scaler backend name (e.g. `container`, `firecracker`,
+     * Look up the scaler backend TYPE (`container`, `firecracker`,
      * `bare-metal`) for a registered agent. Returns null if the agent is
-     * not scaler-managed (i.e. a static / stateful agent).
+     * not scaler-managed (a static / stateful agent).
      *
-     * Used by AgentMetricsAggregator to inject a `scaler` label on each
-     * `kici_agent_*` series so dashboards can split per-scaler instead of
-     * per-agent_id (which is too high-cardinality and doesn't tell the
-     * operator which pool is loaded).
-     */
-    getBackendName(agentId: string): string | null;
+     * Used by AgentMetricsAggregator to stamp the `scaler` label on each
+     * kici_agent_* series. The label MUST be the backend type, not the
+     * operator-chosen scaler name: the Platform catalog filter constrains
+     * it to AGENT_SCALER_VALUES (the four types), so a free-form name is
+     * dropped as bad_label_value. `managedAgentIndex` stores the scaler
+     * name, so resolve it through `backends` to the type.
+     */
+    getBackendType(agentId: string): string | null;
     /**
      * Called from agent-handler.ts when an agent sends config.ack.
      * For Firecracker agents, clears MMDS data (belt-and-suspenders with in-VM iptables).

package/dist/security/global-workflow-policy.d.ts

@@ -1,14 +1,25 @@
 import type { Kysely } from 'kysely';
 import type { Database } from '../db/types.js';
+import type { ClusterSettingsReader } from '../cluster/cluster-settings-reader.js';
 interface GlobalWorkflowPermission {
     allowed: boolean;
     reason?: string;
 }
 /**
- * Org-level permission enforcement for global workflows.
+ * Permission enforcement for global workflows.
  *
- * Three independent axes (each stored as a jsonb array of
- * `{routingKey?, pattern}` entries on `org_settings`):
+ * A fleet-wide master switch gates everything first:
+ * `cluster_settings.global_workflows_enabled`, held by the orchestrator
+ * operator through `kici-admin cluster-settings` (NULL ⇒ the configured
+ * `KICI_GLOBAL_WORKFLOWS_ENABLED` default, off by default). When the switch is
+ * off — or the cluster row is unreadable, which fails closed — no repo may
+ * register or dispatch a global workflow, whatever the per-org lists say.
+ *
+ * With the switch on, three independent per-org axes apply (each stored as a
+ * jsonb array of `{routingKey?, pattern}` entries on `org_settings`). A missing
+ * `org_settings` row means "no per-org restrictions" for the repo and source
+ * axes, not a denial; elevated access still requires an explicit list, so a
+ * missing row grants none.
  *
  * - Workflow-repo allow-list (`global_workflow_allowed_repos`): which repos
  *   may author global workflows. null/empty = any repo. Checked at
@@ -32,15 +43,32 @@ interface GlobalWorkflowPermission {
  */
 export declare class GlobalWorkflowPolicy {
     private readonly db;
-    constructor(db: Kysely<Database>);
+    private readonly clusterSettings;
+    /** Applies when the cluster column is NULL — `config.globalWorkflowsEnabled`. */
+    private readonly defaultEnabled;
+    constructor(db: Kysely<Database>, clusterSettings: ClusterSettingsReader, 
+    /** Applies when the cluster column is NULL — `config.globalWorkflowsEnabled`. */
+    defaultEnabled: boolean);
+    /**
+     * The fleet-wide master gate, consulted before any per-org list.
+     *
+     * Returns `undefined` when the gate passes, or the denial to return
+     * verbatim when it does not — so all three axes share one decision and one
+     * pair of reasons.
+     *
+     * Fails closed on `{ ok: false }`. That case is a database fault, not an
+     * operator choice, and answering it from `defaultEnabled` would open the
+     * feature during an outage the moment that default is ever flipped.
+     */
+    private clusterGate;
     /**
      * Check whether a workflow-authoring repository is allowed to register or
      * dispatch global workflows. The workflow's own routing key is matched
      * against each entry's optional `routingKey` qualifier.
      *
      * Decision flow:
-     * 1. No org_settings row or globally disabled → not allowed (opt-in).
-     * 2. allowed list null/empty → any repo allowed.
+     * 1. Cluster master switch off or unreadable → not allowed (fails closed).
+     * 2. allowed list null/empty (or no org_settings row) → any repo allowed.
      * 3. Entry matches when its `routingKey` is absent OR equals
      *    `workflowRoutingKey`, AND its pattern matches the workflow repo.
      * 4. Otherwise → not allowed.
@@ -52,18 +80,30 @@ export declare class GlobalWorkflowPolicy {
      * (so a source-qualified deny only applies to that specific source).
      *
      * Decision flow:
-     * 1. No org_settings row or globally disabled → not allowed (opt-in).
-     * 2. denied list null/empty → allowed.
+     * 1. Cluster master switch off or unreadable → not allowed (fails closed).
+     * 2. denied list null/empty (or no org_settings row) → allowed.
      * 3. Any deny entry whose routing key (if set) equals the event's routing
      *    key AND whose pattern matches the source repo → not allowed.
      */
     isSourceRepoAllowed(eventRoutingKey: string, sourceRepoIdentifier: string, customerId: string): Promise<GlobalWorkflowPermission>;
     /**
-     * Check whether a workflow-authoring repository has elevated access (i.e.,
-     * may read source-repo secrets during execution). The workflow's routing
-     * key is matched against each entry's optional `routingKey` qualifier.
+     * Check whether a workflow-authoring repository is on the elevated-access
+     * list. The workflow's routing key is matched against each entry's optional
+     * `routingKey` qualifier. Returns false if no org_settings row or the
+     * elevated list is null/empty.
+     *
+     * @deprecated Not enforced, and not enforceable in this shape. It was meant
+     * to gate a global workflow's job reading the *source* repository's secrets,
+     * but the organization-wide dispatch path resolves no secrets at all — it
+     * binds no secret contexts, and writes no secret material into a job config
+     * (asserted by `pipeline/process-webhook-globals-secrets.test.ts`). So there
+     * is no injection for a grant to widen, and this method has no caller.
      *
-     * Returns false if no org_settings row or the elevated list is null/empty.
+     * Granting it would not be a matter of calling this from the dispatch path:
+     * secrets are stored `(org_id, scope, key)` with no repository dimension, so
+     * "the source repository's secrets" is not a set the orchestrator can name
+     * today. The list, its admin route, its CLI mutators and its wire field are
+     * deprecated pending removal at v1.0.0 (`docs/user/deprecations.md`).
      */
     isElevatedAccessAllowed(workflowRoutingKey: string, repoIdentifier: string, customerId: string): Promise<boolean>;
     private getSettings;