@kici-dev/orchestrator 0.1.24 → 0.1.25

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/dist/app.d.ts +24 -0
  2. package/dist/audit/access-log.d.ts +4 -0
  3. package/dist/cancel/cancel-run.d.ts +2 -0
  4. package/dist/cli/api-client.d.ts +2 -0
  5. package/dist/cli/commands/attestations-list.d.ts +49 -0
  6. package/dist/cli/commands/attestations-retry.d.ts +28 -0
  7. package/dist/cli/commands/attestations.d.ts +2 -1
  8. package/dist/cli/commands/cluster.d.ts +3 -0
  9. package/dist/cli/local-github-ingress-url.d.ts +8 -0
  10. package/dist/cli.js +1021 -404
  11. package/dist/cluster/cluster-identity.d.ts +11 -0
  12. package/dist/cluster/reconcile-identity.d.ts +50 -0
  13. package/dist/cold-store/load-access-log-range.d.ts +4 -0
  14. package/dist/config.d.ts +2 -0
  15. package/dist/dashboard/attestation-filters.d.ts +62 -1
  16. package/dist/dashboard/handler.d.ts +24 -4
  17. package/dist/db/migrations/061_execution_runs_environment_id.d.ts +4 -0
  18. package/dist/db/migrations/062_execution_runs_agent_label.d.ts +4 -0
  19. package/dist/db/migrations/063_access_log_agent_label_index.d.ts +10 -0
  20. package/dist/db/migrations/064_execution_jobs_skipped_environments.d.ts +17 -0
  21. package/dist/db/migrations/065_pending_attestations.d.ts +22 -0
  22. package/dist/db/migrations/066_pending_attestations_rejected.d.ts +13 -0
  23. package/dist/db/types.d.ts +58 -0
  24. package/dist/environments/protection/satisfiability.d.ts +14 -9
  25. package/dist/events/circuit-breaker.d.ts +8 -5
  26. package/dist/events/event-router.d.ts +12 -4
  27. package/dist/events/event-store.d.ts +21 -3
  28. package/dist/events/trust-store.d.ts +6 -1
  29. package/dist/events/types.d.ts +6 -1
  30. package/dist/helpers/rate-limiter.d.ts +17 -2
  31. package/dist/index.js +2 -1
  32. package/dist/metrics/prometheus.d.ts +6 -0
  33. package/dist/pipeline/dispatch-matched-workflow.d.ts +6 -0
  34. package/dist/pipeline/manual-schedule.d.ts +1 -1
  35. package/dist/pipeline/process-webhook.d.ts +14 -1
  36. package/dist/pipeline/rerun.d.ts +1 -1
  37. package/dist/pipeline/test-pipeline.d.ts +6 -0
  38. package/dist/provenance/attestation-retrier.d.ts +92 -0
  39. package/dist/provenance/backfill-run.d.ts +41 -0
  40. package/dist/provenance/pending-attestations-repo.d.ts +64 -0
  41. package/dist/provenance/verify-at-ingest.d.ts +10 -3
  42. package/dist/reporting/execution-tracker.d.ts +15 -3
  43. package/dist/reporting/run-aggregator.d.ts +5 -1
  44. package/dist/routes/admin-access-log.d.ts +2 -1
  45. package/dist/routes/admin.d.ts +14 -0
  46. package/dist/routes/github-webhook.d.ts +37 -0
  47. package/dist/routes/health.d.ts +9 -0
  48. package/dist/routes/webhooks.d.ts +2 -4
  49. package/dist/server.js +3462 -1886
  50. package/dist/sources/source-store.d.ts +6 -0
  51. package/dist/stale-detector/workflow-deadline-detector.d.ts +25 -0
  52. package/dist/standalone.js +19335 -18258
  53. package/dist/webhook/dedup.d.ts +14 -0
  54. package/dist/ws/agent-handler.d.ts +23 -0
  55. package/dist/ws/oidc-token-relay.d.ts +31 -21
  56. package/dist/ws/orch-rpc.d.ts +9 -0
  57. package/dist/ws/platform-client.d.ts +18 -1
  58. package/dist/ws/test-relay-handlers.d.ts +2 -0
  59. package/package.json +5 -5
  60. package/sbom.spdx.json +55 -50
@@ -29,6 +29,17 @@ export interface ClusterIdentityDeps {
29
29
  */
30
30
  skipSentinelValidation?: boolean;
31
31
  }
32
+ /**
33
+ * Default cache-storage S3 prefix (mirrors the orchestrator config
34
+ * `cacheStorageS3Prefix` default). Empty means the cache blobs and the
35
+ * cluster-identity sentinel live at the bucket root — the bucket already scopes
36
+ * the cluster. Every place that resolves a sentinel prefix (the orchestrator
37
+ * runtime, the `kici-admin cluster reconcile-identity` CLI, and the staging
38
+ * deploy's self-heal step) MUST fall back to THIS value when no explicit prefix
39
+ * is set, otherwise they compute divergent sentinel keys and crash-loop the
40
+ * orchestrator boot on a spurious "Cluster identity mismatch".
41
+ */
42
+ export declare const DEFAULT_CACHE_STORAGE_S3_PREFIX = "";
32
43
  /**
33
44
  * Build the S3 key for the cluster identity sentinel under an optional prefix.
34
45
  * Strips any trailing slash from the prefix so callers can pass either form.
@@ -0,0 +1,50 @@
1
+ import type { IdempotentStep } from '@kici-dev/shared/idempotency';
2
+ export type ReconcileDirection = 'db-from-sentinel' | 'sentinel-from-db';
3
+ export interface ReconcileS3Config {
4
+ bucket: string;
5
+ /**
6
+ * Optional storage prefix (mirrors `KICI_STORAGE_PREFIX`). The sentinel lives
7
+ * at `<prefix>/.kici-cluster-id`; must match the prefix the orchestrator boots
8
+ * with. When omitted, `clusterSentinelKey` resolves the bucket root — the same
9
+ * as the orchestrator's `DEFAULT_CACHE_STORAGE_S3_PREFIX` (empty) default.
10
+ */
11
+ prefix?: string;
12
+ region?: string;
13
+ endpoint?: string;
14
+ forcePathStyle?: boolean;
15
+ accessKeyId: string;
16
+ secretAccessKey: string;
17
+ }
18
+ export interface IdentityDrift {
19
+ dbClusterId: string | null;
20
+ sentinelClusterId: string | null;
21
+ direction: ReconcileDirection;
22
+ }
23
+ declare function readClusterIdFromDb(databaseUrl: string): Promise<string | null>;
24
+ declare function writeClusterIdToDb(databaseUrl: string, clusterId: string): Promise<void>;
25
+ declare function readSentinel(s3: ReconcileS3Config): Promise<string | null>;
26
+ declare function writeSentinel(s3: ReconcileS3Config, clusterId: string): Promise<void>;
27
+ /**
28
+ * Indirection object for the DB + S3 reads/writes, so the unit test can stub
29
+ * the I/O without an ESM partial-mock dance. `buildReconcileStep` calls every
30
+ * reader/writer through this object; the test overrides its members directly.
31
+ */
32
+ export declare const reconcileIo: {
33
+ readClusterIdFromDb: typeof readClusterIdFromDb;
34
+ writeClusterIdToDb: typeof writeClusterIdToDb;
35
+ readSentinel: typeof readSentinel;
36
+ writeSentinel: typeof writeSentinel;
37
+ };
38
+ /**
39
+ * Build the idempotent step that reconciles the cluster identity in the given
40
+ * direction. `check()` returns drift when the two sides disagree (or null when
41
+ * in sync); `apply()` performs the single write that brings them into
42
+ * agreement. Run it through `runIdempotentStep` (`@kici-dev/core/idempotency`).
43
+ */
44
+ export declare function buildReconcileStep(deps: {
45
+ databaseUrl: string;
46
+ s3: ReconcileS3Config;
47
+ direction: ReconcileDirection;
48
+ }): IdempotentStep<IdentityDrift>;
49
+ export {};
50
+ //# sourceMappingURL=reconcile-identity.d.ts.map
@@ -52,6 +52,10 @@ export interface LoadAccessLogRangeArgs {
52
52
  * `009_access_log_trigram.ts`. Min ~3 chars for the index to help.
53
53
  */
54
54
  q?: string;
55
+ /** Exact-match filter on the agent provenance label (`agent_label` column). */
56
+ agentLabel?: string;
57
+ /** When true, return only agent-attributed rows (`agent_label IS NOT NULL`). */
58
+ agentOnly?: boolean;
55
59
  };
56
60
  limit: number;
57
61
  cursor?: string;
package/dist/config.d.ts CHANGED
@@ -100,6 +100,7 @@ declare const configSchema: z.ZodObject<{
100
100
  eventRouterRetryScanIntervalMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
101
101
  testMode: z.ZodPipe<z.ZodDefault<z.ZodString>, z.ZodTransform<boolean, string>>;
102
102
  testEventFailFirstN: z.ZodOptional<z.ZodString>;
103
+ testMintDeferAudience: z.ZodOptional<z.ZodString>;
103
104
  eventLogMaxPayloadBytes: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
104
105
  logLevel: z.ZodDefault<z.ZodEnum<{
105
106
  error: "error";
@@ -274,6 +275,7 @@ export declare const envDef: import("@kici-dev/shared/env").DefineEnvResult<{
274
275
  bootstrapAdminToken?: string | undefined;
275
276
  orchestratorHostAgentId?: string | undefined;
276
277
  testEventFailFirstN?: string | undefined;
278
+ testMintDeferAudience?: string | undefined;
277
279
  otelExporterOtlpEndpoint?: string | undefined;
278
280
  clusterName?: string | undefined;
279
281
  }>;
@@ -12,6 +12,7 @@ import type { Database } from '../db/types.js';
12
12
  * lists (repository / workflow come back null).
13
13
  */
14
14
  export declare function baseAttestationsQuery(db: Kysely<Database>): import("kysely").SelectQueryBuilder<{
15
+ cluster_meta: import("../db/types.js").ClusterMetaTable;
15
16
  environments: import("../db/types.js").EnvironmentsTable;
16
17
  host_roster: import("../db/types.js").HostRosterTable;
17
18
  dispatch_queue: import("../db/types.js").DispatchQueueTable;
@@ -42,7 +43,6 @@ export declare function baseAttestationsQuery(db: Kysely<Database>): import("kys
42
43
  run_secret_outputs: import("../db/types.js").RunSecretOutputsTable;
43
44
  concurrency_groups: import("../db/types.js").ConcurrencyGroupsTable;
44
45
  sources: import("../db/types.js").SourcesTable;
45
- cluster_meta: import("../db/types.js").ClusterMetaTable;
46
46
  join_tokens: import("../db/types.js").JoinTokenTable;
47
47
  org_settings: import("../db/types.js").OrgSettingsTable;
48
48
  execution_job_needs: import("../db/types.js").ExecutionJobNeedsTable;
@@ -57,9 +57,70 @@ export declare function baseAttestationsQuery(db: Kysely<Database>): import("kys
57
57
  scaler_agent_jobs: import("../db/types.js").ScalerAgentJobsTable;
58
58
  scaler_reservations: import("../db/types.js").ScalerReservationsTable;
59
59
  attestations: import("../db/types.js").AttestationsTable;
60
+ pending_attestations: import("../db/types.js").PendingAttestationsTable;
60
61
  remote_sources: import("../db/types.js").RemoteSourcesTable;
61
62
  }, "execution_runs" | "execution_jobs" | "attestations", {}>;
62
63
  export type AttestationsBaseQuery = ReturnType<typeof baseAttestationsQuery>;
64
+ /**
65
+ * Base query for the deferred-attestation outbox (`pending_attestations`),
66
+ * joined to `execution_jobs` (job name) and `execution_runs` (repo / workflow
67
+ * context) for the org-wide list's pending summaries.
68
+ *
69
+ * Same uuid/text mismatch as `baseAttestationsQuery`: `pending_attestations`
70
+ * `run_id` / `job_id` are TEXT while the `execution_*` keys are `uuid`, and
71
+ * Postgres won't compare `uuid = text` implicitly — so the joins cast the uuid
72
+ * side to text. Both joins are LEFT joins so a pending row with no matching
73
+ * run / job still lists (repository / workflow / job name come back null).
74
+ */
75
+ export declare function basePendingAttestationsQuery(db: Kysely<Database>): import("kysely").SelectQueryBuilder<{
76
+ cluster_meta: import("../db/types.js").ClusterMetaTable;
77
+ environments: import("../db/types.js").EnvironmentsTable;
78
+ host_roster: import("../db/types.js").HostRosterTable;
79
+ dispatch_queue: import("../db/types.js").DispatchQueueTable;
80
+ dedup_cache: import("../db/types.js").DedupCacheTable;
81
+ ip_allocations: import("../db/types.js").IpAllocationTable;
82
+ execution_runs: import("kysely").Nullable<import("../db/types.js").ExecutionRunTable>;
83
+ execution_jobs: import("kysely").Nullable<import("../db/types.js").ExecutionJobTable>;
84
+ execution_steps: import("../db/types.js").ExecutionStepTable;
85
+ raft_state: import("../db/types.js").RaftStateTable;
86
+ secret_audit_log: import("../db/types.js").SecretAuditLogTable;
87
+ scoped_secrets: import("../db/types.js").ScopedSecretsTable;
88
+ environment_bindings: import("../db/types.js").EnvironmentBindingsTable;
89
+ environment_variables: import("../db/types.js").EnvironmentVariablesTable;
90
+ environment_source_overrides: import("../db/types.js").EnvironmentSourceOverridesTable;
91
+ held_runs: import("../db/types.js").HeldRunsTable;
92
+ held_run_approvals: import("../db/types.js").HeldRunApprovalsTable;
93
+ admin_tokens: import("../db/types.js").AdminTokenTable;
94
+ agent_tokens: import("../db/types.js").AgentTokenTable;
95
+ config_versions: import("../db/types.js").ConfigVersionTable;
96
+ kici_events: import("../db/types.js").KiciEventTable;
97
+ generic_webhook_sources: import("../db/types.js").GenericWebhookSourceTable;
98
+ cross_repo_trust: import("../db/types.js").CrossRepoTrustTable;
99
+ test_uploads: import("../db/types.js").TestUploadsTable;
100
+ workflow_registrations: import("../db/types.js").WorkflowRegistrationsTable;
101
+ registry_versions: import("../db/types.js").RegistryVersionsTable;
102
+ cron_last_fired: import("../db/types.js").CronLastFiredTable;
103
+ run_ephemeral_keys: import("../db/types.js").RunEphemeralKeysTable;
104
+ run_secret_outputs: import("../db/types.js").RunSecretOutputsTable;
105
+ concurrency_groups: import("../db/types.js").ConcurrencyGroupsTable;
106
+ sources: import("../db/types.js").SourcesTable;
107
+ join_tokens: import("../db/types.js").JoinTokenTable;
108
+ org_settings: import("../db/types.js").OrgSettingsTable;
109
+ execution_job_needs: import("../db/types.js").ExecutionJobNeedsTable;
110
+ pending_job_contexts: import("../db/types.js").PendingJobContextsTable;
111
+ pending_workflow_contexts: import("../db/types.js").PendingWorkflowContextsTable;
112
+ event_log: import("../db/types.js").EventLogTable;
113
+ access_log: import("../db/types.js").AccessLogTable;
114
+ cold_store_chunk_counts: import("../db/types.js").ColdStoreChunkCountsTable;
115
+ cold_store_chunks: import("../db/types.js").ColdStoreChunksTable;
116
+ check_run_tracking: import("../db/types.js").CheckRunTrackingTable;
117
+ scaler_spawning_agents: import("../db/types.js").ScalerSpawningAgentsTable;
118
+ scaler_agent_jobs: import("../db/types.js").ScalerAgentJobsTable;
119
+ scaler_reservations: import("../db/types.js").ScalerReservationsTable;
120
+ attestations: import("../db/types.js").AttestationsTable;
121
+ pending_attestations: import("../db/types.js").PendingAttestationsTable;
122
+ remote_sources: import("../db/types.js").RemoteSourcesTable;
123
+ }, "execution_runs" | "execution_jobs" | "pending_attestations", {}>;
63
124
  /**
64
125
  * Apply org-wide attestation filters to the base query. Digest is exact-match;
65
126
  * name is an ILIKE substring; status / repository / workflow / job are equality;
@@ -13,7 +13,7 @@
13
13
  */
14
14
  import { type Kysely } from 'kysely';
15
15
  import { type ColdStore } from '@kici-dev/shared';
16
- import type { DashboardRunDetailRequest, DashboardRunsListRequest, DashboardRunsListResponse, DashboardRunsFiltersRequest, DashboardRunsFiltersResponse, DashboardSourcesListRequest, DashboardSourcesListResponse, DashboardStepLogsRequest, DashboardAttestationsListRequest, DashboardAttestationsListAllRequest, DashboardAttestationGetRequest, DashboardPayloadRequest, DashboardOrchLogsRequest, DashboardEventLogListRequest, DashboardEventLogDetailRequest, DashboardEventLogPayloadStreamRequest, DashboardAccessLogListRequest, DashboardEventDlqListRequest, DashboardEventDlqCountRequest, DashboardEventDlqRetryRequest, DashboardEventDlqDiscardRequest, RunRerunRequest, RunCancelRequest, ManualScheduleRequest, DashboardRunStructuredRequest } from '@kici-dev/engine';
16
+ import type { DashboardRunDetailRequest, DashboardRunsListRequest, DashboardRunsListResponse, DashboardRunsFiltersRequest, DashboardRunsFiltersResponse, DashboardSourcesListRequest, DashboardSourcesListResponse, DashboardStepLogsRequest, DashboardAttestationsListRequest, DashboardAttestationsListAllRequest, DashboardAttestationGetRequest, DashboardAttestationRetryRequest, DashboardPayloadRequest, DashboardOrchLogsRequest, DashboardEventLogListRequest, DashboardEventLogDetailRequest, DashboardEventLogPayloadStreamRequest, DashboardAccessLogListRequest, DashboardEventDlqListRequest, DashboardEventDlqCountRequest, DashboardEventDlqRetryRequest, DashboardEventDlqDiscardRequest, RunRerunRequest, RunCancelRequest, ManualScheduleRequest, DashboardRunStructuredRequest } from '@kici-dev/engine';
17
17
  import type { Database } from '../db/types.js';
18
18
  import type { LogStorage } from '../reporting/log-storage.js';
19
19
  import type { CacheStorage } from '../storage/types.js';
@@ -31,6 +31,17 @@ interface DashboardHandlerDeps {
31
31
  * (the same one P1.5 writes bundles to).
32
32
  */
33
33
  provenanceStorage?: CacheStorage | null;
34
+ /**
35
+ * Drain the deferred-attestation outbox on demand (mints in this process,
36
+ * which owns the Platform WS). Backs `dashboard.attestation.retry`. Optional —
37
+ * absent on orchestrators without the retrier wired.
38
+ */
39
+ retryAttestations?: (opts: {
40
+ runId?: string;
41
+ }) => Promise<{
42
+ minted: number;
43
+ stillPending: number;
44
+ }>;
34
45
  /** Send a response message back to Platform over the WS connection. */
35
46
  send: (msg: unknown) => void;
36
47
  /** This orchestrator's instance ID, included in job detail responses. */
@@ -59,7 +70,7 @@ interface DashboardHandlerDeps {
59
70
  * compat with mixed deploys; absent means the orchestrator falls
60
71
  * back to the legacy "no PG row → throw" path.
61
72
  */
62
- onRerun: (runId: string, triggeredBy: string | null, routingKey?: string) => Promise<{
73
+ onRerun: (runId: string, triggeredBy: string | null, triggeredByAgentLabel: string | null, routingKey?: string) => Promise<{
63
74
  newRunId: string;
64
75
  }>;
65
76
  /**
@@ -67,14 +78,14 @@ interface DashboardHandlerDeps {
67
78
  * Returns { cancelledJobs } on success or throws on failure.
68
79
  * The force flag indicates whether to force-cancel (SIGKILL, skip hooks).
69
80
  */
70
- onCancel: (runId: string, cancelledBy: string | null, force?: boolean) => Promise<{
81
+ onCancel: (runId: string, cancelledBy: string | null, cancelledByAgentLabel: string | null, force?: boolean) => Promise<{
71
82
  cancelledJobs: number;
72
83
  }>;
73
84
  /**
74
85
  * Callback for handling manual schedule trigger requests.
75
86
  * Returns { newRunId } on success or throws on failure.
76
87
  */
77
- onManualSchedule: (registrationId: string, triggeredBy: string | null) => Promise<{
88
+ onManualSchedule: (registrationId: string, triggeredBy: string | null, triggeredByAgentLabel: string | null) => Promise<{
78
89
  newRunId: string;
79
90
  }>;
80
91
  /**
@@ -99,6 +110,7 @@ export declare class DashboardHandler {
99
110
  private readonly onManualSchedule;
100
111
  private readonly coldStore;
101
112
  private readonly eventStore;
113
+ private readonly retryAttestations;
102
114
  constructor(deps: DashboardHandlerDeps);
103
115
  /**
104
116
  * Update the bound orgId + routingKey. Called from server.ts after resolving
@@ -295,12 +307,20 @@ export declare class DashboardHandler {
295
307
  * the shared filter builder.
296
308
  */
297
309
  private resolveAttestationsListAll;
310
+ /** Map the deferred-attestation outbox into pending list summaries (page 1). */
311
+ private resolvePendingAttestationSummaries;
298
312
  /**
299
313
  * Handle a dashboard.attestations.list.all request: org-wide, paginated,
300
314
  * filtered list of attestation summaries (metadata only). Access-logged
301
315
  * against the `attestation` target.
302
316
  */
303
317
  handleAttestationsListAll(msg: DashboardAttestationsListAllRequest): Promise<void>;
318
+ /**
319
+ * Handle a dashboard.attestation.retry request: drain the deferred-attestation
320
+ * outbox (optionally scoped to one run) and reply with the mint counts. The
321
+ * mint happens in this orchestrator process, which owns the Platform WS.
322
+ */
323
+ handleAttestationRetry(msg: DashboardAttestationRetryRequest): Promise<void>;
304
324
  /**
305
325
  * Handle a dashboard.attestation.get request: a single attestation by id with
306
326
  * its bundle inlined for the detail page. Resolves the run-owning org for the
@@ -0,0 +1,4 @@
1
+ import { type Kysely } from 'kysely';
2
+ export declare function up(db: Kysely<unknown>): Promise<void>;
3
+ export declare function down(db: Kysely<unknown>): Promise<void>;
4
+ //# sourceMappingURL=061_execution_runs_environment_id.d.ts.map
@@ -0,0 +1,4 @@
1
+ import { type Kysely } from 'kysely';
2
+ export declare function up(db: Kysely<unknown>): Promise<void>;
3
+ export declare function down(db: Kysely<unknown>): Promise<void>;
4
+ //# sourceMappingURL=062_execution_runs_agent_label.d.ts.map
@@ -0,0 +1,10 @@
1
+ import { type Kysely } from 'kysely';
2
+ /**
3
+ * Index `access_log.agent_label` (column added by migration 058) so the
4
+ * operator-facing agent filter (`kici-admin access-log list --agent-label`,
5
+ * the dashboard "agent name" filter) does an indexed exact-match lookup
6
+ * instead of a scan. Idempotent.
7
+ */
8
+ export declare function up(db: Kysely<unknown>): Promise<void>;
9
+ export declare function down(db: Kysely<unknown>): Promise<void>;
10
+ //# sourceMappingURL=063_access_log_agent_label_index.d.ts.map
@@ -0,0 +1,17 @@
1
+ import { type Kysely } from 'kysely';
2
+ /**
3
+ * Add the test-run skipped-environment columns to `execution_jobs`:
4
+ *
5
+ * - `execution_jobs.skipped_environments text NULL` — a JSON-encoded `string[]`
6
+ * of the bound environment names dropped on a test/local run because they
7
+ * disallow local execution (`allowLocalExecution=false`) or are unconfigured.
8
+ * NULL = nothing skipped.
9
+ * - `execution_jobs.env_warning text NULL` — the user-visible warning naming the
10
+ * skipped environments, surfaced on the dashboard run view. NULL = no warning.
11
+ *
12
+ * Idempotent (`ADD COLUMN IF NOT EXISTS`); additive, so staging data is
13
+ * preserved.
14
+ */
15
+ export declare function up(db: Kysely<unknown>): Promise<void>;
16
+ export declare function down(db: Kysely<unknown>): Promise<void>;
17
+ //# sourceMappingURL=064_execution_jobs_skipped_environments.d.ts.map
@@ -0,0 +1,22 @@
1
+ import { type Kysely } from 'kysely';
2
+ /**
3
+ * Add the `pending_attestations` deferred-attestation outbox and an idempotency
4
+ * unique index on `attestations`.
5
+ *
6
+ * When a build's provenance mint fails transiently, the agent freezes and
7
+ * DSSE-signs the statement at build time; the orchestrator records the frozen
8
+ * envelope here instead of failing the job. A Raft-leader-only retrier later
9
+ * mints the deferred token, attaches it, uploads the bundle, and records one
10
+ * `attestations` row — deleting the pending row. The unique index on
11
+ * `attestations (run_id, job_id, subject_digest)` makes that fulfilment
12
+ * idempotent across a cluster (ON CONFLICT DO NOTHING).
13
+ *
14
+ * `origin_kind` holds the non-`live` `AttestationOrigin` values
15
+ * (`deferred` / `offline-backfill`); the `created_at` default anchors the true
16
+ * build time so temporal honesty survives a later mint.
17
+ *
18
+ * Idempotent: guarded on table existence.
19
+ */
20
+ export declare function up(db: Kysely<unknown>): Promise<void>;
21
+ export declare function down(db: Kysely<unknown>): Promise<void>;
22
+ //# sourceMappingURL=065_pending_attestations.d.ts.map
@@ -0,0 +1,13 @@
1
+ import { type Kysely } from 'kysely';
2
+ /**
3
+ * Add `rejected_at` to `pending_attestations`: the terminal-rejection marker for
4
+ * a deferred attestation the Platform definitively cannot mint (run/job absent).
5
+ * NULL while the row is still pending a later mint; set once, the row is skipped
6
+ * by the retrier and drops out of the pending-attestations gauge. Re-armed via
7
+ * `kici-admin attestations retry --include-rejected`.
8
+ *
9
+ * Idempotent: guarded on column existence.
10
+ */
11
+ export declare function up(db: Kysely<unknown>): Promise<void>;
12
+ export declare function down(db: Kysely<unknown>): Promise<void>;
13
+ //# sourceMappingURL=066_pending_attestations_rejected.d.ts.map
@@ -49,6 +49,7 @@ export interface Database {
49
49
  scaler_agent_jobs: ScalerAgentJobsTable;
50
50
  scaler_reservations: ScalerReservationsTable;
51
51
  attestations: AttestationsTable;
52
+ pending_attestations: PendingAttestationsTable;
52
53
  remote_sources: RemoteSourcesTable;
53
54
  host_roster: HostRosterTable;
54
55
  }
@@ -281,10 +282,16 @@ export interface ExecutionRunTable {
281
282
  original_run_id: string | null;
282
283
  /** User identity that triggered this re-run (null for webhook-triggered). Format: "user:email" or "key:name". */
283
284
  triggered_by: string | null;
285
+ /** Agent provenance label when the run was triggered through an agent credential (null otherwise). */
286
+ triggered_by_agent_label: string | null;
284
287
  /** User identity that cancelled this run (null for non-cancelled). */
285
288
  cancelled_by: string | null;
289
+ /** Agent provenance label when the run was cancelled through an agent credential (null otherwise). */
290
+ cancelled_by_agent_label: string | null;
286
291
  /** Environment name for this run (null if no environment applies) */
287
292
  environment: string | null;
293
+ /** Matched environment id for this run (null if no/unresolved environment). */
294
+ environment_id: string | null;
288
295
  /** Trust tier of the contributor for PR runs (null for non-PR events) */
289
296
  trust_tier: string | null;
290
297
  /** Lock file source: 'head' or 'base' (null for non-PR events) */
@@ -395,6 +402,13 @@ export interface ExecutionJobTable {
395
402
  * overwritten with the agent-resolved list for dynamic environments.
396
403
  */
397
404
  environments: string | null;
405
+ /**
406
+ * Bound environments skipped on a test/local run (non-test or unconfigured),
407
+ * JSON-encoded `string[]`. NULL = nothing skipped.
408
+ */
409
+ skipped_environments: string | null;
410
+ /** User-visible warning naming the skipped test-run environments. NULL = none. */
411
+ env_warning: string | null;
398
412
  /** Whether all upstream needs edges are satisfied (dispatch gate). */
399
413
  needs_satisfied: Generated<boolean>;
400
414
  /** Timestamp when needs_satisfied first flipped to true. */
@@ -1643,6 +1657,50 @@ export interface AttestationsTable {
1643
1657
  }
1644
1658
  export type AttestationRow = Selectable<AttestationsTable>;
1645
1659
  export type NewAttestationRow = Insertable<AttestationsTable>;
1660
+ /**
1661
+ * Deferred-attestation outbox: a build whose provenance mint failed transiently
1662
+ * freezes its DSSE-signed statement here; a leader-only retrier mints the token
1663
+ * later and records the fulfilled `attestations` row.
1664
+ */
1665
+ export interface PendingAttestationsTable {
1666
+ /** Random id (primary key). */
1667
+ id: string;
1668
+ /** KiCI run this deferred attestation belongs to. */
1669
+ run_id: string;
1670
+ /** KiCI job this deferred attestation was produced by. */
1671
+ job_id: string;
1672
+ /** Caller-supplied artifact name. */
1673
+ subject_name: string;
1674
+ /** Primary subject digest (lowercase hex). */
1675
+ subject_digest: string;
1676
+ /** Requested token audience. */
1677
+ audience: string;
1678
+ /** The frozen, agent-signed DSSE envelope (JSONB). */
1679
+ dsse_envelope: unknown;
1680
+ /** The ephemeral public-key JWK (JSONB). */
1681
+ public_key: unknown;
1682
+ /** Bundle media type. */
1683
+ media_type: string;
1684
+ /** SHA-256 of the frozen statement payload — the later-mint binding. */
1685
+ statement_hash: string;
1686
+ /** Non-`live` AttestationOrigin: `deferred` | `offline-backfill`. */
1687
+ origin_kind: string;
1688
+ /** Retry attempts so far. */
1689
+ attempt_count: Generated<number>;
1690
+ /** First-capture time = the true build-time anchor. */
1691
+ created_at: Generated<Date>;
1692
+ /** When fulfilment was last attempted. */
1693
+ last_attempt_at: Date | null;
1694
+ /** Last fulfilment error, or NULL. */
1695
+ last_error: string | null;
1696
+ /**
1697
+ * Terminal-rejection time: the Platform definitively rejected the mint
1698
+ * (run/job absent). NULL while the row is still pending a later mint.
1699
+ */
1700
+ rejected_at: Date | null;
1701
+ }
1702
+ export type PendingAttestationRow = Selectable<PendingAttestationsTable>;
1703
+ export type NewPendingAttestationRow = Insertable<PendingAttestationsTable>;
1646
1704
  /**
1647
1705
  * Remote-source table (remote_sources).
1648
1706
  *
@@ -11,9 +11,11 @@
11
11
  *
12
12
  * This module intersects the statically-decidable set rules (branch, trigger
13
13
  * type, repository — only when every pattern is a literal, never a glob) plus
14
- * existence and enabled across the bound environments, and reports the first
15
- * provably-empty intersection. Any glob in a rule makes that rule undecidable,
16
- * so it is skipped here and left to the dispatch-time catch-all
14
+ * the `enabled` gate across the **resolved** bound environments, and reports the
15
+ * first provably-empty intersection. Bound names that resolve to no environment
16
+ * record are lenient skipped, never rejected matching the dispatch-time
17
+ * behavior in `dispatch-matched-workflow.ts`. Any glob in a rule makes that rule
18
+ * undecidable, so it is skipped here and left to the dispatch-time catch-all
17
19
  * (`evaluateMultiEnvGates`).
18
20
  */
19
21
  import { z } from 'zod';
@@ -23,7 +25,6 @@ export declare const UnsatisfiableRule: z.ZodEnum<{
23
25
  enabled: "enabled";
24
26
  repo: "repo";
25
27
  branch: "branch";
26
- existence: "existence";
27
28
  trigger: "trigger";
28
29
  }>;
29
30
  export type UnsatisfiableRule = z.infer<typeof UnsatisfiableRule>;
@@ -36,9 +37,12 @@ export interface UnsatisfiableBinding {
36
37
  }
37
38
  /**
38
39
  * Returns a precise problem when the bound environments can NEVER be jointly
39
- * satisfied (a provably-empty intersection on a decidable rule, a missing
40
- * environment, or a disabled one), else `null`. Glob / undecidable cases return
41
- * `null` and are caught at dispatch by `evaluateMultiEnvGates`.
40
+ * satisfied (a disabled environment, or mutually-exclusive fixed restrictions
41
+ * among the resolved environments), else `null`. Missing (unresolved) names are
42
+ * skipped a bound name with no environment record contributes no protection
43
+ * rules and is lenient at dispatch, so it is not rejected here. Glob /
44
+ * undecidable cases also return `null` and are caught at dispatch by
45
+ * `evaluateMultiEnvGates`.
42
46
  *
43
47
  * `envs[i]` is the resolved `Environment` for `envNames[i]` (undefined when the
44
48
  * name has no environment record). Only the statically-known (non-dynamic) bound
@@ -52,8 +56,9 @@ interface SatisfiabilityLockWorkflow {
52
56
  }
53
57
  /**
54
58
  * Walk every workflow's static jobs and reject the registration when a bound
55
- * environment list is provably unsatisfiable (missing/disabled environment, or
56
- * mutually-exclusive fixed branch/trigger/repo restrictions). Dynamic elements
59
+ * environment list is provably unsatisfiable (a disabled environment, or
60
+ * mutually-exclusive fixed branch/trigger/repo restrictions among the resolved
61
+ * environments — missing names are lenient, never rejected). Dynamic elements
57
62
  * are skipped (unresolvable at registration); the all-must-pass semantics keep
58
63
  * the static subset's exclusivity sound. Throws the first
59
64
  * `UnsatisfiableBinding.message` so the registration route / direct helper
@@ -1,10 +1,11 @@
1
1
  import type { EventRouterConfig } from './types.js';
2
2
  /**
3
- * Circuit breaker for event loop detection and per-workflow rate limiting.
3
+ * Circuit breaker for event loop detection and user-event rate limiting.
4
4
  *
5
5
  * Two-layer protection:
6
6
  * 1. Chain depth: Rejects events exceeding configurable max chain depth.
7
- * 2. Rate limiting: Sliding-window per-workflow rate limiter.
7
+ * 2. Rate limiting: Sliding-window limiter keyed per (source routing key +
8
+ * event name). System events (`__`-prefixed) are exempt upstream.
8
9
  */
9
10
  export declare class EventCircuitBreaker {
10
11
  private readonly config;
@@ -18,10 +19,12 @@ export declare class EventCircuitBreaker {
18
19
  reason?: string;
19
20
  };
20
21
  /**
21
- * Check if a workflow is within its per-minute rate limit.
22
- * Uses a sliding window of 60 seconds.
22
+ * Check whether a rate-limit key is within its per-minute allowance, using a
23
+ * sliding 60-second window. Callers key user events by
24
+ * `<sourceRoutingKey>:<eventName>`; system events (`__`-prefixed) are exempt
25
+ * upstream and never reach here.
23
26
  */
24
- checkRateLimit(workflowKey: string): {
27
+ checkRateLimit(rateKey: string): {
25
28
  allowed: boolean;
26
29
  retryAfterMs?: number;
27
30
  };
@@ -159,11 +159,19 @@ export declare class EventRouter {
159
159
  */
160
160
  private buildSimulatedEvent;
161
161
  /**
162
- * Catch-up: process unprocessed events missed during downtime.
162
+ * Catch-up: process every unprocessed event missed during downtime.
163
163
  *
164
- * Uses lease-based dispatch identical to the live path, so a catch-up
165
- * dispatch failure schedules a retry via the leader-only scanner instead
166
- * of being silently dropped.
164
+ * Pages through the backlog with a keyset cursor: fetch a batch, dispatch
165
+ * each event (lease-based, identical failure semantics to the live path),
166
+ * advance the cursor to the last event of the batch, and repeat until a
167
+ * batch returns fewer than EVENT_CATCHUP_BATCH_SIZE rows. Without this loop
168
+ * only the oldest page (100 events) would be dispatched and the remainder
169
+ * would sit unprocessed — invisible to both the live NOTIFY path and the
170
+ * retry scanner — until the TTL cleanup deleted them undelivered.
171
+ *
172
+ * The store's cursor is strictly monotone over (created_at, id), so a
173
+ * still-unprocessed retrying event at or before the cursor is never
174
+ * re-fetched — that strict advance is what guarantees this loop terminates.
167
175
  */
168
176
  private catchUp;
169
177
  }
@@ -3,6 +3,14 @@ import type { Database } from '../db/types.js';
3
3
  import type { DlqReason, EventRouterConfig, StoredEvent } from './types.js';
4
4
  /** Either a Kysely DB handle or an active transaction. */
5
5
  export type DbExecutor = Kysely<Database> | Transaction<Database>;
6
+ /**
7
+ * Default page size for unprocessed-event catch-up scans.
8
+ *
9
+ * Shared with EventRouter.catchUp so the paginating loop's "a short batch
10
+ * means we're done" termination check compares against the same value the
11
+ * store slices by.
12
+ */
13
+ export declare const EVENT_CATCHUP_BATCH_SIZE = 100;
6
14
  /**
7
15
  * Input shape for writing a new event. Excludes columns the DB fills in
8
16
  * itself (id, processed, created_at, attempts, claimed_*, last_error,
@@ -58,9 +66,19 @@ export declare class EventStore {
58
66
  */
59
67
  getById(id: string): Promise<StoredEvent | null>;
60
68
  /**
61
- * Get unprocessed events for catch-up on reconnect.
62
- * If sinceId is null, returns all unprocessed events.
63
- * Ordered by created_at ASC. Excludes DLQ rows.
69
+ * Get a page of unprocessed events for catch-up on start.
70
+ *
71
+ * Rows are ordered by a deterministic keyset `(created_at ASC, id ASC)` and
72
+ * exclude processed + DLQ rows. When `sinceId` is null, returns the first
73
+ * page from the oldest event. When `sinceId` is given, returns the page
74
+ * strictly after that event's `(created_at, id)` — a composite cursor so
75
+ * that events sharing the reference event's `created_at` are NOT skipped
76
+ * (a bare `created_at > ref` predicate drops same-timestamp siblings) and a
77
+ * still-unprocessed retrying event at or before the cursor is never
78
+ * re-fetched (which is what makes the caller's pagination loop terminate).
79
+ *
80
+ * The caller advances `sinceId` to the last event of each returned page and
81
+ * re-queries until a page returns fewer than `limit` rows.
64
82
  */
65
83
  getUnprocessedSince(sinceId: string | null, limit?: number): Promise<StoredEvent[]>;
66
84
  /**
@@ -17,7 +17,12 @@ interface TrustEntry {
17
17
  *
18
18
  * Same-repo events (same routing key) are always trusted without a DB lookup.
19
19
  * Cross-repo events require an explicit enabled row in the cross_repo_trust table.
20
- * Glob-based event filtering is supported via the allowed_events column.
20
+ * Glob-based event filtering is supported via the allowed_events column, whose
21
+ * value carries a three-way meaning:
22
+ * - `null` — filter unset: every event name is allowed (allow-all).
23
+ * - `'[]'` — an explicit empty allow-list: no event is allowed (deny-all).
24
+ * - `'["glob"]'` — only event names matching one of the globs are allowed.
25
+ * Malformed (unparseable / non-array) allowed_events fails closed (deny).
21
26
  */
22
27
  export declare class TrustStore {
23
28
  private readonly db;
@@ -7,7 +7,12 @@
7
7
  export interface EventRouterConfig {
8
8
  /** Maximum allowed chain depth before circuit breaker trips (default: 10) */
9
9
  maxChainDepth: number;
10
- /** Maximum event emissions per workflow per minute (default: 100) */
10
+ /**
11
+ * Max user-event emissions per (source routing key + event name) per minute
12
+ * (sliding 60s window). System events (`__`-prefixed, e.g. __workflow_complete)
13
+ * are EXEMPT -- they are orchestrator-emitted once per completion and cannot
14
+ * loop, so the storm guard does not apply. Default 100.
15
+ */
11
16
  rateLimitPerWorkflowPerMinute: number;
12
17
  /** TTL for persisted events in seconds (default: 604800 = 7 days) */
13
18
  eventTtlSeconds: number;