@kici-dev/orchestrator 0.12.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/dispatcher.d.ts +39 -0
- package/dist/agent/registry.d.ts +31 -0
- package/dist/app.d.ts +6 -0
- package/dist/cache/cache-upload-settle.d.ts +75 -0
- package/dist/cache/pending-inits.d.ts +9 -0
- package/dist/cancel/cancel-run.d.ts +22 -4
- package/dist/cancel/peer-job-cancel.d.ts +62 -0
- package/dist/cancel/sweep-stuck-cancelling.d.ts +3 -1
- package/dist/cli/api-client.d.ts +73 -0
- package/dist/cli/commands/cluster-settings.d.ts +31 -1
- package/dist/cli/commands/local-hook.d.ts +3 -2
- package/dist/cli/commands/local-trigger.d.ts +7 -0
- package/dist/cli/commands/peer.d.ts +27 -1
- package/dist/cli/commands/scaler.d.ts +40 -4
- package/dist/cli.js +2549 -972
- package/dist/cluster/cluster-settings-reader.d.ts +8 -2
- package/dist/cluster/coordinator-credential.d.ts +45 -0
- package/dist/cluster/coordinator.d.ts +52 -10
- package/dist/cluster/index.d.ts +1 -0
- package/dist/cluster/instance-heartbeat.d.ts +6 -1
- package/dist/cluster/join-token.d.ts +2 -0
- package/dist/cluster/peer-auth-coordinator.d.ts +13 -0
- package/dist/cluster/peer-client.d.ts +41 -4
- package/dist/cluster/peer-credentials.d.ts +31 -0
- package/dist/cluster/peer-forget.d.ts +138 -0
- package/dist/cluster/peer-handler.d.ts +18 -1
- package/dist/cluster/scaler-orphans-peer.d.ts +40 -0
- package/dist/cluster/settings-propagation.d.ts +85 -0
- package/dist/cluster/worker-cluster-settings.d.ts +29 -0
- package/dist/config.d.ts +12 -0
- package/dist/db/migrations/152_org_settings_cache_upload_settle_timeout.d.ts +15 -0
- package/dist/db/migrations/153_execution_runs_cancel_force.d.ts +16 -0
- package/dist/db/migrations/154_cluster_settings_firecracker_api_socket_wait.d.ts +4 -0
- package/dist/db/migrations/155_org_settings_reroute_spawn_retry.d.ts +17 -0
- package/dist/db/migrations/156_kici_events_match_outcome.d.ts +17 -0
- package/dist/db/types.d.ts +31 -0
- package/dist/events/event-router.d.ts +7 -1
- package/dist/events/event-store.d.ts +20 -3
- package/dist/events/types.d.ts +45 -0
- package/dist/helpers/windows-batch.d.ts +59 -0
- package/dist/index.js +252 -53
- package/dist/metrics/prometheus.d.ts +2 -0
- package/dist/orchestrator-core.d.ts +7 -0
- package/dist/pipeline/changed-files.d.ts +51 -0
- package/dist/pipeline/cross-source-tally.d.ts +21 -0
- package/dist/pipeline/degraded-reason.d.ts +8 -0
- package/dist/pipeline/dispatch-matched-workflow.d.ts +10 -0
- package/dist/pipeline/global-eval-round.d.ts +3 -2
- package/dist/pipeline/processor.d.ts +9 -1
- package/dist/pipeline/rerun-inputs.d.ts +16 -7
- package/dist/providers/github/changed-files.d.ts +14 -4
- package/dist/providers/universal-git/changed-files.d.ts +6 -0
- package/dist/providers/universal-git/config.d.ts +4 -0
- package/dist/registration/registration-store.d.ts +10 -0
- package/dist/routes/admin-cluster-settings.d.ts +13 -1
- package/dist/routes/admin-event-inspect.d.ts +36 -0
- package/dist/routes/admin-peer-forget.d.ts +43 -0
- package/dist/routes/admin-scaler-orphans.d.ts +64 -0
- package/dist/routes/admin.d.ts +23 -0
- package/dist/scaler/backend-factory.d.ts +6 -1
- package/dist/scaler/bare-metal-backend.d.ts +17 -15
- package/dist/scaler/bare-metal-launch.d.ts +90 -0
- package/dist/scaler/container-backend.d.ts +2 -0
- package/dist/scaler/container-runtime.d.ts +9 -0
- package/dist/scaler/firecracker-backend.d.ts +181 -7
- package/dist/scaler/live-vms.d.ts +57 -0
- package/dist/scaler/manager.d.ts +138 -5
- package/dist/scaler/orphan-requests.d.ts +23 -0
- package/dist/scaler/overlay-template.d.ts +38 -0
- package/dist/scaler/scaler-state-store.d.ts +7 -0
- package/dist/scaler/spawn-errors.d.ts +17 -0
- package/dist/scaler/startup-orphan-sweep.d.ts +15 -0
- package/dist/scaler/types.d.ts +21 -2
- package/dist/scaler/vm-job-bindings.d.ts +17 -0
- package/dist/secrets/rbac.d.ts +7 -1
- package/dist/server.js +7086 -2557
- package/dist/stale-detector/workflow-deadline-detector.d.ts +7 -1
- package/dist/standalone.js +16473 -11945
- package/dist/storage/s3.d.ts +36 -0
- package/dist/worker/in-memory-execution-tracker.d.ts +5 -0
- package/dist/worker/in-memory-job-queue.d.ts +21 -1
- package/dist/worker/queued-reroute-release.d.ts +39 -0
- package/dist/worker/reroute-spawn-control.d.ts +58 -0
- package/dist/worker/spawn-retry-budget.d.ts +87 -0
- package/dist/worker-core.d.ts +11 -2
- package/dist/ws/agent-handler.d.ts +24 -3
- package/dist/ws/agent-heartbeat.d.ts +2 -2
- package/installer-image-digests.json +3 -3
- package/package.json +5 -5
- package/sbom.spdx.json +53 -53
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Which cluster-settings version each orchestrator in a cluster has applied,
|
|
3
|
+
* compared with the version of the shared `cluster_settings` row.
|
|
4
|
+
*
|
|
5
|
+
* A worker reports the version of the snapshot it pulled, which is the one its
|
|
6
|
+
* next spawn reads. A coordinator reports the version its settings reader
|
|
7
|
+
* currently serves. Pure, because `kici-admin cluster-settings show` imports
|
|
8
|
+
* it to validate and render the report the admin route builds.
|
|
9
|
+
*/
|
|
10
|
+
import { z } from 'zod';
|
|
11
|
+
import type { PeerInfo } from './peer-registry.js';
|
|
12
|
+
export declare const settingsPropagationStatusSchema: z.ZodEnum<{
|
|
13
|
+
ahead: "ahead";
|
|
14
|
+
behind: "behind";
|
|
15
|
+
"in-sync": "in-sync";
|
|
16
|
+
}>;
|
|
17
|
+
export type SettingsPropagationStatus = z.infer<typeof settingsPropagationStatusSchema>;
|
|
18
|
+
export declare const SettingsPropagationStatus: {
|
|
19
|
+
ahead: "ahead";
|
|
20
|
+
behind: "behind";
|
|
21
|
+
"in-sync": "in-sync";
|
|
22
|
+
};
|
|
23
|
+
export declare const settingsPropagationRoleSchema: z.ZodEnum<{
|
|
24
|
+
coordinator: "coordinator";
|
|
25
|
+
worker: "worker";
|
|
26
|
+
}>;
|
|
27
|
+
export declare const settingsPropagationEntrySchema: z.ZodObject<{
|
|
28
|
+
instanceId: z.ZodString;
|
|
29
|
+
role: z.ZodEnum<{
|
|
30
|
+
coordinator: "coordinator";
|
|
31
|
+
worker: "worker";
|
|
32
|
+
}>;
|
|
33
|
+
self: z.ZodBoolean;
|
|
34
|
+
connected: z.ZodBoolean;
|
|
35
|
+
appliedVersion: z.ZodNumber;
|
|
36
|
+
status: z.ZodEnum<{
|
|
37
|
+
ahead: "ahead";
|
|
38
|
+
behind: "behind";
|
|
39
|
+
"in-sync": "in-sync";
|
|
40
|
+
}>;
|
|
41
|
+
lastHeartbeatAt: z.ZodNullable<z.ZodString>;
|
|
42
|
+
lastHeartbeatAgeMs: z.ZodNullable<z.ZodNumber>;
|
|
43
|
+
}, z.core.$loose>;
|
|
44
|
+
export type SettingsPropagationEntry = z.infer<typeof settingsPropagationEntrySchema>;
|
|
45
|
+
export declare const settingsPropagationReportSchema: z.ZodObject<{
|
|
46
|
+
currentVersion: z.ZodNumber;
|
|
47
|
+
reportedBy: z.ZodString;
|
|
48
|
+
generatedAt: z.ZodString;
|
|
49
|
+
orchestrators: z.ZodArray<z.ZodObject<{
|
|
50
|
+
instanceId: z.ZodString;
|
|
51
|
+
role: z.ZodEnum<{
|
|
52
|
+
coordinator: "coordinator";
|
|
53
|
+
worker: "worker";
|
|
54
|
+
}>;
|
|
55
|
+
self: z.ZodBoolean;
|
|
56
|
+
connected: z.ZodBoolean;
|
|
57
|
+
appliedVersion: z.ZodNumber;
|
|
58
|
+
status: z.ZodEnum<{
|
|
59
|
+
ahead: "ahead";
|
|
60
|
+
behind: "behind";
|
|
61
|
+
"in-sync": "in-sync";
|
|
62
|
+
}>;
|
|
63
|
+
lastHeartbeatAt: z.ZodNullable<z.ZodString>;
|
|
64
|
+
lastHeartbeatAgeMs: z.ZodNullable<z.ZodNumber>;
|
|
65
|
+
}, z.core.$loose>>;
|
|
66
|
+
}, z.core.$loose>;
|
|
67
|
+
export type SettingsPropagationReport = z.infer<typeof settingsPropagationReportSchema>;
|
|
68
|
+
/** The peer-registry fields a report reads. */
|
|
69
|
+
export type PropagationPeer = Pick<PeerInfo, 'instanceId' | 'role' | 'connected' | 'lastHeartbeatAt' | 'clusterSettingsVersion'>;
|
|
70
|
+
/**
|
|
71
|
+
* `ahead` means the row's version went backwards. A worker in that state never
|
|
72
|
+
* pulls again, because it pulls only when a coordinator advertises a higher
|
|
73
|
+
* version than its own.
|
|
74
|
+
*/
|
|
75
|
+
export declare function classifySettingsVersion(applied: number, current: number): SettingsPropagationStatus;
|
|
76
|
+
export declare function buildSettingsPropagationReport(input: {
|
|
77
|
+
currentVersion: number;
|
|
78
|
+
self: {
|
|
79
|
+
instanceId: string;
|
|
80
|
+
appliedVersion: number;
|
|
81
|
+
};
|
|
82
|
+
peers: readonly PropagationPeer[];
|
|
83
|
+
now: number;
|
|
84
|
+
}): SettingsPropagationReport;
|
|
85
|
+
//# sourceMappingURL=settings-propagation.d.ts.map
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The cluster-settings snapshot a coordinator serves to DB-less workers.
|
|
3
|
+
*
|
|
4
|
+
* A worker cannot read `cluster_settings`, so every knob it applies at spawn
|
|
5
|
+
* reaches it through this snapshot, pulled over the peer channel whenever a
|
|
6
|
+
* coordinator advertises a newer settings version. Each value is the stored
|
|
7
|
+
* override, or the serving coordinator's configured default when the column
|
|
8
|
+
* is NULL.
|
|
9
|
+
*/
|
|
10
|
+
import type { WorkerClusterSettings } from '@kici-dev/engine';
|
|
11
|
+
import type { ClusterSettingsReader } from './cluster-settings-reader.js';
|
|
12
|
+
/** The configured defaults behind each snapshot field. */
|
|
13
|
+
export interface WorkerClusterSettingDefaults {
|
|
14
|
+
agentTokenTtlMs: number;
|
|
15
|
+
firecrackerApiSocketWaitMs: number;
|
|
16
|
+
}
|
|
17
|
+
/** A worker snapshot and the `cluster_settings.version` its values belong to. */
|
|
18
|
+
export interface WorkerClusterSettingsSnapshot {
|
|
19
|
+
version: number;
|
|
20
|
+
settings: WorkerClusterSettings;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Resolve the worker snapshot and its version from ONE read of the row. A
|
|
24
|
+
* worker stores the version with the values and pulls again only when a
|
|
25
|
+
* coordinator advertises a higher version, so a version read separately from
|
|
26
|
+
* the values could pair a newer version with older values for good.
|
|
27
|
+
*/
|
|
28
|
+
export declare function resolveWorkerClusterSettingsSnapshot(reader: Pick<ClusterSettingsReader, 'getRow'>, defaults: WorkerClusterSettingDefaults): Promise<WorkerClusterSettingsSnapshot>;
|
|
29
|
+
//# sourceMappingURL=worker-cluster-settings.d.ts.map
|
package/dist/config.d.ts
CHANGED
|
@@ -72,6 +72,7 @@ declare const configSchema: z.ZodObject<{
|
|
|
72
72
|
globalEvalCandidateTimeoutMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
73
73
|
globalEvalCacheMax: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
74
74
|
globalEvalWaitTimeoutMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
75
|
+
firecrackerApiSocketWaitMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
75
76
|
queueMaxDepth: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
76
77
|
queueTimeoutMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
77
78
|
unroutableGraceMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
@@ -104,6 +105,8 @@ declare const configSchema: z.ZodObject<{
|
|
|
104
105
|
rerouteSpawnWindowMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
105
106
|
rerouteAckTimeoutMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
106
107
|
rerouteMaxHops: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
108
|
+
rerouteSpawnMaxAttempts: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
109
|
+
rerouteSpawnRetryBackoffMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
107
110
|
rerouteFlapGraceMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
108
111
|
scalerSpawnTimeoutMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
109
112
|
backupStalenessWarnHours: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
@@ -147,6 +150,7 @@ declare const configSchema: z.ZodObject<{
|
|
|
147
150
|
reconnectReplayWindowHours: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
148
151
|
cacheBuildTimeoutMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
149
152
|
cacheMaxTarballBytes: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
153
|
+
cacheUploadSettleTimeoutMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
150
154
|
userCacheQuotaBytes: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
151
155
|
userCacheTtlMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
152
156
|
artifactQuotaBytes: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
@@ -282,6 +286,7 @@ export declare const packagingConfigSchema: z.ZodObject<{
|
|
|
282
286
|
globalEvalCandidateTimeoutMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
283
287
|
globalEvalCacheMax: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
284
288
|
globalEvalWaitTimeoutMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
289
|
+
firecrackerApiSocketWaitMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
285
290
|
queueMaxDepth: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
286
291
|
queueTimeoutMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
287
292
|
unroutableGraceMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
@@ -314,6 +319,8 @@ export declare const packagingConfigSchema: z.ZodObject<{
|
|
|
314
319
|
rerouteSpawnWindowMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
315
320
|
rerouteAckTimeoutMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
316
321
|
rerouteMaxHops: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
322
|
+
rerouteSpawnMaxAttempts: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
323
|
+
rerouteSpawnRetryBackoffMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
317
324
|
rerouteFlapGraceMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
318
325
|
scalerSpawnTimeoutMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
319
326
|
backupStalenessWarnHours: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
@@ -357,6 +364,7 @@ export declare const packagingConfigSchema: z.ZodObject<{
|
|
|
357
364
|
reconnectReplayWindowHours: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
358
365
|
cacheBuildTimeoutMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
359
366
|
cacheMaxTarballBytes: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
367
|
+
cacheUploadSettleTimeoutMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
360
368
|
userCacheQuotaBytes: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
361
369
|
userCacheTtlMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
362
370
|
artifactQuotaBytes: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
|
|
@@ -525,6 +533,7 @@ export declare const envDef: import("@kici-dev/shared/env").DefineEnvResult<{
|
|
|
525
533
|
globalEvalCandidateTimeoutMs: number;
|
|
526
534
|
globalEvalCacheMax: number;
|
|
527
535
|
globalEvalWaitTimeoutMs: number;
|
|
536
|
+
firecrackerApiSocketWaitMs: number;
|
|
528
537
|
queueMaxDepth: number;
|
|
529
538
|
queueTimeoutMs: number;
|
|
530
539
|
unroutableGraceMs: number;
|
|
@@ -554,6 +563,8 @@ export declare const envDef: import("@kici-dev/shared/env").DefineEnvResult<{
|
|
|
554
563
|
rerouteSpawnWindowMs: number;
|
|
555
564
|
rerouteAckTimeoutMs: number;
|
|
556
565
|
rerouteMaxHops: number;
|
|
566
|
+
rerouteSpawnMaxAttempts: number;
|
|
567
|
+
rerouteSpawnRetryBackoffMs: number;
|
|
557
568
|
rerouteFlapGraceMs: number;
|
|
558
569
|
scalerSpawnTimeoutMs: number;
|
|
559
570
|
backupStalenessWarnHours: number;
|
|
@@ -591,6 +602,7 @@ export declare const envDef: import("@kici-dev/shared/env").DefineEnvResult<{
|
|
|
591
602
|
reconnectReplayWindowHours: number;
|
|
592
603
|
cacheBuildTimeoutMs: number;
|
|
593
604
|
cacheMaxTarballBytes: number;
|
|
605
|
+
cacheUploadSettleTimeoutMs: number;
|
|
594
606
|
userCacheQuotaBytes: number;
|
|
595
607
|
userCacheTtlMs: number;
|
|
596
608
|
artifactQuotaBytes: number;
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { type Kysely } from 'kysely';
|
|
2
|
+
/**
|
|
3
|
+
* Add `org_settings.cache_upload_settle_timeout_ms BIGINT` (nullable).
|
|
4
|
+
*
|
|
5
|
+
* Per-org bound on how long a build job's `job.status success` waits for the
|
|
6
|
+
* cache publish (object metadata plus pointer) its agent reported just before
|
|
7
|
+
* it. NULL falls back to the cluster default
|
|
8
|
+
* (`KICI_CACHE_UPLOAD_SETTLE_TIMEOUT_MS`, 10s); 0 turns the wait off. Operators
|
|
9
|
+
* tune it via `kici-admin org-settings cache-upload-settle`.
|
|
10
|
+
*
|
|
11
|
+
* Idempotent: a re-run on a DB that already has the column is a no-op.
|
|
12
|
+
*/
|
|
13
|
+
export declare function up(db: Kysely<unknown>): Promise<void>;
|
|
14
|
+
export declare function down(db: Kysely<unknown>): Promise<void>;
|
|
15
|
+
//# sourceMappingURL=152_org_settings_cache_upload_settle_timeout.d.ts.map
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { type Kysely } from 'kysely';
|
|
2
|
+
/**
|
|
3
|
+
* Add `execution_runs.cancel_force BOOLEAN` (nullable, no default).
|
|
4
|
+
*
|
|
5
|
+
* TRUE once a forced cancel reached the run. The leader's stuck-cancelling
|
|
6
|
+
* sweep reads it to re-send a cancel that did not complete with `force` set,
|
|
7
|
+
* so a forced cancel whose first forward to a sibling coordinator was lost is
|
|
8
|
+
* not re-sent as a graceful one. NULL (every existing row, and every run no
|
|
9
|
+
* forced cancel reached) re-sends gracefully. No backfill: nothing recorded
|
|
10
|
+
* whether an earlier cancel was forced.
|
|
11
|
+
*
|
|
12
|
+
* Idempotent: a re-run on a DB that already has the column is a no-op.
|
|
13
|
+
*/
|
|
14
|
+
export declare function up(db: Kysely<unknown>): Promise<void>;
|
|
15
|
+
export declare function down(db: Kysely<unknown>): Promise<void>;
|
|
16
|
+
//# sourceMappingURL=153_execution_runs_cancel_force.d.ts.map
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { type Kysely } from 'kysely';
|
|
2
|
+
/**
|
|
3
|
+
* Add the two per-org spawn-retry columns a worker applies to a rerouted job (both nullable):
|
|
4
|
+
*
|
|
5
|
+
* - `reroute_spawn_max_attempts INTEGER` — agent spawns a worker attempts for one rerouted
|
|
6
|
+
* job before it gives the job back to its coordinator. NULL falls back to the cluster
|
|
7
|
+
* default (`rerouteSpawnMaxAttempts`, 3).
|
|
8
|
+
* - `reroute_spawn_retry_backoff_ms BIGINT` — wait after a failed spawn before the next
|
|
9
|
+
* attempt. NULL falls back to `rerouteSpawnRetryBackoffMs` (5 s).
|
|
10
|
+
*
|
|
11
|
+
* Operators tune them per org via `kici-admin org-settings reroute set`.
|
|
12
|
+
*
|
|
13
|
+
* Idempotent: each add is guarded on column existence.
|
|
14
|
+
*/
|
|
15
|
+
export declare function up(db: Kysely<unknown>): Promise<void>;
|
|
16
|
+
export declare function down(db: Kysely<unknown>): Promise<void>;
|
|
17
|
+
//# sourceMappingURL=155_org_settings_reroute_spawn_retry.d.ts.map
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { type Kysely } from 'kysely';
|
|
2
|
+
/**
|
|
3
|
+
* Record how the event router resolved each internal event:
|
|
4
|
+
*
|
|
5
|
+
* - `match_outcome TEXT` — `matched`, `buffered`, `no-registration`,
|
|
6
|
+
* `no-target-repo`, `trust-blocked` or `no-trigger-match`.
|
|
7
|
+
* - `matched_count INTEGER` — how many registrations the event was dispatched to.
|
|
8
|
+
*
|
|
9
|
+
* Both are written when the event is processed and are NULL before that, and
|
|
10
|
+
* for rows processed before these columns existed. Nullable with no default,
|
|
11
|
+
* so the ALTER is catalog-only on a large table.
|
|
12
|
+
*
|
|
13
|
+
* Idempotent: `ADD COLUMN IF NOT EXISTS`.
|
|
14
|
+
*/
|
|
15
|
+
export declare function up(db: Kysely<unknown>): Promise<void>;
|
|
16
|
+
export declare function down(db: Kysely<unknown>): Promise<void>;
|
|
17
|
+
//# sourceMappingURL=156_kici_events_match_outcome.d.ts.map
|
package/dist/db/types.d.ts
CHANGED
|
@@ -514,6 +514,13 @@ export interface ExecutionRunTable {
|
|
|
514
514
|
* cancelled and for one that entered `cancelling` before the column existed.
|
|
515
515
|
*/
|
|
516
516
|
cancelling_at: ColumnType<Date | null, Date | null | undefined, Date | null>;
|
|
517
|
+
/**
|
|
518
|
+
* TRUE once a forced cancel reached the run (migration 153). The
|
|
519
|
+
* stuck-cancelling re-drive reads it so a cancel that did not complete is
|
|
520
|
+
* re-sent with `force`. Never reset to FALSE: a later graceful cancel does
|
|
521
|
+
* not undo a forced one. NULL re-drives gracefully.
|
|
522
|
+
*/
|
|
523
|
+
cancel_force: ColumnType<boolean | null, boolean | null | undefined, boolean | null>;
|
|
517
524
|
/**
|
|
518
525
|
* The run's status generation (migration 150). Starts at 0 and rises by one
|
|
519
526
|
* each time the run leaves a terminal status to continue. Forwarded on every
|
|
@@ -1499,6 +1506,10 @@ export interface KiciEventTable {
|
|
|
1499
1506
|
dlq_at: Date | null;
|
|
1500
1507
|
/** Short DLQ reason: 'exhausted_retries' | 'non_retryable' */
|
|
1501
1508
|
dlq_reason: string | null;
|
|
1509
|
+
/** How the router resolved the event (an `EventMatchOutcome`); NULL until processed */
|
|
1510
|
+
match_outcome: string | null;
|
|
1511
|
+
/** Registrations the event was dispatched to; NULL until processed */
|
|
1512
|
+
matched_count: number | null;
|
|
1502
1513
|
}
|
|
1503
1514
|
export type KiciEvent = Selectable<KiciEventTable>;
|
|
1504
1515
|
export type NewKiciEvent = Insertable<KiciEventTable>;
|
|
@@ -1982,6 +1993,16 @@ export interface OrgSettingsTable {
|
|
|
1982
1993
|
* (config.rerouteMaxHops). Plain INTEGER — pg returns a number on select.
|
|
1983
1994
|
*/
|
|
1984
1995
|
reroute_max_hops: ColumnType<number | null, number | null | undefined, number | null>;
|
|
1996
|
+
/**
|
|
1997
|
+
* Per-org agent spawns a worker attempts for one rerouted job. Null = cluster
|
|
1998
|
+
* default (config.rerouteSpawnMaxAttempts). Plain INTEGER.
|
|
1999
|
+
*/
|
|
2000
|
+
reroute_spawn_max_attempts: ColumnType<number | null, number | null | undefined, number | null>;
|
|
2001
|
+
/**
|
|
2002
|
+
* Per-org wait (ms) after a failed spawn of a rerouted job. Null = cluster
|
|
2003
|
+
* default (config.rerouteSpawnRetryBackoffMs). BIGINT.
|
|
2004
|
+
*/
|
|
2005
|
+
reroute_spawn_retry_backoff_ms: ColumnType<string | null, number | null | undefined, number | null>;
|
|
1985
2006
|
/**
|
|
1986
2007
|
* Per-org staleness threshold (hours) for the DB-backup freshness diagnostic.
|
|
1987
2008
|
* Null = cluster default (config.backupStalenessWarnHours). Plain INTEGER.
|
|
@@ -1994,6 +2015,14 @@ export interface OrgSettingsTable {
|
|
|
1994
2015
|
* returns a string on select; accept a number on insert/update.
|
|
1995
2016
|
*/
|
|
1996
2017
|
queue_timeout_ms: ColumnType<string | null, number | null | undefined, number | null>;
|
|
2018
|
+
/**
|
|
2019
|
+
* Per-org bound (ms) on how long a build job's success waits for the cache
|
|
2020
|
+
* publish its agent reported just before it. 0 turns the wait off. Null =
|
|
2021
|
+
* cluster default (config.cacheUploadSettleTimeoutMs /
|
|
2022
|
+
* KICI_CACHE_UPLOAD_SETTLE_TIMEOUT_MS). Postgres BIGINT — pg returns a string
|
|
2023
|
+
* on select; accept a number on insert/update.
|
|
2024
|
+
*/
|
|
2025
|
+
cache_upload_settle_timeout_ms: ColumnType<string | null, number | null | undefined, number | null>;
|
|
1997
2026
|
/**
|
|
1998
2027
|
* Per-org expiry (seconds) for a held approval element before it is rejected
|
|
1999
2028
|
* and its run/job/step fails. NOT NULL, default 86400 (one day). An SDK
|
|
@@ -2182,6 +2211,8 @@ export interface ClusterSettingsTable {
|
|
|
2182
2211
|
* per round.
|
|
2183
2212
|
*/
|
|
2184
2213
|
global_eval_wait_timeout_ms: ColumnType<string | null, number | null | undefined, number | null>;
|
|
2214
|
+
/** How long a Firecracker spawn waits for the VM's API socket. NULL: config default. */
|
|
2215
|
+
firecracker_api_socket_wait_ms: ColumnType<string | null, number | null | undefined, number | null>;
|
|
2185
2216
|
/**
|
|
2186
2217
|
* Retention window in days for `check_run_tracking` rows. The hourly cleanup
|
|
2187
2218
|
* sweep deletes rows untouched for longer than this; 0 disables the sweep.
|
|
@@ -6,7 +6,7 @@ import type { EventStore } from './event-store.js';
|
|
|
6
6
|
import type { EventCircuitBreaker } from './circuit-breaker.js';
|
|
7
7
|
import type { TrustStore } from './trust-store.js';
|
|
8
8
|
import type { RegisteredWorkflow, RegistrationIndex } from '../registration/registration-index.js';
|
|
9
|
-
import type
|
|
9
|
+
import { type EventRouterConfig, type StoredEvent } from './types.js';
|
|
10
10
|
import type { ClusterSettingsReader } from '../cluster/cluster-settings-reader.js';
|
|
11
11
|
/**
|
|
12
12
|
* System events are exempt from the per-source event-storm rate limiter: they
|
|
@@ -213,6 +213,10 @@ export declare class EventRouter {
|
|
|
213
213
|
* schedule a retry or move the event to the DLQ. Swallowing the error
|
|
214
214
|
* here would silently lose the event — exactly the failure mode the
|
|
215
215
|
* lease pattern fixes.
|
|
216
|
+
*
|
|
217
|
+
* Returns how the event resolved, which `dispatchAndRecord` persists on the
|
|
218
|
+
* processed row: `matched` when it dispatched at least one workflow, else
|
|
219
|
+
* the reason nothing ran.
|
|
216
220
|
*/
|
|
217
221
|
private processSubscriptions;
|
|
218
222
|
/**
|
|
@@ -225,6 +229,8 @@ export declare class EventRouter {
|
|
|
225
229
|
* itself dispatched by a failure-lifecycle trigger is neither buffered nor
|
|
226
230
|
* dispatched — a broken notifier must not re-trigger the batch on its own
|
|
227
231
|
* failure. The chain-depth breaker remains the backstop for anything missed.
|
|
232
|
+
*
|
|
233
|
+
* `bufferedCount` counts the decisions appended to a batch window.
|
|
228
234
|
*/
|
|
229
235
|
private bufferBatchDecisions;
|
|
230
236
|
/** Open (or reuse) the registration's batch window and append the failed run. */
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { type Kysely, type Transaction } from 'kysely';
|
|
2
2
|
import type { Database } from '../db/types.js';
|
|
3
|
-
import type
|
|
3
|
+
import { EventMatchOutcome, type DlqReason, type EventMatchResult, type EventRouterConfig, type StoredEvent } from './types.js';
|
|
4
4
|
/** Either a Kysely DB handle or an active transaction. */
|
|
5
5
|
export type DbExecutor = Kysely<Database> | Transaction<Database>;
|
|
6
6
|
/**
|
|
@@ -81,11 +81,28 @@ export declare class EventStore {
|
|
|
81
81
|
* re-queries until a page returns fewer than `limit` rows.
|
|
82
82
|
*/
|
|
83
83
|
getUnprocessedSince(sinceId: string | null, limit?: number): Promise<StoredEvent[]>;
|
|
84
|
+
/**
|
|
85
|
+
* List events newest first, for `kici-admin event list`. Every filter is
|
|
86
|
+
* optional except the limit. `before` keeps events created before a
|
|
87
|
+
* timestamp. `afterEventId` is the next-page cursor: the events that sort
|
|
88
|
+
* after that event in this order, compared on the stored `(created_at, id)`
|
|
89
|
+
* so events created in the same millisecond are neither skipped nor repeated.
|
|
90
|
+
*/
|
|
91
|
+
list(opts: {
|
|
92
|
+
name?: string;
|
|
93
|
+
outcome?: EventMatchOutcome;
|
|
94
|
+
since?: Date;
|
|
95
|
+
before?: Date;
|
|
96
|
+
afterEventId?: string;
|
|
97
|
+
limit: number;
|
|
98
|
+
sourceRoutingKey?: string;
|
|
99
|
+
}): Promise<StoredEvent[]>;
|
|
84
100
|
/**
|
|
85
101
|
* Mark an event as fully processed. Called after a successful dispatch.
|
|
86
|
-
* Clears any prior lease so the row is unambiguously terminal
|
|
102
|
+
* Clears any prior lease so the row is unambiguously terminal, and records
|
|
103
|
+
* how the router resolved it when `result` is given.
|
|
87
104
|
*/
|
|
88
|
-
markProcessed(id: string): Promise<void>;
|
|
105
|
+
markProcessed(id: string, result?: EventMatchResult): Promise<void>;
|
|
89
106
|
/**
|
|
90
107
|
* Atomically take a dispatch lease on an event. Replaces the previous
|
|
91
108
|
* "flip processed=true upfront" pattern that silently lost events when
|
package/dist/events/types.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Event routing configuration and types.
|
|
3
3
|
*/
|
|
4
|
+
import { z } from 'zod';
|
|
4
5
|
/**
|
|
5
6
|
* Configuration for the event routing system.
|
|
6
7
|
*/
|
|
@@ -45,6 +46,22 @@ export interface EventRouterConfig {
|
|
|
45
46
|
* Reason an event landed in the DLQ.
|
|
46
47
|
*/
|
|
47
48
|
export type DlqReason = 'exhausted_retries' | 'non_retryable';
|
|
49
|
+
/** How the router resolved one event against the registered workflows. */
|
|
50
|
+
export declare const EventMatchOutcome: z.ZodEnum<{
|
|
51
|
+
buffered: "buffered";
|
|
52
|
+
matched: "matched";
|
|
53
|
+
"no-registration": "no-registration";
|
|
54
|
+
"no-target-repo": "no-target-repo";
|
|
55
|
+
"no-trigger-match": "no-trigger-match";
|
|
56
|
+
"trust-blocked": "trust-blocked";
|
|
57
|
+
}>;
|
|
58
|
+
export type EventMatchOutcome = z.infer<typeof EventMatchOutcome>;
|
|
59
|
+
/** What the router resolved for one event, persisted on its processed row. */
|
|
60
|
+
export interface EventMatchResult {
|
|
61
|
+
outcome: EventMatchOutcome;
|
|
62
|
+
/** Registrations the event was dispatched to. */
|
|
63
|
+
matchedCount: number;
|
|
64
|
+
}
|
|
48
65
|
/**
|
|
49
66
|
* A stored internal event (matches DB row shape but with JS types).
|
|
50
67
|
*/
|
|
@@ -70,7 +87,35 @@ export interface StoredEvent {
|
|
|
70
87
|
nextRetryAt: Date | null;
|
|
71
88
|
dlqAt: Date | null;
|
|
72
89
|
dlqReason: DlqReason | null;
|
|
90
|
+
/**
|
|
91
|
+
* How the router resolved the event. Set when the event is processed; null
|
|
92
|
+
* before, and for rows processed before the column existed.
|
|
93
|
+
*/
|
|
94
|
+
matchOutcome: EventMatchOutcome | null;
|
|
95
|
+
/** Registrations the event was dispatched to; null when `matchOutcome` is. */
|
|
96
|
+
matchedCount: number | null;
|
|
73
97
|
}
|
|
98
|
+
/**
|
|
99
|
+
* An event payload safe to show an operator. An event-scaler scale-up carries
|
|
100
|
+
* a single-use claim code an agent redeems for credentials, so every read
|
|
101
|
+
* surface (event show, the dead-letter queue) replaces it with `[redacted]`.
|
|
102
|
+
*/
|
|
103
|
+
export declare function redactEventPayload(payload: Record<string, unknown>): Record<string, unknown>;
|
|
104
|
+
/** Where an internal event is in its dispatch lifecycle. */
|
|
105
|
+
export declare const EventProcessingState: z.ZodEnum<{
|
|
106
|
+
dlq: "dlq";
|
|
107
|
+
leased: "leased";
|
|
108
|
+
pending: "pending";
|
|
109
|
+
processed: "processed";
|
|
110
|
+
retrying: "retrying";
|
|
111
|
+
}>;
|
|
112
|
+
export type EventProcessingState = z.infer<typeof EventProcessingState>;
|
|
113
|
+
/**
|
|
114
|
+
* Derive an event's dispatch state from its row: DLQ wins, then processed;
|
|
115
|
+
* an unprocessed row is leased while a node holds it, retrying once a failed
|
|
116
|
+
* dispatch scheduled another attempt, and pending before its first lease.
|
|
117
|
+
*/
|
|
118
|
+
export declare function eventProcessingState(e: StoredEvent): EventProcessingState;
|
|
74
119
|
/**
|
|
75
120
|
* Default event router configuration.
|
|
76
121
|
*/
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Running a Windows batch file (`.cmd` / `.bat`) through cmd.exe.
|
|
3
|
+
*
|
|
4
|
+
* Node.js refuses to start a batch file without a shell (it throws `EINVAL`),
|
|
5
|
+
* and every launcher a KiCI package ships on Windows is a batch file. Two
|
|
6
|
+
* callers run one: the service installer, which registers the command with
|
|
7
|
+
* shawl, and the bare-metal scaler, which spawns it.
|
|
8
|
+
*
|
|
9
|
+
* Both quote a token on cmd.exe's command line only when it contains a space
|
|
10
|
+
* or a tab — shawl does, and so does libuv when Node.js builds a Windows
|
|
11
|
+
* command line — so one safety rule fits both.
|
|
12
|
+
*/
|
|
13
|
+
/**
|
|
14
|
+
* The cmd.exe arguments that run a batch-file launcher, ahead of its path.
|
|
15
|
+
*
|
|
16
|
+
* shawl stops the service with a ctrl-C to its console, and cmd.exe gets it
|
|
17
|
+
* too: when the batch file's node process exits, cmd.exe asks "Terminate batch
|
|
18
|
+
* job (Y/N)?" and waits for an answer. shawl sees the service exit only when its
|
|
19
|
+
* stop timeout kills cmd.exe. With stdin from NUL ({@link BATCH_STDIN}), cmd.exe
|
|
20
|
+
* reads end-of-input at that prompt and exits with node.
|
|
21
|
+
*
|
|
22
|
+
* `call` keeps the launcher path out of the first position after `/c`, where
|
|
23
|
+
* cmd.exe strips the quotes from a path that contains `(`, `)` or `&`. `/e:on`
|
|
24
|
+
* and `/v:off` pin command extensions (the launchers use `%~dp0`) and delayed
|
|
25
|
+
* expansion whatever the host's registry defaults are.
|
|
26
|
+
*/
|
|
27
|
+
export declare const BATCH_WRAPPER_ARGS: readonly ['/d', '/e:on', '/v:off', '/c', 'call'];
|
|
28
|
+
/** The redirect that closes {@link BATCH_WRAPPER_ARGS}: the launcher's stdin is NUL. */
|
|
29
|
+
export declare const BATCH_STDIN = "<NUL";
|
|
30
|
+
/** A `.cmd` or `.bat` file, which Windows runs through cmd.exe. */
|
|
31
|
+
export declare function isBatchFile(executablePath: string): boolean;
|
|
32
|
+
/**
|
|
33
|
+
* The cmd.exe that COMSPEC names in `env`, as Node itself uses to run a shell on
|
|
34
|
+
* Windows. A COMSPEC that names another shell is not used: the wrapper
|
|
35
|
+
* arguments are cmd.exe's.
|
|
36
|
+
*/
|
|
37
|
+
export declare function cmdExePath(env?: NodeJS.ProcessEnv): string;
|
|
38
|
+
/**
|
|
39
|
+
* Why cmd.exe would read part of `token` as syntax, or null when it would not.
|
|
40
|
+
*
|
|
41
|
+
* A token is quoted on cmd.exe's command line only when it contains a space or
|
|
42
|
+
* a tab. Inside quotes cmd.exe reads `&`, `|`, `<`, `>`, `(`, `)` and the `,`
|
|
43
|
+
* `;` `=` delimiters as text; outside quotes it reads them as syntax. `%` and
|
|
44
|
+
* `^` change even inside quotes (`call` expands `%` again and doubles `^`), and
|
|
45
|
+
* a `"` ends the quoting.
|
|
46
|
+
*
|
|
47
|
+
* @param use - what the token runs as, completing "cannot run <token> …".
|
|
48
|
+
* @param remedy - the sentence that tells the operator what to change.
|
|
49
|
+
*/
|
|
50
|
+
export declare function cmdSafetyRefusal(token: string, use: string, remedy: string): string | null;
|
|
51
|
+
/** Throw the {@link cmdSafetyRefusal} message for `token`, if it has one. */
|
|
52
|
+
export declare function assertCmdSafe(token: string, use: string, remedy: string): void;
|
|
53
|
+
/**
|
|
54
|
+
* The cmd.exe command that runs `command` — a batch file and its arguments —
|
|
55
|
+
* with stdin from NUL. The caller checks each token with
|
|
56
|
+
* {@link assertCmdSafe} first.
|
|
57
|
+
*/
|
|
58
|
+
export declare function batchFileCommand(command: readonly string[], env?: NodeJS.ProcessEnv): string[];
|
|
59
|
+
//# sourceMappingURL=windows-batch.d.ts.map
|