@oisincoveney/pipeline 3.24.3 → 4.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -0
- package/dist/argo-graph.js +4 -2
- package/dist/argo-submit.d.ts +1 -1
- package/dist/argo-submit.js +39 -31
- package/dist/argo-workflow.d.ts +0 -1
- package/dist/cli/doctor.js +5 -3
- package/dist/cli/factory-commands.js +1 -2
- package/dist/cli/program.d.ts +1 -1
- package/dist/cli/program.js +8 -2
- package/dist/cli/run-command.d.ts +1 -1
- package/dist/cli/run-commands.js +7 -5
- package/dist/cli/run-service.d.ts +0 -1
- package/dist/cli/run-service.js +11 -8
- package/dist/cli/submit-options.js +10 -4
- package/dist/commands/coordinator-command.js +35 -0
- package/dist/commands/pipeline-command.js +4 -2
- package/dist/commands/runner-command-command.js +11 -6
- package/dist/commands/runner-node-command.js +31 -0
- package/dist/commands/ticket/create.js +4 -2
- package/dist/commands/ticket/sequence.js +2 -1
- package/dist/commands/ticket/shared.d.ts +4 -1
- package/dist/config/defaults.d.ts +0 -1
- package/dist/config/load.d.ts +0 -1
- package/dist/config/load.js +3 -1
- package/dist/config/schema/catalog.d.ts +2 -1
- package/dist/config/schema/catalog.js +2 -1
- package/dist/config/schemas.d.ts +7 -3
- package/dist/config/schemas.js +3 -1
- package/dist/config/validate.d.ts +0 -1
- package/dist/config.d.ts +2 -2
- package/dist/context/repo-map.js +5 -2
- package/dist/coordinator/composition.js +169 -0
- package/dist/coordinator/config.js +71 -0
- package/dist/coordinator/run-owner.js +64 -0
- package/dist/coordinator/wave-advance.js +148 -0
- package/dist/coordinator/wave-scheduler.js +50 -0
- package/dist/factory/create-experiment.js +252 -84
- package/dist/hooks.d.ts +0 -1
- package/dist/index.d.ts +3 -2
- package/dist/index.js +6 -1
- package/dist/install-commands/claude-code.js +4 -0
- package/dist/install-commands/opencode.js +2 -1
- package/dist/install-commands/planner.js +6 -3
- package/dist/loop/argo-poll.d.ts +0 -1
- package/dist/loop/argo-poll.js +3 -64
- package/dist/loop/controller-deps.js +9 -7
- package/dist/loop/controller.js +5 -3
- package/dist/loop/loop-controller-entrypoint.js +2 -1
- package/dist/loop/run-terminal-poll.js +69 -0
- package/dist/mcp/gateway-doctor.js +1 -31
- package/dist/mcp/repo-local-backends.js +2 -1
- package/dist/moka-global-config.d.ts +0 -1
- package/dist/moka-global-config.js +6 -3
- package/dist/moka-submit.d.ts +17 -3
- package/dist/pipeline-runtime.d.ts +5 -4
- package/dist/pipeline-runtime.js +4 -2
- package/dist/planning/compile.d.ts +2 -2
- package/dist/planning/generate.d.ts +3 -3
- package/dist/planning/generate.js +5 -3
- package/dist/planning/graph.d.ts +0 -1
- package/dist/remote/argo/model.d.ts +1 -1
- package/dist/remote/jobs/execution-identity.js +70 -0
- package/dist/remote/jobs/job-outcome.js +108 -0
- package/dist/remote/jobs/manifest.js +96 -0
- package/dist/remote/jobs/model.d.ts +158 -0
- package/dist/remote/jobs/model.js +162 -0
- package/dist/remote/jobs/node-bootstrap.js +213 -0
- package/dist/remote/jobs/policy.js +123 -0
- package/dist/remote/jobs/reconcile.js +100 -0
- package/dist/remote/jobs/run-spec-client.js +64 -0
- package/dist/remote/jobs/run-spec-path.js +31 -0
- package/dist/remote/jobs/run-spec-server.d.ts +34 -0
- package/dist/remote/jobs/run-spec-server.js +116 -0
- package/dist/remote/jobs/submit.d.ts +23 -0
- package/dist/remote/jobs/submit.js +116 -0
- package/dist/remote/jobs/watch.js +59 -0
- package/dist/remote/submit/argo-submission.d.ts +3 -1
- package/dist/remote/submit/compilation.d.ts +2 -0
- package/dist/remote/submit/hook-events.d.ts +1 -0
- package/dist/remote/submit/io.d.ts +7 -1
- package/dist/remote/submit/jobs-submission.d.ts +59 -0
- package/dist/remote/submit/jobs-submission.js +97 -0
- package/dist/remote/submit/jobs-submit-result.d.ts +25 -0
- package/dist/remote/submit/jobs-submit-result.js +34 -0
- package/dist/remote/submit/options.d.ts +0 -1
- package/dist/remote/submit/service.d.ts +40 -1
- package/dist/remote/submit/service.js +23 -1
- package/dist/run-control/contracts.d.ts +12 -1
- package/dist/run-control/contracts.js +28 -1
- package/dist/run-control/execution-store.js +0 -0
- package/dist/run-control/next-node.js +2 -1
- package/dist/run-control/postgres/postgres-run-control-store.js +4 -4
- package/dist/run-control/resume-command.js +2 -1
- package/dist/run-control/run-artifacts-command.js +2 -1
- package/dist/run-control/run-record.js +3 -1
- package/dist/run-control/runtime-reporter.js +3 -1
- package/dist/run-control/supervisor.js +2 -1
- package/dist/run-state/git-refs.js +2 -1
- package/dist/runner-command/environment-commands.js +91 -0
- package/dist/runner-command/lifecycle-context.js +5 -2
- package/dist/runner-command/lifecycle.js +1 -1
- package/dist/runner-command/pre-schedule.js +3 -2
- package/dist/runner-command/run.js +65 -48
- package/dist/runner-command/select-ready-wave.js +1 -1
- package/dist/runner-command-contract.d.ts +5 -6
- package/dist/runner-command-contract.js +11 -10
- package/dist/runner-event-schema.d.ts +12 -13
- package/dist/runner.d.ts +2 -2
- package/dist/runner.js +6 -5
- package/dist/runtime/actor-ids.d.ts +1 -0
- package/dist/runtime/agent-node/agent-node.js +3 -2
- package/dist/runtime/agent-node/output-finalization.js +3 -2
- package/dist/runtime/agent-node/prompt-rendering.js +2 -1
- package/dist/runtime/agent-node/session-execution.js +17 -9
- package/dist/runtime/builtins/builtins.js +10 -4
- package/dist/runtime/changed-files/changed-files.js +5 -2
- package/dist/runtime/contracts/contracts.d.ts +3 -0
- package/dist/runtime/drain-merge/drain-merge.js +2 -1
- package/dist/runtime/durable-store/durable-store.d.ts +2 -0
- package/dist/runtime/durable-store/postgres/client.js +29 -0
- package/dist/runtime/durable-store/postgres/migrate-substrate.js +6 -4
- package/dist/runtime/durable-store/postgres/postgres-store.js +2 -3
- package/dist/runtime/events/events.js +27 -17
- package/dist/runtime/events/index.js +2 -2
- package/dist/runtime/execution-envelope.js +44 -0
- package/dist/runtime/gates/kinds/acceptance/acceptance.js +2 -1
- package/dist/runtime/handoff.d.ts +2 -1
- package/dist/runtime/hooks/module-hook.js +2 -1
- package/dist/runtime/journal-acquisition.d.ts +2 -1
- package/dist/runtime/json-validation/json-validation.js +2 -1
- package/dist/runtime/launch-plan-args.js +39 -0
- package/dist/runtime/node-execution.js +5 -3
- package/dist/runtime/node-state-store.d.ts +3 -0
- package/dist/runtime/open-pull-request/open-pull-request.js +76 -18
- package/dist/runtime/opencode-runtime.js +32 -10
- package/dist/runtime/opencode-session-executor.js +8 -21
- package/dist/runtime/parallel-node/parallel-node.js +2 -1
- package/dist/runtime/run-journal.d.ts +2 -0
- package/dist/runtime/scheduled-dependencies.d.ts +1 -0
- package/dist/runtime/scheduled-dependencies.js +2 -1
- package/dist/runtime/scheduler.js +5 -3
- package/dist/runtime/services/backlog-service.d.ts +1 -2
- package/dist/runtime/services/kubernetes-argo-service.d.ts +0 -1
- package/dist/runtime/services/kubernetes-jobs-read-service.js +115 -0
- package/dist/runtime/services/kubernetes-jobs-service.d.ts +37 -0
- package/dist/runtime/services/kubernetes-jobs-service.js +45 -0
- package/dist/runtime/services/kubernetes-jobs-watch-service.js +61 -0
- package/dist/runtime/services/kubernetes-lease-service.js +21 -0
- package/dist/runtime/services/mcp-gateway-service.js +63 -15
- package/dist/runtime/services/runner-event-sink-http-service.js +2 -1
- package/dist/runtime/workflow-execution.js +2 -1
- package/dist/runtime/workflow-lifecycle.js +2 -1
- package/dist/runtime/yeet-event-record.js +104 -0
- package/dist/runtime/yeet-executor.js +184 -0
- package/dist/schedule/backlog-context.js +4 -2
- package/dist/schedule/passes/open-pull-request.js +4 -2
- package/dist/schedule/prompts.js +2 -1
- package/dist/schema-boundary.d.ts +4 -1
- package/dist/tickets/apply-ticket-plan.js +2 -1
- package/dist/tickets/backlog-task-store.d.ts +2 -1
- package/dist/tickets/backlog-task-store.js +4 -2
- package/dist/tickets/ticket-graph-dto.d.ts +6 -7
- package/dist/tickets/ticket-graph-dto.js +5 -5
- package/dist/tickets/ticket-graph.d.ts +1 -2
- package/dist/tickets/ticket-plan.d.ts +1 -1
- package/dist/tickets/ticket-selection.d.ts +2 -0
- package/dist/workflow-submit-contract.d.ts +0 -1
- package/docs/adr-moka-postgres-schema-isolation.md +48 -0
- package/docs/pipeline-console-runner-contract.md +10 -6
- package/docs/runtime-test-subsystems.md +200 -0
- package/package.json +3 -36
|
@@ -8,6 +8,7 @@ const REASONING_EFFORTS = [
|
|
|
8
8
|
"xhigh"
|
|
9
9
|
];
|
|
10
10
|
const RUNNER_TYPES = ["opencode", "command"];
|
|
11
|
+
const RUNTIME_EXECUTORS = ["sdk", "yeet"];
|
|
11
12
|
const HOOK_EVENTS = [
|
|
12
13
|
"workflow.start",
|
|
13
14
|
"workflow.success",
|
|
@@ -65,4 +66,4 @@ const DEFAULT_RUNNER_COMMAND_GIT_COMMITTER = {
|
|
|
65
66
|
name: "oisin-bot"
|
|
66
67
|
};
|
|
67
68
|
//#endregion
|
|
68
|
-
export { BUILTIN_GATES, DEFAULT_RUNNER_COMMAND_GIT_COMMITTER, FILESYSTEM_MODES, HOOK_EVENTS, ID_RE, MCP_GATEWAY_BACKEND_LOCALITIES, MCP_GATEWAY_WORKSPACE_PATH_SOURCES, NETWORK_MODES, OUTPUT_FORMATS, PIPELINE_GATEWAY_SERVER_ID, REASONING_EFFORTS, RETRY_REASONS, RUNNER_TYPES, SCHEDULE_BASELINES, SCHEDULE_STRATEGIES, SCHEDULING_ROLES, TOOL_NAMES };
|
|
69
|
+
export { BUILTIN_GATES, DEFAULT_RUNNER_COMMAND_GIT_COMMITTER, FILESYSTEM_MODES, HOOK_EVENTS, ID_RE, MCP_GATEWAY_BACKEND_LOCALITIES, MCP_GATEWAY_WORKSPACE_PATH_SOURCES, NETWORK_MODES, OUTPUT_FORMATS, PIPELINE_GATEWAY_SERVER_ID, REASONING_EFFORTS, RETRY_REASONS, RUNNER_TYPES, RUNTIME_EXECUTORS, SCHEDULE_BASELINES, SCHEDULE_STRATEGIES, SCHEDULING_ROLES, TOOL_NAMES };
|
package/dist/config/schemas.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import
|
|
1
|
+
import "../schema-boundary.js";
|
|
2
|
+
import { GATE_KINDS, HOOK_EVENTS, MCP_GATEWAY_BACKEND_LOCALITIES, MCP_GATEWAY_WORKSPACE_PATH_SOURCES, NODE_KINDS, RUNNER_TYPES, RUNTIME_EXECUTORS, SCHEDULE_BASELINES, SCHEDULING_ROLES } from "./schema/catalog.js";
|
|
2
3
|
import * as Schema from "effect/Schema";
|
|
3
|
-
|
|
4
4
|
//#region src/config/schemas.d.ts
|
|
5
5
|
type PipelineConfigErrorCode = "PIPELINE_CONFIG_LEGACY_UNSUPPORTED" | "PIPELINE_CONFIG_PARSE_ERROR" | "PIPELINE_CONFIG_VALIDATION_ERROR";
|
|
6
6
|
interface PipelineConfigIssue {
|
|
@@ -214,6 +214,9 @@ declare const configSchema: Schema.Struct<{
|
|
|
214
214
|
readonly enabled: Schema.withDecodingDefault<Schema.Boolean, never>;
|
|
215
215
|
readonly token_budget: Schema.withDecodingDefault<Schema.Number, never>;
|
|
216
216
|
}>>;
|
|
217
|
+
readonly runtime: Schema.withDecodingDefault<Schema.Struct<{
|
|
218
|
+
readonly executor: Schema.withDecodingDefault<Schema.Literals<readonly ["sdk", "yeet"]>, never>;
|
|
219
|
+
}>, never>;
|
|
217
220
|
readonly runner_command: Schema.withDecodingDefault<Schema.Struct<{
|
|
218
221
|
readonly environment: Schema.withDecodingDefault<Schema.Struct<{
|
|
219
222
|
readonly setup: Schema.withDecodingDefault<Schema.mutable<Schema.$Array<Schema.Struct<{
|
|
@@ -417,6 +420,7 @@ declare const configSchema: Schema.Struct<{
|
|
|
417
420
|
}>;
|
|
418
421
|
type PipelineConfig = typeof configSchema.Type;
|
|
419
422
|
type RunnerType = (typeof RUNNER_TYPES)[number];
|
|
423
|
+
type RuntimeExecutorKind = (typeof RUNTIME_EXECUTORS)[number];
|
|
420
424
|
type WorkflowNodeKind = (typeof NODE_KINDS)[number];
|
|
421
425
|
type HookEvent = (typeof HOOK_EVENTS)[number];
|
|
422
426
|
type GateKind = (typeof GATE_KINDS)[number];
|
|
@@ -433,4 +437,4 @@ interface PipelineConfigValidationOptions {
|
|
|
433
437
|
allowMissingLintFileReferences?: boolean;
|
|
434
438
|
}
|
|
435
439
|
//#endregion
|
|
436
|
-
export { GateKind, HookEvent, McpGatewayBackendLocality, McpGatewayWorkspacePathSource, PipelineConfig, PipelineConfigError, PipelineConfigErrorCode, PipelineConfigIssue, PipelineConfigParts, PipelineConfigValidationOptions, RunnerType, ScheduleBaseline, SchedulingRole, WorkflowNodeKind, configSchema, workflowSchema };
|
|
440
|
+
export { GateKind, HookEvent, McpGatewayBackendLocality, McpGatewayWorkspacePathSource, PipelineConfig, PipelineConfigError, PipelineConfigErrorCode, PipelineConfigIssue, PipelineConfigParts, PipelineConfigValidationOptions, RunnerType, RuntimeExecutorKind, ScheduleBaseline, SchedulingRole, WorkflowNodeKind, configSchema, workflowSchema };
|
package/dist/config/schemas.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { mutableArray, nonEmptyMutableArray, nonNegativeInteger, positiveInteger, positiveNumber, requiredString, stringRecord, struct, unknownRecord, withDefault } from "../schema-boundary.js";
|
|
2
|
-
import { BUILTIN_GATES, DEFAULT_RUNNER_COMMAND_GIT_COMMITTER, FILESYSTEM_MODES, NETWORK_MODES, OUTPUT_FORMATS, REASONING_EFFORTS, RETRY_REASONS, RUNNER_TYPES, SCHEDULE_BASELINES, SCHEDULE_STRATEGIES, SCHEDULING_ROLES, TOOL_NAMES } from "./schema/catalog.js";
|
|
2
|
+
import { BUILTIN_GATES, DEFAULT_RUNNER_COMMAND_GIT_COMMITTER, FILESYSTEM_MODES, NETWORK_MODES, OUTPUT_FORMATS, REASONING_EFFORTS, RETRY_REASONS, RUNNER_TYPES, RUNTIME_EXECUTORS, SCHEDULE_BASELINES, SCHEDULE_STRATEGIES, SCHEDULING_ROLES, TOOL_NAMES } from "./schema/catalog.js";
|
|
3
3
|
import { mcpGatewaySchema, mcpServerSchema } from "./schema/mcp.js";
|
|
4
4
|
import * as Schema from "effect/Schema";
|
|
5
5
|
//#region src/config/schemas.ts
|
|
@@ -363,6 +363,7 @@ const tokenBudgetSchema = struct({
|
|
|
363
363
|
max_context_pct: withDefault(positiveNumber.check(Schema.isLessThanOrEqualTo(100)), 50),
|
|
364
364
|
model_context_windows: withDefault(strictRecord(positiveInteger), {})
|
|
365
365
|
});
|
|
366
|
+
const runtimeConfigSchema = struct({ executor: withDefault(Schema.Literals(RUNTIME_EXECUTORS), "sdk") });
|
|
366
367
|
const contextHandoffSchema = struct({
|
|
367
368
|
enabled: withDefault(Schema.Boolean, false),
|
|
368
369
|
model: Schema.optional(Schema.String)
|
|
@@ -390,6 +391,7 @@ const pipelineConfigCoreShape = {
|
|
|
390
391
|
orchestrator: Schema.optional(orchestratorSchema),
|
|
391
392
|
parallel_worktrees: Schema.optional(parallelWorktreesSchema),
|
|
392
393
|
repo_map: Schema.optional(repoMapSchema),
|
|
394
|
+
runtime: withDefault(runtimeConfigSchema, { executor: "sdk" }),
|
|
393
395
|
runner_command: withDefault(runnerCommandConfigSchema, {
|
|
394
396
|
environment: {
|
|
395
397
|
setup: [],
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { PipelineConfig, PipelineConfigValidationOptions } from "./schemas.js";
|
|
2
|
-
|
|
3
2
|
//#region src/config/validate.d.ts
|
|
4
3
|
declare const validatePipelineConfig: (rawConfig: PipelineConfig, projectRoot?: string, options?: PipelineConfigValidationOptions) => PipelineConfig;
|
|
5
4
|
//#endregion
|
package/dist/config.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { DEFAULT_OPENCODE_ECOSYSTEM_MANIFEST, OPENCODE_ECOSYSTEM_MANIFEST_PATH, OpenCodeEcosystemManifest, PIPELINE_CONFIG_PATH, PROFILES_CONFIG_PATH, RUNNERS_CONFIG_PATH, parseOpenCodeEcosystemManifest } from "./config/defaults.js";
|
|
2
|
-
import { GateKind, HookEvent, McpGatewayBackendLocality, McpGatewayWorkspacePathSource, PipelineConfig, PipelineConfigError, PipelineConfigErrorCode, PipelineConfigIssue, PipelineConfigParts, PipelineConfigValidationOptions, RunnerType, ScheduleBaseline, SchedulingRole, WorkflowNodeKind, workflowSchema } from "./config/schemas.js";
|
|
2
|
+
import { GateKind, HookEvent, McpGatewayBackendLocality, McpGatewayWorkspacePathSource, PipelineConfig, PipelineConfigError, PipelineConfigErrorCode, PipelineConfigIssue, PipelineConfigParts, PipelineConfigValidationOptions, RunnerType, RuntimeExecutorKind, ScheduleBaseline, SchedulingRole, WorkflowNodeKind, workflowSchema } from "./config/schemas.js";
|
|
3
3
|
import { loadPackagePipelineConfig, loadPipelineConfig, parsePipelineConfigParts, parsePipelineConfigYaml } from "./config/load.js";
|
|
4
4
|
import { validatePipelineConfig } from "./config/validate.js";
|
|
5
|
-
export { DEFAULT_OPENCODE_ECOSYSTEM_MANIFEST, type GateKind, type HookEvent, type McpGatewayBackendLocality, type McpGatewayWorkspacePathSource, OPENCODE_ECOSYSTEM_MANIFEST_PATH, type OpenCodeEcosystemManifest, PIPELINE_CONFIG_PATH, PROFILES_CONFIG_PATH, type PipelineConfig, PipelineConfigError, type PipelineConfigErrorCode, type PipelineConfigIssue, type PipelineConfigParts, type PipelineConfigValidationOptions, RUNNERS_CONFIG_PATH, type RunnerType, type ScheduleBaseline, type SchedulingRole, type WorkflowNodeKind, loadPackagePipelineConfig, loadPipelineConfig, parseOpenCodeEcosystemManifest, parsePipelineConfigParts, parsePipelineConfigYaml, validatePipelineConfig, workflowSchema };
|
|
5
|
+
export { DEFAULT_OPENCODE_ECOSYSTEM_MANIFEST, type GateKind, type HookEvent, type McpGatewayBackendLocality, type McpGatewayWorkspacePathSource, OPENCODE_ECOSYSTEM_MANIFEST_PATH, type OpenCodeEcosystemManifest, PIPELINE_CONFIG_PATH, PROFILES_CONFIG_PATH, type PipelineConfig, PipelineConfigError, type PipelineConfigErrorCode, type PipelineConfigIssue, type PipelineConfigParts, type PipelineConfigValidationOptions, RUNNERS_CONFIG_PATH, type RunnerType, type RuntimeExecutorKind, type ScheduleBaseline, type SchedulingRole, type WorkflowNodeKind, loadPackagePipelineConfig, loadPipelineConfig, parseOpenCodeEcosystemManifest, parsePipelineConfigParts, parsePipelineConfigYaml, validatePipelineConfig, workflowSchema };
|
package/dist/context/repo-map.js
CHANGED
|
@@ -100,7 +100,9 @@ const parseFileTags = (root, file, parser, content) => {
|
|
|
100
100
|
};
|
|
101
101
|
const tagFileEffect = (root, file) => Effect.gen(function* effectBody() {
|
|
102
102
|
const service = yield* RepoIoService;
|
|
103
|
-
|
|
103
|
+
const parser = yield* service.createParser();
|
|
104
|
+
const content = yield* service.readText(file);
|
|
105
|
+
return yield* parseFileTags(root, file, parser, content);
|
|
104
106
|
});
|
|
105
107
|
const definitionKey = (def) => `def:${def.path}#${def.name}#${def.startLine}`;
|
|
106
108
|
const isSeeded = (def, seedNames, artifacts) => {
|
|
@@ -187,7 +189,8 @@ const selectWithinBudget = (ranked, budget, estimateTokens) => {
|
|
|
187
189
|
const buildRepoMapContextEffect = (input) => Effect.gen(function* effectBody() {
|
|
188
190
|
const estimateTokens$1 = input.estimateTokens ?? estimateTokens;
|
|
189
191
|
const files = yield* discoverFilesEffect(input.worktreePath);
|
|
190
|
-
const
|
|
192
|
+
const fileTags = yield* Effect.all(files.map((file) => tagFileEffect(input.worktreePath, file)), { concurrency: "unbounded" });
|
|
193
|
+
const ranked = rankDefinitions(buildGraph(fileTags, input));
|
|
191
194
|
const { context, estimatedTokens, selected } = selectWithinBudget(ranked, input.tokenBudget, estimateTokens$1);
|
|
192
195
|
return {
|
|
193
196
|
budget: input.tokenBudget,
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
import { reconcileAllRuns } from "../remote/jobs/reconcile.js";
|
|
2
|
+
import { createRunSpecRegistry, startRunSpecServer } from "../remote/jobs/run-spec-server.js";
|
|
3
|
+
import { runJobFleetWatchLoop } from "../remote/jobs/watch.js";
|
|
4
|
+
import { submitParsedMoka } from "../remote/submit/service.js";
|
|
5
|
+
import { withRunControlStore } from "../run-control/command-context.js";
|
|
6
|
+
import { kubernetesJobFleetReadApi } from "../runtime/services/kubernetes-jobs-read-service.js";
|
|
7
|
+
import { kubernetesJobFleetWatchApi } from "../runtime/services/kubernetes-jobs-watch-service.js";
|
|
8
|
+
import { loadCoordinatorJobsConfig } from "./config.js";
|
|
9
|
+
import { createRunScopedOwnerReference } from "./run-owner.js";
|
|
10
|
+
import { submitWaveJobs } from "./wave-advance.js";
|
|
11
|
+
import * as DateTime from "effect/DateTime";
|
|
12
|
+
import * as Effect from "effect/Effect";
|
|
13
|
+
import * as ManagedRuntime from "effect/ManagedRuntime";
|
|
14
|
+
import * as Layer from "effect/Layer";
|
|
15
|
+
import * as Cause from "effect/Cause";
|
|
16
|
+
import { randomBytes } from "node:crypto";
|
|
17
|
+
import * as Fiber from "effect/Fiber";
|
|
18
|
+
//#region src/coordinator/composition.ts
|
|
19
|
+
/**
|
|
20
|
+
* ENG-57.1/57.2/57.3: the coordinator's composition root (doc-1 decision 3,
|
|
21
|
+
* ENG-56/57). Wires the coordinator's own {@link RunSpecRegistry} + HTTP server,
|
|
22
|
+
* its Jobs transport (`submitWaveJobs`, ENG-57.2's terminal-gated wave scheduler
|
|
23
|
+
* — the one scheduler both static and dynamic schedules share) from a validated
|
|
24
|
+
* {@link loadCoordinatorJobsConfig} config, and (ENG-57.3) the always-finalize
|
|
25
|
+
* Job/pod watch loop (`runJobFleetWatchLoop`, ENG-19) over the durable
|
|
26
|
+
* {@link RunControlStore} the ENG-19 reconciler already understands. `submit`
|
|
27
|
+
* populates `jobsTransport` on every call, so a compiled (static) run submits
|
|
28
|
+
* through Jobs waves. A dynamic run's planner stays on the Argo path (ENG-57.2
|
|
29
|
+
* Option B, see `../remote/submit/service`) until ENG-57.8 moves it onto Jobs.
|
|
30
|
+
*
|
|
31
|
+
* ENG-57.3: the watch loop is started as its own fiber (`Effect.runFork`) so it
|
|
32
|
+
* runs the coordinator's whole lifetime alongside the RunSpec HTTP server,
|
|
33
|
+
* rather than blocking `createCoordinator`. `runJobFleetWatchLoop` itself
|
|
34
|
+
* reconciles once on startup (before any watch event can arrive), on every
|
|
35
|
+
* Job/pod change, and after every reconnect — so coordinator restart, watch
|
|
36
|
+
* loss, and normal operation all drive the same reconcile path (AC#1/AC#3). The
|
|
37
|
+
* fiber is interrupted from `close()`, which — because the loop's store
|
|
38
|
+
* resolution runs inside `Effect.scoped` (`withRunControlStore` /
|
|
39
|
+
* `withRunControlStoreScoped`) — also releases the store's connection in the
|
|
40
|
+
* same step. A permanent loop failure (reconnects exhausted) is never silent:
|
|
41
|
+
* it is logged and, by default, treated as fatal for the process so an
|
|
42
|
+
* orchestrator (k8s liveness) restarts the coordinator, which reconciles again
|
|
43
|
+
* on the next startup.
|
|
44
|
+
*/
|
|
45
|
+
/** Coordinator process env vars — all optional, all default to existing
|
|
46
|
+
* loopback/ephemeral-port behaviour so local dev and tests are unaffected;
|
|
47
|
+
* the chart Deployment sets these explicitly for cluster reachability. */
|
|
48
|
+
const RUN_SPEC_HOST_ENV = "COORDINATOR_RUN_SPEC_HOST";
|
|
49
|
+
const RUN_SPEC_PORT_ENV = "COORDINATOR_RUN_SPEC_PORT";
|
|
50
|
+
const RUN_SPEC_BASE_URL_ENV = "COORDINATOR_RUN_SPEC_BASE_URL";
|
|
51
|
+
const RUN_SPEC_TOKEN_ENV = "COORDINATOR_RUN_SPEC_TOKEN";
|
|
52
|
+
var CoordinatorRunSpecPortEnvError = class extends Error {
|
|
53
|
+
constructor(value) {
|
|
54
|
+
super(`${RUN_SPEC_PORT_ENV}=${JSON.stringify(value)} is not a positive integer port (1-65535)`);
|
|
55
|
+
this.name = "CoordinatorRunSpecPortEnvError";
|
|
56
|
+
}
|
|
57
|
+
};
|
|
58
|
+
const parseRunSpecPortEnv = (env) => {
|
|
59
|
+
const raw = env[RUN_SPEC_PORT_ENV];
|
|
60
|
+
if (raw === void 0 || raw.length === 0) return;
|
|
61
|
+
const port = Number(raw);
|
|
62
|
+
if (!Number.isInteger(port) || port <= 0 || port > 65535) throw new CoordinatorRunSpecPortEnvError(raw);
|
|
63
|
+
return port;
|
|
64
|
+
};
|
|
65
|
+
const generateFallbackRunId = () => `run-${randomBytes(8).toString("hex")}`;
|
|
66
|
+
/** ENG-57.3: boundary-bridge runtime for the coordinator's watch-loop fiber —
|
|
67
|
+
* mirrors `wave-advance.ts`'s `waveRuntime`/`run-owner.ts`'s `leaseRuntime`.
|
|
68
|
+
* `Effect.runFork`/`Effect.runPromise` are reserved for this one composition-root
|
|
69
|
+
* seam; domain code never calls them directly. */
|
|
70
|
+
const watchLoopRuntime = ManagedRuntime.make(Layer.empty);
|
|
71
|
+
/** ENG-57.3 default `onWatchLoopFailure`: a permanently-failed watch loop is
|
|
72
|
+
* never left running silently — log it and exit so an orchestrator restarts
|
|
73
|
+
* the coordinator (which reconciles again on the next startup). */
|
|
74
|
+
const defaultOnWatchLoopFailure = (error) => {
|
|
75
|
+
console.error(`coordinator: job fleet watch loop failed permanently: ${String(error)}`);
|
|
76
|
+
process.exit(1);
|
|
77
|
+
};
|
|
78
|
+
/** ENG-57.3: builds the always-finalize watch-loop effect for one resolved
|
|
79
|
+
* {@link RunControlStore} — the same reconcile path (`reconcileAllRuns`, ENG-19)
|
|
80
|
+
* drives the startup pass, every event-triggered pass, and every
|
|
81
|
+
* post-reconnect pass, so dynamic and static Jobs runs finalize through one
|
|
82
|
+
* run-id-keyed path (AC#4). */
|
|
83
|
+
const buildWatchLoopEffect = (dependencies, config, store) => runJobFleetWatchLoop({
|
|
84
|
+
...dependencies.watchLoop,
|
|
85
|
+
namespace: config.namespace,
|
|
86
|
+
reconcile: reconcileAllRuns({
|
|
87
|
+
fleet: dependencies.jobFleetReadApi ?? kubernetesJobFleetReadApi({}),
|
|
88
|
+
namespace: config.namespace,
|
|
89
|
+
now: () => DateTime.formatIso(DateTime.nowUnsafe()),
|
|
90
|
+
store
|
|
91
|
+
}).pipe(Effect.tap((reports) => Effect.sync(() => dependencies.onReconcile?.(reports))), Effect.asVoid),
|
|
92
|
+
watch: dependencies.jobFleetWatchApi ?? kubernetesJobFleetWatchApi({})
|
|
93
|
+
});
|
|
94
|
+
/** ENG-57.3: starts the watch loop as its own fiber over the coordinator's
|
|
95
|
+
* lifetime. Resolves the store via the injected test override, or the
|
|
96
|
+
* production `withRunControlStore` seam (Postgres-backed, scoped so
|
|
97
|
+
* interrupting the returned fiber also releases the store's connection). A
|
|
98
|
+
* permanent failure is reported via `onWatchLoopFailure`, never left silent. */
|
|
99
|
+
const startWatchLoop = (dependencies, config) => {
|
|
100
|
+
const onWatchLoopFailure = dependencies.onWatchLoopFailure ?? defaultOnWatchLoopFailure;
|
|
101
|
+
const watchLoopEffect = dependencies.runControlStore === void 0 ? withRunControlStore((store) => buildWatchLoopEffect(dependencies, config, store)) : buildWatchLoopEffect(dependencies, config, dependencies.runControlStore);
|
|
102
|
+
return watchLoopRuntime.runFork(watchLoopEffect.pipe(Effect.tapCause((cause) => Effect.sync(() => onWatchLoopFailure(Cause.squash(cause))))));
|
|
103
|
+
};
|
|
104
|
+
const resolveRunSpecBind = (dependencies, env) => {
|
|
105
|
+
const host = dependencies.runSpecBind?.host ?? env[RUN_SPEC_HOST_ENV];
|
|
106
|
+
const port = dependencies.runSpecBind?.port ?? parseRunSpecPortEnv(env);
|
|
107
|
+
return {
|
|
108
|
+
...host === void 0 ? {} : { host },
|
|
109
|
+
...port === void 0 ? {} : { port }
|
|
110
|
+
};
|
|
111
|
+
};
|
|
112
|
+
const resolveExpectedToken = (dependencies, env) => {
|
|
113
|
+
const token = dependencies.expectedToken ?? env[RUN_SPEC_TOKEN_ENV];
|
|
114
|
+
return token === void 0 || token.length === 0 ? void 0 : token;
|
|
115
|
+
};
|
|
116
|
+
/** AC#1: fills in the coordinator's own defaults ONLY where the caller's
|
|
117
|
+
* options left them unset — mirrors how `sharedJobOptions`
|
|
118
|
+
* (`../remote/submit/jobs-submission`) reads `imagePullSecretName` /
|
|
119
|
+
* `serviceAccountName` straight off `options`. */
|
|
120
|
+
const resolveSubmitOptions = (options, config) => ({
|
|
121
|
+
...options,
|
|
122
|
+
...options.imagePullSecretName === void 0 && config.imagePullSecretName !== void 0 ? { imagePullSecretName: config.imagePullSecretName } : {},
|
|
123
|
+
...options.serviceAccountName === void 0 && config.serviceAccountName !== void 0 ? { serviceAccountName: config.serviceAccountName } : {}
|
|
124
|
+
});
|
|
125
|
+
const createCoordinator = async (dependencies = {}) => {
|
|
126
|
+
const env = dependencies.env ?? process.env;
|
|
127
|
+
const config = loadCoordinatorJobsConfig(dependencies.homeDir);
|
|
128
|
+
const registry = createRunSpecRegistry();
|
|
129
|
+
const expectedToken = resolveExpectedToken(dependencies, env);
|
|
130
|
+
const runSpecServer = await startRunSpecServer({
|
|
131
|
+
...expectedToken === void 0 ? {} : { expectedToken },
|
|
132
|
+
registry
|
|
133
|
+
}, resolveRunSpecBind(dependencies, env));
|
|
134
|
+
const runSpecBaseUrl = dependencies.runSpecBaseUrl ?? env[RUN_SPEC_BASE_URL_ENV] ?? runSpecServer.baseUrl;
|
|
135
|
+
const watchLoopFiber = startWatchLoop(dependencies, config);
|
|
136
|
+
const submit = async (options, callerDependencies = {}) => {
|
|
137
|
+
const runId = options.run?.id ?? (dependencies.generateRunId ?? generateFallbackRunId)();
|
|
138
|
+
const ownerReference = await createRunScopedOwnerReference({
|
|
139
|
+
...dependencies.leaseDependencies === void 0 ? {} : { dependencies: dependencies.leaseDependencies },
|
|
140
|
+
namespace: config.namespace,
|
|
141
|
+
runId
|
|
142
|
+
});
|
|
143
|
+
const jobsTransport = {
|
|
144
|
+
envFromSecretNames: config.envFromSecretNames,
|
|
145
|
+
ownerReference,
|
|
146
|
+
registry,
|
|
147
|
+
runSpecBaseUrl,
|
|
148
|
+
submitJobs: submitWaveJobs,
|
|
149
|
+
...dependencies.jobsDependencies === void 0 ? {} : { jobsDependencies: dependencies.jobsDependencies }
|
|
150
|
+
};
|
|
151
|
+
return await submitParsedMoka(resolveSubmitOptions(options, config), {
|
|
152
|
+
...callerDependencies,
|
|
153
|
+
generateRunId: () => runId,
|
|
154
|
+
jobsTransport
|
|
155
|
+
});
|
|
156
|
+
};
|
|
157
|
+
return {
|
|
158
|
+
close: async () => {
|
|
159
|
+
await watchLoopRuntime.runPromise(Fiber.interrupt(watchLoopFiber));
|
|
160
|
+
await runSpecServer.close();
|
|
161
|
+
},
|
|
162
|
+
config,
|
|
163
|
+
registry,
|
|
164
|
+
runSpecServer,
|
|
165
|
+
submit
|
|
166
|
+
};
|
|
167
|
+
};
|
|
168
|
+
//#endregion
|
|
169
|
+
export { CoordinatorRunSpecPortEnvError, createCoordinator };
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import { parseResultWithSchema, requiredString, struct } from "../schema-boundary.js";
|
|
2
|
+
import { PipelineConfigError, configIssuesFromSchemaIssues } from "../config/schemas.js";
|
|
3
|
+
import { ConfigIoService, runConfigIoSync } from "../runtime/services/config-io-service.js";
|
|
4
|
+
import "../config.js";
|
|
5
|
+
import { mokaGlobalConfigPath } from "../moka-global-config.js";
|
|
6
|
+
import * as Effect from "effect/Effect";
|
|
7
|
+
import * as Option$1 from "effect/Option";
|
|
8
|
+
import * as Schema from "effect/Schema";
|
|
9
|
+
//#region src/coordinator/config.ts
|
|
10
|
+
/**
|
|
11
|
+
* ENG-57.1: the coordinator's own narrow slice of the moka global config
|
|
12
|
+
* (`~/.config/moka/config.yaml` inside the coordinator pod, same fixed path
|
|
13
|
+
* {@link mokaGlobalConfigPath} resolves for the laptop `moka submit` CLI).
|
|
14
|
+
* Deliberately narrower than `mokaGlobalConfigSchema` (`../moka-global-config`):
|
|
15
|
+
* the coordinator only ever drives the Jobs transport (doc-1 decision 3), so
|
|
16
|
+
* broker/event-bridge fields (`brokerAuth`, `eventUrl`, `eventAuthSecretName`,
|
|
17
|
+
* ...) that only the Argo path needs are not required here.
|
|
18
|
+
*/
|
|
19
|
+
const coordinatorSubmitConfigSchema = struct({
|
|
20
|
+
gitCredentialsSecretName: requiredString,
|
|
21
|
+
githubAuthSecretName: requiredString,
|
|
22
|
+
imagePullSecretName: Schema.optional(requiredString),
|
|
23
|
+
npmRegistryAuthSecretName: Schema.optional(requiredString),
|
|
24
|
+
serviceAccountName: Schema.optional(requiredString)
|
|
25
|
+
});
|
|
26
|
+
const coordinatorConfigSchema = struct({ momokaya: struct({
|
|
27
|
+
kubernetes: struct({ namespace: requiredString }),
|
|
28
|
+
submit: coordinatorSubmitConfigSchema
|
|
29
|
+
}) });
|
|
30
|
+
const configValidationError = (sourcePath, issues) => new PipelineConfigError("PIPELINE_CONFIG_VALIDATION_ERROR", [`Invalid ${sourcePath}:`, ...issues.map((issue) => issue.path !== void 0 && issue.path !== "" ? `- ${issue.path}: ${issue.message}` : `- ${issue.message}`)].join("\n"), issues);
|
|
31
|
+
const configMissingError = (sourcePath) => new PipelineConfigError("PIPELINE_CONFIG_VALIDATION_ERROR", `Coordinator config not found at ${sourcePath}. Configure momokaya.kubernetes.namespace and momokaya.submit.gitCredentialsSecretName/githubAuthSecretName (Jobs transport prerequisites, doc-1 decision 3) in ${sourcePath}.`, []);
|
|
32
|
+
const noEnvFromSecretsError = (sourcePath) => new PipelineConfigError("PIPELINE_CONFIG_VALIDATION_ERROR", `${sourcePath}: momokaya.submit must configure at least one of gitCredentialsSecretName, githubAuthSecretName, npmRegistryAuthSecretName for the Jobs transport (buildRunnerNodeJobOptionsSchema requires a non-empty envFromSecretNames).`, []);
|
|
33
|
+
const toJobsConfig = (config, sourcePath) => {
|
|
34
|
+
const submit = config.momokaya.submit;
|
|
35
|
+
const envFromSecretNames = [
|
|
36
|
+
submit.gitCredentialsSecretName,
|
|
37
|
+
submit.githubAuthSecretName,
|
|
38
|
+
submit.npmRegistryAuthSecretName
|
|
39
|
+
].filter((name) => name !== void 0);
|
|
40
|
+
if (envFromSecretNames.length === 0) return Effect.fail(noEnvFromSecretsError(sourcePath));
|
|
41
|
+
return Effect.succeed({
|
|
42
|
+
envFromSecretNames,
|
|
43
|
+
namespace: config.momokaya.kubernetes.namespace,
|
|
44
|
+
...submit.imagePullSecretName === void 0 ? {} : { imagePullSecretName: submit.imagePullSecretName },
|
|
45
|
+
...submit.serviceAccountName === void 0 ? {} : { serviceAccountName: submit.serviceAccountName }
|
|
46
|
+
});
|
|
47
|
+
};
|
|
48
|
+
const parseCoordinatorConfigEffect = Effect.fn("parseCoordinatorConfig")(function* (source, sourcePath) {
|
|
49
|
+
const yaml = yield* (yield* ConfigIoService).parseYaml(source, sourcePath);
|
|
50
|
+
const parsed = parseResultWithSchema(coordinatorConfigSchema, yaml, { onExcessProperty: "preserve" });
|
|
51
|
+
if (!parsed.ok) return yield* Effect.fail(configValidationError(sourcePath, configIssuesFromSchemaIssues(parsed.issues)));
|
|
52
|
+
return parsed.value;
|
|
53
|
+
});
|
|
54
|
+
/**
|
|
55
|
+
* Loads + validates the coordinator's config file from disk (the fixed
|
|
56
|
+
* `~/.config/moka/config.yaml` path). AC#1 ("VALIDATED config") / AC#3
|
|
57
|
+
* ("missing Jobs prerequisites -> TYPED config error before run creation"):
|
|
58
|
+
* both a missing file and an empty `envFromSecretNames` list throw a
|
|
59
|
+
* {@link PipelineConfigError} before any RunSpec server or Job gets created.
|
|
60
|
+
*/
|
|
61
|
+
const loadCoordinatorJobsConfig = (homeDir) => {
|
|
62
|
+
const sourcePath = mokaGlobalConfigPath(homeDir);
|
|
63
|
+
return runConfigIoSync(Effect.fn("loadCoordinatorJobsConfig")(function* () {
|
|
64
|
+
const source = yield* (yield* ConfigIoService).readOptionalText(sourcePath);
|
|
65
|
+
if (Option$1.isNone(source)) return yield* Effect.fail(configMissingError(sourcePath));
|
|
66
|
+
const config = yield* parseCoordinatorConfigEffect(source.value, sourcePath);
|
|
67
|
+
return yield* toJobsConfig(config, sourcePath);
|
|
68
|
+
})());
|
|
69
|
+
};
|
|
70
|
+
//#endregion
|
|
71
|
+
export { coordinatorConfigSchema, loadCoordinatorJobsConfig };
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { parseStrictWithSchema, parseWithSchema, requiredString, struct } from "../schema-boundary.js";
|
|
2
|
+
import { jobOwnerReferenceSchema } from "../remote/jobs/model.js";
|
|
3
|
+
import { KubernetesLeaseService, KubernetesLeaseServiceLive } from "../runtime/services/kubernetes-lease-service.js";
|
|
4
|
+
import * as Effect from "effect/Effect";
|
|
5
|
+
import * as ManagedRuntime from "effect/ManagedRuntime";
|
|
6
|
+
import * as Schema from "effect/Schema";
|
|
7
|
+
//#region src/coordinator/run-owner.ts
|
|
8
|
+
/**
|
|
9
|
+
* ENG-57.1: the coordinator's run-scoped ownerReference (doc-1 decision 3).
|
|
10
|
+
* One `coordination.k8s.io/v1` Lease per run; every Job the run submits
|
|
11
|
+
* carries this Lease as its ownerReference, so deleting/expiring the Lease
|
|
12
|
+
* garbage-collects the run's Jobs without a per-run ConfigMap.
|
|
13
|
+
*/
|
|
14
|
+
var RunOwnerLeaseError = class extends Error {
|
|
15
|
+
constructor(runId, cause) {
|
|
16
|
+
super(`Failed to create the run-scoped ownerReference Lease for run "${runId}": ${String(cause)}`);
|
|
17
|
+
this.name = "RunOwnerLeaseError";
|
|
18
|
+
this.cause = cause;
|
|
19
|
+
}
|
|
20
|
+
};
|
|
21
|
+
const leaseRuntime = ManagedRuntime.make(KubernetesLeaseServiceLive);
|
|
22
|
+
const runLeaseEffect = async (effect) => await leaseRuntime.runPromise(effect);
|
|
23
|
+
const createdLeaseSchema = struct({ metadata: struct({
|
|
24
|
+
name: Schema.optional(requiredString),
|
|
25
|
+
namespace: Schema.optional(requiredString),
|
|
26
|
+
uid: Schema.optional(requiredString)
|
|
27
|
+
}) });
|
|
28
|
+
const leaseBody = (runId) => ({
|
|
29
|
+
apiVersion: "coordination.k8s.io/v1",
|
|
30
|
+
kind: "Lease",
|
|
31
|
+
metadata: { name: `run-${runId}` }
|
|
32
|
+
});
|
|
33
|
+
/**
|
|
34
|
+
* Creates the run's ownerReference Lease and returns the validated
|
|
35
|
+
* {@link JobOwnerReference} every submitted Job points at. Wraps both
|
|
36
|
+
* client rejection (e.g. the API server refuses the Lease) and schema
|
|
37
|
+
* validation failure (e.g. the response carries an empty/missing `uid`)
|
|
38
|
+
* in one typed {@link RunOwnerLeaseError} — never a bare throw, never a
|
|
39
|
+
* swallowed failure.
|
|
40
|
+
*/
|
|
41
|
+
const createRunScopedOwnerReference = async (input) => {
|
|
42
|
+
const body = leaseBody(input.runId);
|
|
43
|
+
try {
|
|
44
|
+
const response = await runLeaseEffect(Effect.suspend(() => Effect.gen(function* effectBody() {
|
|
45
|
+
return yield* (yield* KubernetesLeaseService).createLease({
|
|
46
|
+
body,
|
|
47
|
+
dependencies: input.dependencies ?? {},
|
|
48
|
+
namespace: input.namespace,
|
|
49
|
+
options: input.options ?? {}
|
|
50
|
+
});
|
|
51
|
+
})));
|
|
52
|
+
const created = parseWithSchema(createdLeaseSchema, response, { onExcessProperty: "preserve" });
|
|
53
|
+
return parseStrictWithSchema(jobOwnerReferenceSchema, {
|
|
54
|
+
apiVersion: "coordination.k8s.io/v1",
|
|
55
|
+
kind: "Lease",
|
|
56
|
+
name: created.metadata.name ?? body.metadata?.name ?? `run-${input.runId}`,
|
|
57
|
+
uid: created.metadata.uid
|
|
58
|
+
});
|
|
59
|
+
} catch (error) {
|
|
60
|
+
throw new RunOwnerLeaseError(input.runId, error);
|
|
61
|
+
}
|
|
62
|
+
};
|
|
63
|
+
//#endregion
|
|
64
|
+
export { RunOwnerLeaseError, createRunScopedOwnerReference };
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
import "../schema-boundary.js";
|
|
2
|
+
import { compileScheduleArtifact, parseScheduleArtifact } from "../planning/generate.js";
|
|
3
|
+
import "../remote/jobs/job-outcome.js";
|
|
4
|
+
import { submitRunnerNodeJobs } from "../remote/jobs/submit.js";
|
|
5
|
+
import { parseJobsSubmitResult } from "../remote/submit/jobs-submit-result.js";
|
|
6
|
+
import { defaultBuildNodeRunSpec, requireCompiledSchedule, requireSubmitNamespace, sharedJobOptions } from "../remote/submit/jobs-submission.js";
|
|
7
|
+
import { inMemoryExecutionStore } from "../run-control/execution-store.js";
|
|
8
|
+
import { planNextWave } from "./wave-scheduler.js";
|
|
9
|
+
import * as DateTime from "effect/DateTime";
|
|
10
|
+
import * as Effect from "effect/Effect";
|
|
11
|
+
import * as ManagedRuntime from "effect/ManagedRuntime";
|
|
12
|
+
import * as Option$1 from "effect/Option";
|
|
13
|
+
import * as R from "effect/Record";
|
|
14
|
+
import * as Schema from "effect/Schema";
|
|
15
|
+
import * as Arr from "effect/Array";
|
|
16
|
+
import * as HashMap from "effect/HashMap";
|
|
17
|
+
import * as Layer from "effect/Layer";
|
|
18
|
+
import * as Str from "effect/String";
|
|
19
|
+
//#region src/coordinator/wave-advance.ts
|
|
20
|
+
/**
|
|
21
|
+
* ENG-57.2: the wave-advance orchestrator — the single scheduler both static and
|
|
22
|
+
* dynamic (post-planner) schedules run through. It is a re-entrant TICK: given
|
|
23
|
+
* the run's full node list and the durable {@link ExecutionStore}, it decides the
|
|
24
|
+
* next wave with the pure {@link planNextWave} state machine, terminalizes
|
|
25
|
+
* dependents of failed predecessors, and submits ONLY the currently ready nodes
|
|
26
|
+
* through ENG-57.1's create-or-observe single-node primitive (generation-fenced
|
|
27
|
+
* by ENG-57.6). Re-running the tick with unchanged inputs is a no-op: the
|
|
28
|
+
* ExecutionStore CAS and the Job create-or-observe make it idempotent across
|
|
29
|
+
* duplicate callbacks, repeated reconciliation, and coordinator restart.
|
|
30
|
+
*
|
|
31
|
+
* The FIRST tick runs at submit time ({@link submitWaveJobs}) over an empty store
|
|
32
|
+
* → the root wave. Later ticks are driven by the reconcile loop through
|
|
33
|
+
* {@link runWaveTick}, which first folds cluster truth into the ExecutionStore.
|
|
34
|
+
*/
|
|
35
|
+
/**
|
|
36
|
+
* All first-time node submissions in this ticket share execution generation 0:
|
|
37
|
+
* ENG-57.2 CONSUMES execution generations (ENG-57.6) but does not manage
|
|
38
|
+
* semantic retries, so every node's first (and only) execution here is g0.
|
|
39
|
+
*/
|
|
40
|
+
const FIRST_EXECUTION_GENERATION = 0;
|
|
41
|
+
/** {@link submitWaveJobs} is the `SubmitMokaJobsTransport.submitJobs` seam itself
|
|
42
|
+
* (a Promise-returning transport), so it is the boundary that bridges the Effect
|
|
43
|
+
* world back to Promise — same pattern as run-owner's `leaseRuntime` and
|
|
44
|
+
* remote/jobs/submit's `jobsRuntime`. `advanceWave` needs no services. */
|
|
45
|
+
const waveRuntime = ManagedRuntime.make(Layer.empty);
|
|
46
|
+
const runWaveEffect = async (effect) => await waveRuntime.runPromise(effect);
|
|
47
|
+
/** Build the terminal-status map {@link planNextWave} gates on by reading the
|
|
48
|
+
* authoritative record per node from the ExecutionStore (no bulk read exists). */
|
|
49
|
+
const buildTerminalMap = (store, nodes, runId) => Effect.forEach(nodes, (node) => store.getExecution({
|
|
50
|
+
nodeId: node.id,
|
|
51
|
+
runId
|
|
52
|
+
}).pipe(Effect.map((record) => [node.id, record])), { concurrency: 1 }).pipe(Effect.map((entries) => HashMap.fromIterable(entries.flatMap(([nodeId, record]) => Arr.map(Arr.fromOption(record), (stored) => [nodeId, stored.status])))));
|
|
53
|
+
/** CAS-terminalize one node to `blocked` (a failed predecessor cut it off). The
|
|
54
|
+
* store's compare-and-set makes a repeat write idempotent, never a regression. */
|
|
55
|
+
const recordBlocked = (deps, nodeId) => deps.executionStore.recordTerminal({
|
|
56
|
+
at: deps.now(),
|
|
57
|
+
attempt: 1,
|
|
58
|
+
identity: {
|
|
59
|
+
generation: FIRST_EXECUTION_GENERATION,
|
|
60
|
+
nodeId,
|
|
61
|
+
runId: deps.runId
|
|
62
|
+
},
|
|
63
|
+
status: "blocked"
|
|
64
|
+
}).pipe(Effect.asVoid);
|
|
65
|
+
/**
|
|
66
|
+
* One scheduler tick: plan the next wave, terminalize newly-blocked dependents,
|
|
67
|
+
* and submit the ready nodes. Pure of k8s/clock specifics — those arrive through
|
|
68
|
+
* {@link AdvanceWaveDeps.submitReadyNodes} and {@link AdvanceWaveDeps.now}.
|
|
69
|
+
*/
|
|
70
|
+
const advanceWave = (deps) => Effect.gen(function* body() {
|
|
71
|
+
const terminalStatus = yield* buildTerminalMap(deps.executionStore, deps.nodes, deps.runId);
|
|
72
|
+
const plan = planNextWave(deps.nodes, terminalStatus);
|
|
73
|
+
yield* Effect.forEach(plan.blocked, (nodeId) => recordBlocked(deps, nodeId), { concurrency: 1 });
|
|
74
|
+
return {
|
|
75
|
+
plan,
|
|
76
|
+
submitted: yield* Arr.match(plan.ready, {
|
|
77
|
+
onEmpty: () => Effect.succeed([]),
|
|
78
|
+
onNonEmpty: (ready) => deps.submitReadyNodes(ready, FIRST_EXECUTION_GENERATION)
|
|
79
|
+
})
|
|
80
|
+
};
|
|
81
|
+
});
|
|
82
|
+
/** A wave-submit requested a node id absent from the compiled schedule — a
|
|
83
|
+
* scheduler invariant violation, never expected in normal operation. */
|
|
84
|
+
var WaveSubmitMissingNodeError = class extends Schema.TaggedErrorClass()("WaveSubmitMissingNodeError", { message: Schema.String }) {
|
|
85
|
+
constructor(nodeId, runId) {
|
|
86
|
+
super({ message: `wave submit: node ${nodeId} missing from compiled schedule for run ${runId}` });
|
|
87
|
+
}
|
|
88
|
+
};
|
|
89
|
+
/**
|
|
90
|
+
* The `SubmitMokaJobsTransport.submitJobs` seam (ENG-57.1/57.2): submit a run's
|
|
91
|
+
* FIRST wave as per-node Jobs. Compiles the schedule, then runs a single
|
|
92
|
+
* {@link advanceWave} tick over a fresh (empty) ExecutionStore — which resolves
|
|
93
|
+
* to the root wave (nodes with no predecessors), fanning parallel roots out and
|
|
94
|
+
* NEVER eagerly submitting a downstream node. Later waves are the reconcile
|
|
95
|
+
* driver's concern ({@link runWaveTick}). Replaces ENG-57.1's single-ready-node
|
|
96
|
+
* primitive so static and dynamic (post-schedule) runs share this one scheduler.
|
|
97
|
+
*/
|
|
98
|
+
const submitWaveJobs = async (input) => {
|
|
99
|
+
const scheduleYaml = requireCompiledSchedule(input.plan);
|
|
100
|
+
const namespace = requireSubmitNamespace(input.options);
|
|
101
|
+
const nodes = compileScheduleArtifact(input.plan.config, parseScheduleArtifact(scheduleYaml, "schedule.yaml"), input.worktreePath).plan.topologicalOrder;
|
|
102
|
+
const nodeById = HashMap.fromIterable(Arr.map(nodes, (node) => [node.id, node]));
|
|
103
|
+
const buildRunSpec = input.buildRunSpec ?? defaultBuildNodeRunSpec;
|
|
104
|
+
const nodeCwd = input.nodeCwd ?? "/workspace";
|
|
105
|
+
const submitNodeJobs = input.submitNodeJobs ?? submitRunnerNodeJobs;
|
|
106
|
+
const shared = sharedJobOptions(input, namespace);
|
|
107
|
+
const jobsDependencies = input.jobsDependencies ?? {};
|
|
108
|
+
const submitReadyNodes = (nodeIds, generation) => Effect.tryPromise({
|
|
109
|
+
catch: (error) => error,
|
|
110
|
+
try: async () => {
|
|
111
|
+
const specs = R.fromEntries(Arr.map(nodeIds, (id) => {
|
|
112
|
+
const node = Option$1.getOrThrowWith(HashMap.get(nodeById, id), () => new WaveSubmitMissingNodeError(id, input.plan.runId));
|
|
113
|
+
return [id, buildRunSpec({
|
|
114
|
+
context: input.context,
|
|
115
|
+
node,
|
|
116
|
+
nodeCwd,
|
|
117
|
+
plan: input.plan
|
|
118
|
+
})];
|
|
119
|
+
}));
|
|
120
|
+
input.registry.register({
|
|
121
|
+
runId: input.plan.runId,
|
|
122
|
+
specs
|
|
123
|
+
});
|
|
124
|
+
return await submitNodeJobs({
|
|
125
|
+
...shared,
|
|
126
|
+
generation,
|
|
127
|
+
nodeIds: [...nodeIds]
|
|
128
|
+
}, jobsDependencies);
|
|
129
|
+
}
|
|
130
|
+
});
|
|
131
|
+
const result = await runWaveEffect(advanceWave({
|
|
132
|
+
executionStore: inMemoryExecutionStore(),
|
|
133
|
+
nodes,
|
|
134
|
+
now: () => DateTime.formatIso(DateTime.nowUnsafe()),
|
|
135
|
+
runId: input.plan.runId,
|
|
136
|
+
submitReadyNodes
|
|
137
|
+
}));
|
|
138
|
+
return parseJobsSubmitResult({
|
|
139
|
+
jobs: [...result.submitted],
|
|
140
|
+
namespace,
|
|
141
|
+
runId: input.plan.runId,
|
|
142
|
+
waves: [{ nodeIds: [...result.plan.ready] }],
|
|
143
|
+
workflowName: input.plan.workflowId,
|
|
144
|
+
...Str.isEmpty(input.ownerReference.uid) ? {} : { workflowUid: input.ownerReference.uid }
|
|
145
|
+
});
|
|
146
|
+
};
|
|
147
|
+
//#endregion
|
|
148
|
+
export { WaveSubmitMissingNodeError, advanceWave, submitWaveJobs };
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import * as Option$1 from "effect/Option";
|
|
2
|
+
import * as Arr from "effect/Array";
|
|
3
|
+
import * as HashMap from "effect/HashMap";
|
|
4
|
+
import * as Result from "effect/Result";
|
|
5
|
+
import * as HashSet from "effect/HashSet";
|
|
6
|
+
//#region src/coordinator/wave-scheduler.ts
|
|
7
|
+
/**
|
|
8
|
+
* ENG-57.2: the pure terminal-gated wave state machine shared by static and
|
|
9
|
+
* dynamic schedules. Given the full node list and the current terminal-status
|
|
10
|
+
* map (sourced from `ExecutionStore`), decides which nodes are ready to
|
|
11
|
+
* submit THIS tick, which are permanently blocked by a failed/terminal-failed
|
|
12
|
+
* predecessor, and whether the whole run is complete.
|
|
13
|
+
*
|
|
14
|
+
* No IO, no clock, no k8s client — a node's "already submitted" state lives
|
|
15
|
+
* outside this module (derived from cluster observation), so calling this
|
|
16
|
+
* repeatedly with the same inputs is always safe and side-effect free.
|
|
17
|
+
*/
|
|
18
|
+
/** Terminal statuses that block dependents rather than releasing them (AC#5). */
|
|
19
|
+
const FAILURE_STATUSES = HashSet.fromIterable([
|
|
20
|
+
"failed",
|
|
21
|
+
"timed_out",
|
|
22
|
+
"aborted",
|
|
23
|
+
"blocked"
|
|
24
|
+
]);
|
|
25
|
+
const statusOf = (terminalStatus, nodeId) => HashMap.get(terminalStatus, nodeId);
|
|
26
|
+
const nodeVerdict = (node, terminalStatus) => {
|
|
27
|
+
if (Option$1.isSome(statusOf(terminalStatus, node.id))) return "settled";
|
|
28
|
+
if (node.needs.some((needId) => Option$1.match(statusOf(terminalStatus, needId), {
|
|
29
|
+
onNone: () => false,
|
|
30
|
+
onSome: (needStatus) => HashSet.has(FAILURE_STATUSES, needStatus)
|
|
31
|
+
}))) return "blocked";
|
|
32
|
+
return node.needs.every((needId) => Option$1.match(statusOf(terminalStatus, needId), {
|
|
33
|
+
onNone: () => false,
|
|
34
|
+
onSome: (needStatus) => needStatus === "passed"
|
|
35
|
+
})) ? "ready" : "waiting";
|
|
36
|
+
};
|
|
37
|
+
const planNextWave = (nodes, terminalStatus) => {
|
|
38
|
+
const verdicts = Arr.map(nodes, (node) => ({
|
|
39
|
+
id: node.id,
|
|
40
|
+
verdict: nodeVerdict(node, terminalStatus)
|
|
41
|
+
}));
|
|
42
|
+
const ready = Arr.filterMap(verdicts, (v) => v.verdict === "ready" ? Result.succeed(v.id) : Result.failVoid);
|
|
43
|
+
return {
|
|
44
|
+
blocked: Arr.filterMap(verdicts, (v) => v.verdict === "blocked" ? Result.succeed(v.id) : Result.failVoid),
|
|
45
|
+
ready,
|
|
46
|
+
runComplete: nodes.every((node) => Option$1.isSome(statusOf(terminalStatus, node.id)))
|
|
47
|
+
};
|
|
48
|
+
};
|
|
49
|
+
//#endregion
|
|
50
|
+
export { planNextWave };
|