@oisincoveney/pipeline 1.9.0 → 1.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -9
- package/defaults/install-manifest.json +2 -39
- package/dist/config.js +4 -3
- package/dist/index.js +16 -2
- package/dist/install-commands.js +5 -22
- package/dist/kubernetes-runner.js +134 -0
- package/dist/mcp/bootstrap.js +237 -0
- package/dist/mcp/launch-plan.js +125 -0
- package/dist/mcp/native-config.js +23 -0
- package/dist/path-refs.js +11 -0
- package/dist/pipeline-init.js +57 -373
- package/dist/pipeline-runtime.js +3 -2
- package/dist/runner-event-sink.js +120 -0
- package/dist/runner-job-contract.js +273 -0
- package/dist/runner.js +23 -118
- package/docs/mcp-gateway.md +91 -0
- package/docs/mcp-host-isolation.md +39 -0
- package/docs/operator-guide.md +88 -0
- package/docs/pipeline-console-runner-contract.md +83 -0
- package/package.json +2 -1
package/README.md
CHANGED
|
@@ -10,7 +10,7 @@ artifacts.
|
|
|
10
10
|
|
|
11
11
|
- Bun 1.1 or newer
|
|
12
12
|
- Node.js 22.13 or newer
|
|
13
|
-
- `npx`, `backlog`, `uvx`, and Docker on `PATH` for default
|
|
13
|
+
- `npx`, `backlog`, `uvx`, and Docker on `PATH` for default skills and MCP setup
|
|
14
14
|
- At least one configured runner CLI on `PATH`: `codex`, `claude`,
|
|
15
15
|
`opencode`, `kimi`, `pi`, or a declared command runner
|
|
16
16
|
|
|
@@ -28,11 +28,13 @@ Scaffold the default YAML workflow:
|
|
|
28
28
|
pipe init
|
|
29
29
|
```
|
|
30
30
|
|
|
31
|
-
`pipe init` installs default skills with
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
31
|
+
`pipe init` installs default project skills with
|
|
32
|
+
`npx skills add oisincoveney/skills` and registers default MCP servers with the
|
|
33
|
+
MCPM CLI from https://mcpm.sh/. Default profiles point at the installed
|
|
34
|
+
`.agents/skills/<skill>/SKILL.md` files in the target repository. The package
|
|
35
|
+
invokes MCPM through `uvx --python 3.12 mcpm`, so generated `.mcp.json` entries
|
|
36
|
+
do not depend on a globally installed `mcpm` binary. The default Qdrant/memory
|
|
37
|
+
MCP is the Momokaya remote endpoint
|
|
36
38
|
`https://memory-mcp.momokaya.ee/mcp/`.
|
|
37
39
|
|
|
38
40
|
The default GitHub MCP registration uses GitHub's official container in
|
|
@@ -107,6 +109,21 @@ pipe "Implement PIPE-123 user-facing behavior"
|
|
|
107
109
|
Use `PIPELINE_TARGET_PATH=/path/to/worktree` when invoking from outside the
|
|
108
110
|
target repository.
|
|
109
111
|
|
|
112
|
+
## Pipeline Console Runner Image
|
|
113
|
+
|
|
114
|
+
`oisin-pipeline` is also the runner package/image used by `pipeline-console`.
|
|
115
|
+
The console owns Kubernetes Job creation, run listing, cancellation, event
|
|
116
|
+
storage, Kueue discovery, and UI rendering. This package owns the in-container
|
|
117
|
+
`runner-job` command: payload validation, existing runtime invocation, event
|
|
118
|
+
translation, authenticated event posting, signal cancellation, and final event
|
|
119
|
+
flushing.
|
|
120
|
+
|
|
121
|
+
The console starts the image with `OISIN_PIPELINE_RUNNER_PAYLOAD_JSON` and the
|
|
122
|
+
runner-side event token. The payload contract is documented in
|
|
123
|
+
[`docs/pipeline-console-runner-contract.md`](docs/pipeline-console-runner-contract.md).
|
|
124
|
+
Use `PIPELINE_TARGET_PATH=/path/to/worktree` when the checked-out target repo is
|
|
125
|
+
mounted somewhere other than the process working directory.
|
|
126
|
+
|
|
110
127
|
## Minimal YAML
|
|
111
128
|
|
|
112
129
|
`.pipeline/runners.yaml`:
|
|
@@ -259,9 +276,10 @@ branches share a base SHA, and merges passing branches into an integration
|
|
|
259
276
|
branch in declaration order. It reports merge conflicts; it does not resolve
|
|
260
277
|
them automatically.
|
|
261
278
|
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
279
|
+
Default profile skills are installed into `.agents/skills` by `pipe init`.
|
|
280
|
+
Runtime MCP projection and host-specific isolation policy live in `src/mcp`; see
|
|
281
|
+
[`docs/mcp-host-isolation.md`](docs/mcp-host-isolation.md) and
|
|
282
|
+
[`docs/mcp-gateway.md`](docs/mcp-gateway.md).
|
|
265
283
|
|
|
266
284
|
## Generated Host Resources
|
|
267
285
|
|
|
@@ -2,45 +2,8 @@
|
|
|
2
2
|
"version": 1,
|
|
3
3
|
"skills": [
|
|
4
4
|
{
|
|
5
|
-
"source": "
|
|
6
|
-
"
|
|
7
|
-
"using-superpowers",
|
|
8
|
-
"writing-plans",
|
|
9
|
-
"dispatching-parallel-agents",
|
|
10
|
-
"requesting-code-review",
|
|
11
|
-
"receiving-code-review",
|
|
12
|
-
"verification-before-completion"
|
|
13
|
-
]
|
|
14
|
-
},
|
|
15
|
-
{
|
|
16
|
-
"source": "addyosmani/agent-skills",
|
|
17
|
-
"skills": [
|
|
18
|
-
"context-engineering",
|
|
19
|
-
"source-driven-development",
|
|
20
|
-
"spec-driven-development",
|
|
21
|
-
"planning-and-task-breakdown",
|
|
22
|
-
"test-driven-development",
|
|
23
|
-
"incremental-implementation",
|
|
24
|
-
"debugging-and-error-recovery",
|
|
25
|
-
"code-review-and-quality",
|
|
26
|
-
"doubt-driven-development",
|
|
27
|
-
"security-and-hardening",
|
|
28
|
-
"performance-optimization",
|
|
29
|
-
"documentation-and-adrs",
|
|
30
|
-
"deprecation-and-migration"
|
|
31
|
-
]
|
|
32
|
-
},
|
|
33
|
-
{
|
|
34
|
-
"source": "cursor/plugins",
|
|
35
|
-
"skills": ["thermo-nuclear-code-quality-review"]
|
|
36
|
-
},
|
|
37
|
-
{
|
|
38
|
-
"source": "trailofbits/skills",
|
|
39
|
-
"skills": ["semgrep", "supply-chain-risk-auditor"]
|
|
40
|
-
},
|
|
41
|
-
{
|
|
42
|
-
"source": "vercel-labs/agent-skills",
|
|
43
|
-
"skills": ["vercel-react-best-practices", "web-design-guidelines"]
|
|
5
|
+
"source": "oisincoveney/skills",
|
|
6
|
+
"args": ["--agent", "codex", "--skill", "*", "--yes", "--copy"]
|
|
44
7
|
}
|
|
45
8
|
],
|
|
46
9
|
"mcps": [
|
package/dist/config.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
|
+
import { resolveFileReference } from "./path-refs.js";
|
|
1
2
|
import { existsSync, readFileSync } from "node:fs";
|
|
2
|
-
import {
|
|
3
|
+
import { join } from "node:path";
|
|
3
4
|
import { parseDocument } from "yaml";
|
|
4
5
|
import { z } from "zod";
|
|
5
6
|
//#region src/config.ts
|
|
@@ -428,7 +429,7 @@ function resolveMcpServerRef(id, ref, projectRoot) {
|
|
|
428
429
|
path: `mcp_servers.${id}.ref.path`,
|
|
429
430
|
message: "MCP server refs require a project root"
|
|
430
431
|
}]);
|
|
431
|
-
const filePath =
|
|
432
|
+
const filePath = resolveFileReference(projectRoot, ref.path);
|
|
432
433
|
if (!existsSync(filePath)) throw validationError([{
|
|
433
434
|
path: `mcp_servers.${id}.ref.path`,
|
|
434
435
|
message: `referenced MCP config file '${ref.path}' does not exist`
|
|
@@ -655,7 +656,7 @@ function validateListCapability(path, requested, supported, label, issues) {
|
|
|
655
656
|
}
|
|
656
657
|
function validatePath(path, value, projectRoot, issues, options = {}) {
|
|
657
658
|
if (!(value && projectRoot)) return;
|
|
658
|
-
if (!existsSync(
|
|
659
|
+
if (!existsSync(resolveFileReference(projectRoot, value))) {
|
|
659
660
|
if (options.allowMissingLintFileReferences && isLintableMissingFileReferencePath(path)) return;
|
|
660
661
|
issues.push({
|
|
661
662
|
path,
|
package/dist/index.js
CHANGED
|
@@ -2,9 +2,11 @@
|
|
|
2
2
|
import { PipelineConfigError, loadPipelineConfig, tryLoadPipelineConfig } from "./config.js";
|
|
3
3
|
import { compileWorkflowPlan } from "./workflow-planner.js";
|
|
4
4
|
import { formatInstallCommandsResult, installCommands, parseCommandHost } from "./install-commands.js";
|
|
5
|
-
import { DEFAULT_MCPM_ARGS, formatPipelineInitResult, initPipelineProject } from "./pipeline-init.js";
|
|
6
5
|
import { createOrchestratorLaunchPlan, createRunnerLaunchPlan } from "./runner.js";
|
|
7
6
|
import { formatConfigError, runPipelineFromConfig } from "./pipeline-runtime.js";
|
|
7
|
+
import { runKubernetesRunnerJob } from "./kubernetes-runner.js";
|
|
8
|
+
import { DEFAULT_MCPM_ARGS } from "./mcp/bootstrap.js";
|
|
9
|
+
import { formatPipelineInitResult, initPipelineProject } from "./pipeline-init.js";
|
|
8
10
|
import { existsSync, realpathSync } from "node:fs";
|
|
9
11
|
import { resolve } from "node:path";
|
|
10
12
|
import { fileURLToPath } from "node:url";
|
|
@@ -146,7 +148,8 @@ const BUILTIN_PIPE_COMMANDS = new Set([
|
|
|
146
148
|
"explain-plan",
|
|
147
149
|
"doctor",
|
|
148
150
|
"init",
|
|
149
|
-
"install-commands"
|
|
151
|
+
"install-commands",
|
|
152
|
+
"runner-job"
|
|
150
153
|
]);
|
|
151
154
|
function createCliProgram() {
|
|
152
155
|
const configuredPipeline = tryLoadPipelineConfig(process.env.PIPELINE_TARGET_PATH ?? process.cwd(), { allowMissingLintFileReferences: true });
|
|
@@ -200,6 +203,10 @@ function createCliProgram() {
|
|
|
200
203
|
});
|
|
201
204
|
console.log(formatInstallCommandsResult(result));
|
|
202
205
|
});
|
|
206
|
+
program.command("runner-job").description("Run an in-pod pipeline runner job from the console payload").action(async () => {
|
|
207
|
+
const exitCode = await runKubernetesRunnerJob();
|
|
208
|
+
process.exitCode = exitCode;
|
|
209
|
+
});
|
|
203
210
|
const configuredEntrypointCommands = registerConfiguredEntrypointCommands(program, configuredPipeline);
|
|
204
211
|
if (configuredEntrypointCommands.size > 0) program.configureHelp({ subcommandTerm(command) {
|
|
205
212
|
if (configuredEntrypointCommands.has(command.name())) return command.name();
|
|
@@ -380,6 +387,10 @@ function normalizeEntrypointPath(path) {
|
|
|
380
387
|
}
|
|
381
388
|
if (isCliEntrypoint(process.argv)) runCli(process.argv).catch((err) => {
|
|
382
389
|
if (err instanceof CommanderError) process.exit(err.exitCode);
|
|
390
|
+
if (hasExitCode(err)) {
|
|
391
|
+
if (err.message) console.error(err.message);
|
|
392
|
+
process.exit(err.exitCode);
|
|
393
|
+
}
|
|
383
394
|
if (err instanceof Error) {
|
|
384
395
|
if (err instanceof PipelineConfigError) console.error(formatConfigError(err));
|
|
385
396
|
else console.error(err.message);
|
|
@@ -388,6 +399,9 @@ if (isCliEntrypoint(process.argv)) runCli(process.argv).catch((err) => {
|
|
|
388
399
|
console.error(String(err));
|
|
389
400
|
process.exit(1);
|
|
390
401
|
});
|
|
402
|
+
function hasExitCode(err) {
|
|
403
|
+
return err instanceof Error && "exitCode" in err && typeof err.exitCode === "number";
|
|
404
|
+
}
|
|
391
405
|
function formatWorkflowPlan(config, worktreePath, workflowId) {
|
|
392
406
|
const plan = compileWorkflowPlan(config, workflowId);
|
|
393
407
|
const workflow = config.workflows[plan.workflowId];
|
package/dist/install-commands.js
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
|
+
import { resolveFileReference } from "./path-refs.js";
|
|
1
2
|
import { loadPipelineConfig } from "./config.js";
|
|
3
|
+
import { codexNativeMcpConfig } from "./mcp/native-config.js";
|
|
2
4
|
import { compileWorkflowPlan } from "./workflow-planner.js";
|
|
3
5
|
import { existsSync, readFileSync } from "node:fs";
|
|
4
6
|
import { dirname, join, relative } from "node:path";
|
|
@@ -402,7 +404,7 @@ function codexDefinitions(config, cwd) {
|
|
|
402
404
|
name: id,
|
|
403
405
|
sandbox_mode: profile.filesystem?.mode === "workspace-write" ? "workspace-write" : "read-only",
|
|
404
406
|
...codexAgentSkillConfig(config, profile, cwd),
|
|
405
|
-
...
|
|
407
|
+
...codexNativeMcpConfig(config, profile)
|
|
406
408
|
}).trimEnd()}\n`,
|
|
407
409
|
host: "codex",
|
|
408
410
|
invocation: invocationForHost("codex"),
|
|
@@ -413,33 +415,14 @@ function codexDefinitions(config, cwd) {
|
|
|
413
415
|
function codexAgentSkillConfig(config, profile, cwd) {
|
|
414
416
|
const skillConfig = (profile.skills ?? []).flatMap((id) => {
|
|
415
417
|
const skillPath = config.skills[id]?.path;
|
|
416
|
-
const
|
|
417
|
-
return
|
|
418
|
+
const resolvedSkillPath = skillPath ? resolveFileReference(cwd, skillPath) : void 0;
|
|
419
|
+
return skillPath && resolvedSkillPath && existsSync(resolvedSkillPath) ? [{
|
|
418
420
|
enabled: true,
|
|
419
421
|
path: skillPath.replaceAll("\\", "/")
|
|
420
422
|
}] : [];
|
|
421
423
|
});
|
|
422
424
|
return skillConfig.length > 0 ? { skills: { config: skillConfig } } : {};
|
|
423
425
|
}
|
|
424
|
-
function codexAgentMcpConfig(config, profile) {
|
|
425
|
-
const mcpServers = Object.fromEntries((profile.mcp_servers ?? []).flatMap((id) => {
|
|
426
|
-
const server = config.mcp_servers[id];
|
|
427
|
-
return server ? [[id, codexAgentMcpServerConfig(server)]] : [];
|
|
428
|
-
}));
|
|
429
|
-
return Object.keys(mcpServers).length > 0 ? { mcp_servers: mcpServers } : {};
|
|
430
|
-
}
|
|
431
|
-
function codexAgentMcpServerConfig(server) {
|
|
432
|
-
if (typeof server.url === "string") return {
|
|
433
|
-
...server.bearer_token_env_var ? { bearer_token_env_var: server.bearer_token_env_var } : {},
|
|
434
|
-
...server.headers ? { http_headers: server.headers } : {},
|
|
435
|
-
url: server.url
|
|
436
|
-
};
|
|
437
|
-
return {
|
|
438
|
-
...server.args ? { args: server.args } : {},
|
|
439
|
-
command: server.command,
|
|
440
|
-
...server.env ? { env: server.env } : {}
|
|
441
|
-
};
|
|
442
|
-
}
|
|
443
426
|
function kimiDefinitions(config) {
|
|
444
427
|
return [
|
|
445
428
|
...entrypointCommandDefinitions("kimi", config, (id, entrypoint) => ({
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
import { PipelineConfigError } from "./config.js";
|
|
2
|
+
import { runPipelineFromConfig } from "./pipeline-runtime.js";
|
|
3
|
+
import { RUNNER_PAYLOAD_ENV, parseRunnerJobPayload, resolveRunnerEventSinkAuthToken } from "./runner-job-contract.js";
|
|
4
|
+
import { createRunnerEventSink } from "./runner-event-sink.js";
|
|
5
|
+
//#region src/kubernetes-runner.ts
|
|
6
|
+
const EXIT_PASS = 0;
|
|
7
|
+
const EXIT_FAIL = 1;
|
|
8
|
+
const EXIT_CANCELLED = 130;
|
|
9
|
+
const EXIT_VALIDATION = 64;
|
|
10
|
+
const EXIT_STARTUP = 70;
|
|
11
|
+
async function runKubernetesRunnerJob(options = {}) {
|
|
12
|
+
const prepared = prepareRunnerJob(options);
|
|
13
|
+
if (prepared.exitCode !== void 0) return prepared.exitCode;
|
|
14
|
+
const { authToken, env, payload, stderr } = prepared.job;
|
|
15
|
+
const controller = new AbortController();
|
|
16
|
+
const signalEmitter = options.signalEmitter ?? process;
|
|
17
|
+
const forceExit = options.onForceExit ?? ((exitCode) => process.exit(exitCode));
|
|
18
|
+
let signalExitCode;
|
|
19
|
+
let signalCount = 0;
|
|
20
|
+
let signalFinalResultRecorded = false;
|
|
21
|
+
const sink = createRunnerEventSink({
|
|
22
|
+
authHeader: payload.eventSink.authHeader,
|
|
23
|
+
authToken,
|
|
24
|
+
fetch: options.fetch,
|
|
25
|
+
runId: payload.run.runId,
|
|
26
|
+
url: payload.eventSink.url
|
|
27
|
+
});
|
|
28
|
+
const handleSignal = (exitCode) => {
|
|
29
|
+
signalCount += 1;
|
|
30
|
+
if (signalCount === 1) {
|
|
31
|
+
signalExitCode = exitCode;
|
|
32
|
+
sink.recordCancellation(payload.selector.workflowId);
|
|
33
|
+
signalFinalResultRecorded = true;
|
|
34
|
+
controller.abort();
|
|
35
|
+
return;
|
|
36
|
+
}
|
|
37
|
+
forceExit(signalExitCode ?? exitCode);
|
|
38
|
+
};
|
|
39
|
+
const handleSigterm = () => handleSignal(EXIT_CANCELLED);
|
|
40
|
+
const handleSigint = () => handleSignal(EXIT_CANCELLED);
|
|
41
|
+
signalEmitter.on("SIGTERM", handleSigterm);
|
|
42
|
+
signalEmitter.on("SIGINT", handleSigint);
|
|
43
|
+
const runner = options.pipelineRunner ?? runPipelineFromConfig;
|
|
44
|
+
let sawWorkflowFinish = false;
|
|
45
|
+
try {
|
|
46
|
+
const result = await runner({
|
|
47
|
+
reporter: (event) => {
|
|
48
|
+
if (event.type === "workflow.finish") sawWorkflowFinish = true;
|
|
49
|
+
sink.recordRuntimeEvent(event);
|
|
50
|
+
},
|
|
51
|
+
runId: payload.run.runId,
|
|
52
|
+
signal: controller.signal,
|
|
53
|
+
task: payload.task.prompt,
|
|
54
|
+
workflowId: payload.selector.workflowId,
|
|
55
|
+
worktreePath: env.PIPELINE_TARGET_PATH ?? options.cwd ?? process.cwd()
|
|
56
|
+
});
|
|
57
|
+
if (!(sawWorkflowFinish || signalFinalResultRecorded)) sink.recordFinalResult(result.outcome, result.plan.workflowId);
|
|
58
|
+
if (await flushAndReport(sink.flush, stderr) && !signalExitCode) return EXIT_STARTUP;
|
|
59
|
+
return signalExitCode ?? exitCodeForRuntimeResult(result);
|
|
60
|
+
} catch (err) {
|
|
61
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
62
|
+
stderr.write(`${message}\n`);
|
|
63
|
+
await flushAndReport(sink.flush, stderr);
|
|
64
|
+
if (err instanceof PipelineConfigError) return signalExitCode ?? EXIT_VALIDATION;
|
|
65
|
+
return signalExitCode ?? EXIT_STARTUP;
|
|
66
|
+
} finally {
|
|
67
|
+
removeSignalListener(signalEmitter, "SIGTERM", handleSigterm);
|
|
68
|
+
removeSignalListener(signalEmitter, "SIGINT", handleSigint);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
function prepareRunnerJob(options) {
|
|
72
|
+
const env = options.env ?? process.env;
|
|
73
|
+
const stderr = options.stderr ?? process.stderr;
|
|
74
|
+
const payloadRaw = env[RUNNER_PAYLOAD_ENV];
|
|
75
|
+
if (!payloadRaw) {
|
|
76
|
+
stderr.write(`${RUNNER_PAYLOAD_ENV} is required\n`);
|
|
77
|
+
return { exitCode: EXIT_VALIDATION };
|
|
78
|
+
}
|
|
79
|
+
const payload = parsePayload(payloadRaw, stderr);
|
|
80
|
+
if (!payload) return { exitCode: EXIT_VALIDATION };
|
|
81
|
+
const authToken = resolveAuthToken(env, stderr);
|
|
82
|
+
if (!authToken) return { exitCode: EXIT_VALIDATION };
|
|
83
|
+
return { job: {
|
|
84
|
+
authToken,
|
|
85
|
+
env,
|
|
86
|
+
payload,
|
|
87
|
+
stderr
|
|
88
|
+
} };
|
|
89
|
+
}
|
|
90
|
+
function parsePayload(payloadRaw, stderr) {
|
|
91
|
+
try {
|
|
92
|
+
return parseRunnerJobPayload(payloadRaw);
|
|
93
|
+
} catch (err) {
|
|
94
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
95
|
+
stderr.write(`${message}\n`);
|
|
96
|
+
return null;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
function resolveAuthToken(env, stderr) {
|
|
100
|
+
try {
|
|
101
|
+
return resolveRunnerEventSinkAuthToken({ env });
|
|
102
|
+
} catch (err) {
|
|
103
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
104
|
+
stderr.write(`${message}\n`);
|
|
105
|
+
return null;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
function exitCodeForRuntimeResult(result) {
|
|
109
|
+
switch (result.outcome) {
|
|
110
|
+
case "PASS": return EXIT_PASS;
|
|
111
|
+
case "FAIL": return EXIT_FAIL;
|
|
112
|
+
case "CANCELLED": return EXIT_CANCELLED;
|
|
113
|
+
default: return EXIT_STARTUP;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
async function flushAndReport(flush, stderr) {
|
|
117
|
+
try {
|
|
118
|
+
await flush();
|
|
119
|
+
return false;
|
|
120
|
+
} catch (err) {
|
|
121
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
122
|
+
stderr.write(`Event sink flush failed: ${message}\n`);
|
|
123
|
+
return true;
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
function removeSignalListener(emitter, event, listener) {
|
|
127
|
+
if (emitter.off) {
|
|
128
|
+
emitter.off(event, listener);
|
|
129
|
+
return;
|
|
130
|
+
}
|
|
131
|
+
emitter.removeListener?.(event, listener);
|
|
132
|
+
}
|
|
133
|
+
//#endregion
|
|
134
|
+
export { runKubernetesRunnerJob };
|
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
import { readFileSync } from "node:fs";
|
|
2
|
+
import { z } from "zod";
|
|
3
|
+
import { execa } from "execa";
|
|
4
|
+
//#region src/mcp/bootstrap.ts
|
|
5
|
+
var PipelineMcpInstallError = class extends Error {
|
|
6
|
+
constructor(message) {
|
|
7
|
+
super(message);
|
|
8
|
+
this.name = "PipelineMcpInstallError";
|
|
9
|
+
}
|
|
10
|
+
};
|
|
11
|
+
var PipelineMcpMissingCredentialError = class extends PipelineMcpInstallError {
|
|
12
|
+
headerName;
|
|
13
|
+
missingEnv;
|
|
14
|
+
serverName;
|
|
15
|
+
constructor(serverName, headerName, missingEnv) {
|
|
16
|
+
super([`MCP server ${serverName} requires ${headerName} credentials before it can be registered.`, `Set ${missingEnv.join(" or ")} and re-run pipeline init.`].join("\n"));
|
|
17
|
+
this.name = "PipelineMcpMissingCredentialError";
|
|
18
|
+
this.serverName = serverName;
|
|
19
|
+
this.headerName = headerName;
|
|
20
|
+
this.missingEnv = missingEnv;
|
|
21
|
+
}
|
|
22
|
+
};
|
|
23
|
+
var PipelineDefaultManifestError = class extends Error {
|
|
24
|
+
constructor(message) {
|
|
25
|
+
super(message);
|
|
26
|
+
this.name = "PipelineDefaultManifestError";
|
|
27
|
+
}
|
|
28
|
+
};
|
|
29
|
+
const DEFAULT_MCPM_ARGS = [
|
|
30
|
+
"--python",
|
|
31
|
+
"3.12",
|
|
32
|
+
"mcpm"
|
|
33
|
+
];
|
|
34
|
+
const DEFAULT_INSTALL_MANIFEST_URL = new URL("../../defaults/install-manifest.json", import.meta.url);
|
|
35
|
+
const pipelineMcpHeaderSourceSchema = z.object({
|
|
36
|
+
env: z.string().min(1),
|
|
37
|
+
prefix: z.string().optional(),
|
|
38
|
+
suffix: z.string().optional()
|
|
39
|
+
}).strict();
|
|
40
|
+
const pipelineMcpHeaderValueSchema = z.union([z.string(), z.object({ sources: z.array(pipelineMcpHeaderSourceSchema).min(1) }).strict()]);
|
|
41
|
+
const pipelineMcpInstallSpecSchema = z.object({
|
|
42
|
+
args: z.array(z.string()).optional(),
|
|
43
|
+
catalog: z.string().min(1).optional(),
|
|
44
|
+
command: z.string().min(1).optional(),
|
|
45
|
+
env: z.record(z.string(), z.string()).optional(),
|
|
46
|
+
headers: z.record(z.string(), pipelineMcpHeaderValueSchema).optional(),
|
|
47
|
+
name: z.string().min(1),
|
|
48
|
+
optionalRegistration: z.boolean().optional(),
|
|
49
|
+
transport: z.enum(["remote", "stdio"]),
|
|
50
|
+
url: z.string().url().optional()
|
|
51
|
+
}).strict().superRefine((spec, ctx) => {
|
|
52
|
+
if (spec.catalog) return;
|
|
53
|
+
if (spec.transport === "remote") {
|
|
54
|
+
if (!spec.url) ctx.addIssue({
|
|
55
|
+
code: "custom",
|
|
56
|
+
message: "remote MCP install spec must declare url or catalog",
|
|
57
|
+
path: ["url"]
|
|
58
|
+
});
|
|
59
|
+
if (spec.command) ctx.addIssue({
|
|
60
|
+
code: "custom",
|
|
61
|
+
message: "remote MCP install spec cannot declare command",
|
|
62
|
+
path: ["command"]
|
|
63
|
+
});
|
|
64
|
+
if (spec.args) ctx.addIssue({
|
|
65
|
+
code: "custom",
|
|
66
|
+
message: "remote MCP install spec cannot declare args",
|
|
67
|
+
path: ["args"]
|
|
68
|
+
});
|
|
69
|
+
if (spec.env) ctx.addIssue({
|
|
70
|
+
code: "custom",
|
|
71
|
+
message: "remote MCP install spec cannot declare env",
|
|
72
|
+
path: ["env"]
|
|
73
|
+
});
|
|
74
|
+
return;
|
|
75
|
+
}
|
|
76
|
+
if (!spec.command) ctx.addIssue({
|
|
77
|
+
code: "custom",
|
|
78
|
+
message: "stdio MCP install spec must declare command or catalog",
|
|
79
|
+
path: ["command"]
|
|
80
|
+
});
|
|
81
|
+
if (spec.headers) ctx.addIssue({
|
|
82
|
+
code: "custom",
|
|
83
|
+
message: "stdio MCP install spec cannot declare headers",
|
|
84
|
+
path: ["headers"]
|
|
85
|
+
});
|
|
86
|
+
if (spec.url) ctx.addIssue({
|
|
87
|
+
code: "custom",
|
|
88
|
+
message: "stdio MCP install spec cannot declare url",
|
|
89
|
+
path: ["url"]
|
|
90
|
+
});
|
|
91
|
+
});
|
|
92
|
+
const defaultInstallManifestSchema = z.object({
|
|
93
|
+
mcps: z.array(pipelineMcpInstallSpecSchema),
|
|
94
|
+
skills: z.array(z.object({
|
|
95
|
+
args: z.array(z.string()).optional(),
|
|
96
|
+
source: z.string().min(1)
|
|
97
|
+
}).strict()).default([]),
|
|
98
|
+
version: z.literal(1)
|
|
99
|
+
}).strict();
|
|
100
|
+
function loadDefaultInstallManifest() {
|
|
101
|
+
const raw = JSON.parse(readFileSync(DEFAULT_INSTALL_MANIFEST_URL, "utf8"));
|
|
102
|
+
const parsed = defaultInstallManifestSchema.safeParse(raw);
|
|
103
|
+
if (!parsed.success) throw new PipelineDefaultManifestError(["Invalid defaults/install-manifest.json.", ...parsed.error.issues.map((issue) => [issue.path.join("."), issue.message].filter(Boolean).join(": "))].join("\n"));
|
|
104
|
+
return parsed.data;
|
|
105
|
+
}
|
|
106
|
+
const DEFAULT_INSTALL_MANIFEST = loadDefaultInstallManifest();
|
|
107
|
+
const DEFAULT_MCP_INSTALLS = DEFAULT_INSTALL_MANIFEST.mcps;
|
|
108
|
+
const DEFAULT_SKILL_INSTALLS = DEFAULT_INSTALL_MANIFEST.skills;
|
|
109
|
+
function defaultMcpJson() {
|
|
110
|
+
return `${JSON.stringify({ mcpServers: Object.fromEntries([
|
|
111
|
+
["backlog", "oisin-pipeline-backlog"],
|
|
112
|
+
["context7", "oisin-pipeline-context7"],
|
|
113
|
+
["github-readonly", "oisin-pipeline-github-readonly"],
|
|
114
|
+
["playwright", "oisin-pipeline-playwright"],
|
|
115
|
+
["qdrant", "oisin-pipeline-qdrant"],
|
|
116
|
+
["semgrep", "oisin-pipeline-semgrep"],
|
|
117
|
+
["serena", "oisin-pipeline-serena"]
|
|
118
|
+
].map(([server, installName]) => [server, {
|
|
119
|
+
args: [
|
|
120
|
+
...DEFAULT_MCPM_ARGS,
|
|
121
|
+
"run",
|
|
122
|
+
installName
|
|
123
|
+
],
|
|
124
|
+
command: "uvx"
|
|
125
|
+
}])) }, null, 2)}\n`;
|
|
126
|
+
}
|
|
127
|
+
async function installDefaultMcpsWithCli(specs, cwd) {
|
|
128
|
+
const skipped = [];
|
|
129
|
+
for (const spec of specs) {
|
|
130
|
+
const install = mcpInstallArgs(spec);
|
|
131
|
+
if ("skipped" in install) {
|
|
132
|
+
skipped.push(install.skipped);
|
|
133
|
+
continue;
|
|
134
|
+
}
|
|
135
|
+
try {
|
|
136
|
+
await execa("uvx", [...DEFAULT_MCPM_ARGS, ...install.args], {
|
|
137
|
+
cwd,
|
|
138
|
+
env: {
|
|
139
|
+
MCPM_FORCE: "true",
|
|
140
|
+
MCPM_JSON_OUTPUT: "true",
|
|
141
|
+
MCPM_NON_INTERACTIVE: "true"
|
|
142
|
+
},
|
|
143
|
+
stdin: "ignore"
|
|
144
|
+
});
|
|
145
|
+
} catch (err) {
|
|
146
|
+
const error = err;
|
|
147
|
+
throw new PipelineMcpInstallError([
|
|
148
|
+
`Failed to register MCP server ${spec.name} with MCPM.`,
|
|
149
|
+
"Pipeline init runs MCPM through `uvx --python 3.12 mcpm`.",
|
|
150
|
+
"Install uv/uvx from https://docs.astral.sh/uv/ and re-run pipeline init.",
|
|
151
|
+
redactMcpInstallOutput(error.shortMessage, install.redactions),
|
|
152
|
+
redactMcpInstallOutput(error.stderr, install.redactions),
|
|
153
|
+
redactMcpInstallOutput(error.stdout, install.redactions)
|
|
154
|
+
].filter(Boolean).join("\n"));
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
return { skipped };
|
|
158
|
+
}
|
|
159
|
+
function mcpInstallArgs(spec) {
|
|
160
|
+
if (spec.catalog) return {
|
|
161
|
+
args: [
|
|
162
|
+
"install",
|
|
163
|
+
spec.catalog,
|
|
164
|
+
"--force",
|
|
165
|
+
"--alias",
|
|
166
|
+
spec.name
|
|
167
|
+
],
|
|
168
|
+
redactions: []
|
|
169
|
+
};
|
|
170
|
+
const args = [
|
|
171
|
+
"new",
|
|
172
|
+
spec.name,
|
|
173
|
+
"--type",
|
|
174
|
+
spec.transport,
|
|
175
|
+
"--force"
|
|
176
|
+
];
|
|
177
|
+
if (spec.transport === "remote") {
|
|
178
|
+
if (!spec.url) throw new PipelineMcpInstallError(`MCP server ${spec.name} is remote but has no url.`);
|
|
179
|
+
const redactions = [];
|
|
180
|
+
try {
|
|
181
|
+
const headers = Object.entries(spec.headers ?? {}).flatMap(([key, value]) => {
|
|
182
|
+
const headerValue = resolveMcpHeaderValue(spec.name, key, value);
|
|
183
|
+
redactions.push(headerValue);
|
|
184
|
+
return ["--headers", `${key}=${headerValue}`];
|
|
185
|
+
});
|
|
186
|
+
return {
|
|
187
|
+
args: [
|
|
188
|
+
...args,
|
|
189
|
+
"--url",
|
|
190
|
+
spec.url,
|
|
191
|
+
...headers
|
|
192
|
+
],
|
|
193
|
+
redactions
|
|
194
|
+
};
|
|
195
|
+
} catch (err) {
|
|
196
|
+
if (spec.optionalRegistration && err instanceof PipelineMcpMissingCredentialError) return { skipped: {
|
|
197
|
+
missingEnv: err.missingEnv,
|
|
198
|
+
name: spec.name,
|
|
199
|
+
reason: `missing ${err.headerName} credentials`
|
|
200
|
+
} };
|
|
201
|
+
throw err;
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
if (!spec.command) throw new PipelineMcpInstallError(`MCP server ${spec.name} is stdio but has no command.`);
|
|
205
|
+
return {
|
|
206
|
+
args: [
|
|
207
|
+
...args,
|
|
208
|
+
"--command",
|
|
209
|
+
spec.command,
|
|
210
|
+
...spec.args?.length ? ["--args", spec.args.join(" ")] : [],
|
|
211
|
+
...Object.entries(spec.env ?? {}).flatMap(([key, value]) => ["--env", `${key}=${value}`])
|
|
212
|
+
],
|
|
213
|
+
redactions: []
|
|
214
|
+
};
|
|
215
|
+
}
|
|
216
|
+
const MCP_CREDENTIAL_PATTERN = /^\S+\s+(.+)$/;
|
|
217
|
+
function redactMcpInstallOutput(value, redactions) {
|
|
218
|
+
if (!value) return value;
|
|
219
|
+
const sensitiveValues = redactions.flatMap((item) => {
|
|
220
|
+
const trimmed = item.trim();
|
|
221
|
+
const credential = trimmed.match(MCP_CREDENTIAL_PATTERN)?.[1]?.trim();
|
|
222
|
+
return credential ? [trimmed, credential] : [trimmed];
|
|
223
|
+
}).filter((item) => item.length > 0);
|
|
224
|
+
const escaped = [...new Set(sensitiveValues)].sort((a, b) => b.length - a.length).map((item) => item.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"));
|
|
225
|
+
const sensitivePattern = escaped.length > 0 ? new RegExp(escaped.join("|"), "g") : null;
|
|
226
|
+
return (sensitivePattern ? value.replace(sensitivePattern, "[REDACTED]") : value).replace(/Authorization=[^\r\n'"]+/gi, "Authorization=[REDACTED]");
|
|
227
|
+
}
|
|
228
|
+
function resolveMcpHeaderValue(serverName, headerName, header) {
|
|
229
|
+
if (typeof header === "string") return header;
|
|
230
|
+
for (const source of header.sources ?? []) {
|
|
231
|
+
const rawValue = process.env[source.env];
|
|
232
|
+
if (rawValue && rawValue.trim().length > 0) return `${source.prefix ?? ""}${rawValue}${source.suffix ?? ""}`;
|
|
233
|
+
}
|
|
234
|
+
throw new PipelineMcpMissingCredentialError(serverName, headerName, header.sources.map((source) => source.env));
|
|
235
|
+
}
|
|
236
|
+
//#endregion
|
|
237
|
+
export { DEFAULT_MCPM_ARGS, DEFAULT_MCP_INSTALLS, DEFAULT_SKILL_INSTALLS, defaultMcpJson, installDefaultMcpsWithCli };
|