@kbediako/codex-orchestrator 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +7 -0
- package/README.md +238 -0
- package/dist/bin/codex-orchestrator.js +507 -0
- package/dist/orchestrator/src/agents/builder.js +16 -0
- package/dist/orchestrator/src/agents/index.js +4 -0
- package/dist/orchestrator/src/agents/planner.js +17 -0
- package/dist/orchestrator/src/agents/reviewer.js +13 -0
- package/dist/orchestrator/src/agents/tester.js +13 -0
- package/dist/orchestrator/src/cli/adapters/CommandBuilder.js +20 -0
- package/dist/orchestrator/src/cli/adapters/CommandPlanner.js +164 -0
- package/dist/orchestrator/src/cli/adapters/CommandReviewer.js +32 -0
- package/dist/orchestrator/src/cli/adapters/CommandTester.js +33 -0
- package/dist/orchestrator/src/cli/adapters/index.js +4 -0
- package/dist/orchestrator/src/cli/config/userConfig.js +28 -0
- package/dist/orchestrator/src/cli/doctor.js +48 -0
- package/dist/orchestrator/src/cli/events/runEvents.js +84 -0
- package/dist/orchestrator/src/cli/exec/command.js +56 -0
- package/dist/orchestrator/src/cli/exec/context.js +108 -0
- package/dist/orchestrator/src/cli/exec/experience.js +77 -0
- package/dist/orchestrator/src/cli/exec/finalization.js +140 -0
- package/dist/orchestrator/src/cli/exec/learning.js +62 -0
- package/dist/orchestrator/src/cli/exec/stageRunner.js +71 -0
- package/dist/orchestrator/src/cli/exec/summary.js +109 -0
- package/dist/orchestrator/src/cli/exec/telemetry.js +18 -0
- package/dist/orchestrator/src/cli/exec/tfgrpo.js +200 -0
- package/dist/orchestrator/src/cli/exec/tfgrpoArtifacts.js +19 -0
- package/dist/orchestrator/src/cli/exec/types.js +1 -0
- package/dist/orchestrator/src/cli/init.js +64 -0
- package/dist/orchestrator/src/cli/mcp.js +124 -0
- package/dist/orchestrator/src/cli/metrics/metricsAggregator.js +404 -0
- package/dist/orchestrator/src/cli/metrics/metricsRecorder.js +138 -0
- package/dist/orchestrator/src/cli/orchestrator.js +554 -0
- package/dist/orchestrator/src/cli/pipelines/defaultDiagnostics.js +32 -0
- package/dist/orchestrator/src/cli/pipelines/designReference.js +72 -0
- package/dist/orchestrator/src/cli/pipelines/hiFiDesignToolkit.js +71 -0
- package/dist/orchestrator/src/cli/pipelines/index.js +34 -0
- package/dist/orchestrator/src/cli/run/environment.js +24 -0
- package/dist/orchestrator/src/cli/run/manifest.js +367 -0
- package/dist/orchestrator/src/cli/run/manifestPersister.js +88 -0
- package/dist/orchestrator/src/cli/run/runPaths.js +30 -0
- package/dist/orchestrator/src/cli/selfCheck.js +12 -0
- package/dist/orchestrator/src/cli/services/commandRunner.js +420 -0
- package/dist/orchestrator/src/cli/services/controlPlaneService.js +107 -0
- package/dist/orchestrator/src/cli/services/execRuntime.js +69 -0
- package/dist/orchestrator/src/cli/services/pipelineResolver.js +47 -0
- package/dist/orchestrator/src/cli/services/runPreparation.js +82 -0
- package/dist/orchestrator/src/cli/services/runSummaryWriter.js +35 -0
- package/dist/orchestrator/src/cli/services/schedulerService.js +42 -0
- package/dist/orchestrator/src/cli/tasks/taskMetadata.js +19 -0
- package/dist/orchestrator/src/cli/telemetry/schema.js +8 -0
- package/dist/orchestrator/src/cli/types.js +1 -0
- package/dist/orchestrator/src/cli/ui/HudApp.js +112 -0
- package/dist/orchestrator/src/cli/ui/controller.js +26 -0
- package/dist/orchestrator/src/cli/ui/store.js +240 -0
- package/dist/orchestrator/src/cli/utils/enforcementMode.js +12 -0
- package/dist/orchestrator/src/cli/utils/fs.js +8 -0
- package/dist/orchestrator/src/cli/utils/interactive.js +25 -0
- package/dist/orchestrator/src/cli/utils/jsonlWriter.js +10 -0
- package/dist/orchestrator/src/cli/utils/optionalDeps.js +30 -0
- package/dist/orchestrator/src/cli/utils/packageInfo.js +25 -0
- package/dist/orchestrator/src/cli/utils/planFormatter.js +49 -0
- package/dist/orchestrator/src/cli/utils/runId.js +7 -0
- package/dist/orchestrator/src/cli/utils/specGuardRunner.js +26 -0
- package/dist/orchestrator/src/cli/utils/strings.js +8 -0
- package/dist/orchestrator/src/cli/utils/time.js +6 -0
- package/dist/orchestrator/src/control-plane/drift-reporter.js +109 -0
- package/dist/orchestrator/src/control-plane/index.js +3 -0
- package/dist/orchestrator/src/control-plane/request-builder.js +217 -0
- package/dist/orchestrator/src/control-plane/types.js +1 -0
- package/dist/orchestrator/src/control-plane/validator.js +50 -0
- package/dist/orchestrator/src/credentials/CredentialBroker.js +1 -0
- package/dist/orchestrator/src/events/EventBus.js +25 -0
- package/dist/orchestrator/src/learning/crystalizer.js +108 -0
- package/dist/orchestrator/src/learning/harvester.js +146 -0
- package/dist/orchestrator/src/learning/manifest.js +56 -0
- package/dist/orchestrator/src/learning/runner.js +177 -0
- package/dist/orchestrator/src/learning/validator.js +164 -0
- package/dist/orchestrator/src/logger.js +20 -0
- package/dist/orchestrator/src/manager.js +388 -0
- package/dist/orchestrator/src/persistence/ArtifactStager.js +95 -0
- package/dist/orchestrator/src/persistence/ExperienceStore.js +210 -0
- package/dist/orchestrator/src/persistence/PersistenceCoordinator.js +65 -0
- package/dist/orchestrator/src/persistence/RunManifestWriter.js +23 -0
- package/dist/orchestrator/src/persistence/TaskStateStore.js +172 -0
- package/dist/orchestrator/src/persistence/identifierGuards.js +1 -0
- package/dist/orchestrator/src/persistence/lockFile.js +26 -0
- package/dist/orchestrator/src/persistence/sanitizeIdentifier.js +26 -0
- package/dist/orchestrator/src/persistence/sanitizeRunId.js +8 -0
- package/dist/orchestrator/src/persistence/sanitizeTaskId.js +8 -0
- package/dist/orchestrator/src/persistence/writeAtomicFile.js +4 -0
- package/dist/orchestrator/src/privacy/guard.js +111 -0
- package/dist/orchestrator/src/scheduler/index.js +1 -0
- package/dist/orchestrator/src/scheduler/plan.js +171 -0
- package/dist/orchestrator/src/scheduler/types.js +1 -0
- package/dist/orchestrator/src/sync/CloudRunsClient.js +1 -0
- package/dist/orchestrator/src/sync/CloudRunsHttpClient.js +82 -0
- package/dist/orchestrator/src/sync/CloudSyncWorker.js +206 -0
- package/dist/orchestrator/src/sync/createCloudSyncWorker.js +15 -0
- package/dist/orchestrator/src/types.js +1 -0
- package/dist/orchestrator/src/utils/atomicWrite.js +15 -0
- package/dist/orchestrator/src/utils/errorMessage.js +14 -0
- package/dist/orchestrator/src/utils/executionMode.js +69 -0
- package/dist/packages/control-plane-schemas/src/index.js +1 -0
- package/dist/packages/control-plane-schemas/src/run-request.js +548 -0
- package/dist/packages/orchestrator/src/exec/handle-service.js +203 -0
- package/dist/packages/orchestrator/src/exec/session-manager.js +147 -0
- package/dist/packages/orchestrator/src/exec/unified-exec.js +432 -0
- package/dist/packages/orchestrator/src/index.js +3 -0
- package/dist/packages/orchestrator/src/instructions/loader.js +101 -0
- package/dist/packages/orchestrator/src/instructions/promptPacks.js +151 -0
- package/dist/packages/orchestrator/src/notifications/index.js +74 -0
- package/dist/packages/orchestrator/src/telemetry/otel-exporter.js +142 -0
- package/dist/packages/orchestrator/src/tool-orchestrator.js +161 -0
- package/dist/packages/sdk-node/src/orchestrator.js +195 -0
- package/dist/packages/shared/config/designConfig.js +495 -0
- package/dist/packages/shared/config/env.js +37 -0
- package/dist/packages/shared/config/index.js +2 -0
- package/dist/packages/shared/design-artifacts/writer.js +221 -0
- package/dist/packages/shared/events/serializer.js +84 -0
- package/dist/packages/shared/events/types.js +1 -0
- package/dist/packages/shared/manifest/artifactUtils.js +36 -0
- package/dist/packages/shared/manifest/designArtifacts.js +665 -0
- package/dist/packages/shared/manifest/fileIO.js +29 -0
- package/dist/packages/shared/manifest/toolRuns.js +78 -0
- package/dist/packages/shared/manifest/toolkitArtifacts.js +223 -0
- package/dist/packages/shared/manifest/types.js +5 -0
- package/dist/packages/shared/manifest/validator.js +73 -0
- package/dist/packages/shared/manifest/writer.js +2 -0
- package/dist/packages/shared/streams/stdio.js +112 -0
- package/dist/scripts/design/pipeline/advanced-assets.js +466 -0
- package/dist/scripts/design/pipeline/componentize.js +74 -0
- package/dist/scripts/design/pipeline/context.js +34 -0
- package/dist/scripts/design/pipeline/extract.js +249 -0
- package/dist/scripts/design/pipeline/optionalDeps.js +107 -0
- package/dist/scripts/design/pipeline/prepare.js +46 -0
- package/dist/scripts/design/pipeline/reference.js +94 -0
- package/dist/scripts/design/pipeline/state.js +206 -0
- package/dist/scripts/design/pipeline/toolkit/common.js +94 -0
- package/dist/scripts/design/pipeline/toolkit/extract.js +258 -0
- package/dist/scripts/design/pipeline/toolkit/publish.js +202 -0
- package/dist/scripts/design/pipeline/toolkit/publishActions.js +12 -0
- package/dist/scripts/design/pipeline/toolkit/reference.js +846 -0
- package/dist/scripts/design/pipeline/toolkit/snapshot.js +882 -0
- package/dist/scripts/design/pipeline/toolkit/tokens.js +456 -0
- package/dist/scripts/design/pipeline/visual-regression.js +137 -0
- package/dist/scripts/design/pipeline/write-artifacts.js +61 -0
- package/package.json +97 -0
- package/schemas/manifest.json +1064 -0
- package/templates/README.md +12 -0
- package/templates/codex/mcp-client.json +8 -0
|
@@ -0,0 +1,420 @@
|
|
|
1
|
+
import { createWriteStream } from 'node:fs';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import { ToolInvocationFailedError } from '../../../../packages/orchestrator/src/index.js';
|
|
4
|
+
import { getCliExecRunner, getPrivacyGuard, getExecHandleService } from './execRuntime.js';
|
|
5
|
+
import { logger } from '../../logger.js';
|
|
6
|
+
import { relativeToRepo } from '../run/runPaths.js';
|
|
7
|
+
import { appendCommandError, updateCommandStatus } from '../run/manifest.js';
|
|
8
|
+
import { persistManifest } from '../run/manifestPersister.js';
|
|
9
|
+
import { slugify } from '../utils/strings.js';
|
|
10
|
+
import { isoTimestamp } from '../utils/time.js';
|
|
11
|
+
import { EnvUtils } from '../../../../packages/shared/config/index.js';
|
|
12
|
+
import { findPackageRoot } from '../utils/packageInfo.js';
|
|
13
|
+
const MAX_BUFFERED_OUTPUT_BYTES = 64 * 1024;
|
|
14
|
+
const EMIT_COMMAND_STREAM_MIRRORS = EnvUtils.getBoolean('CODEX_ORCHESTRATOR_EMIT_COMMAND_STREAMS', false);
|
|
15
|
+
const MAX_CAPTURED_CHUNK_EVENTS = EnvUtils.getInt('CODEX_ORCHESTRATOR_EXEC_EVENT_MAX_CHUNKS', 0);
|
|
16
|
+
const PACKAGE_ROOT = findPackageRoot();
|
|
17
|
+
export async function runCommandStage(context, hooks = {}) {
|
|
18
|
+
const { env, paths, manifest, stage, index, events, persister, envOverrides } = context;
|
|
19
|
+
const entryIndex = index - 1;
|
|
20
|
+
const entry = updateCommandStatus(manifest, entryIndex, {
|
|
21
|
+
status: 'running',
|
|
22
|
+
started_at: isoTimestamp(),
|
|
23
|
+
exit_code: null,
|
|
24
|
+
summary: null
|
|
25
|
+
});
|
|
26
|
+
const logFile = join(paths.commandsDir, `${String(index).padStart(2, '0')}-${slugify(stage.id)}.ndjson`);
|
|
27
|
+
entry.log_path = relativeToRepo(env, logFile);
|
|
28
|
+
await persistManifest(paths, manifest, persister, { force: true });
|
|
29
|
+
events?.stageStarted({
|
|
30
|
+
stageId: stage.id,
|
|
31
|
+
stageIndex: index,
|
|
32
|
+
title: stage.title,
|
|
33
|
+
kind: 'command',
|
|
34
|
+
logPath: entry.log_path,
|
|
35
|
+
status: entry.status
|
|
36
|
+
});
|
|
37
|
+
const runnerLog = createWriteStream(paths.logPath, { flags: 'a' });
|
|
38
|
+
const commandLog = createWriteStream(logFile, { flags: 'a' });
|
|
39
|
+
const privacyLogPath = join(paths.runDir, 'privacy-decisions.ndjson');
|
|
40
|
+
const privacyLog = createWriteStream(privacyLogPath, { flags: 'a' });
|
|
41
|
+
const writeEvent = (message) => {
|
|
42
|
+
const payload = `${JSON.stringify({ ...message, timestamp: isoTimestamp(), index })}\n`;
|
|
43
|
+
runnerLog.write(payload);
|
|
44
|
+
commandLog.write(payload);
|
|
45
|
+
};
|
|
46
|
+
writeEvent({ type: 'command:start', command: stage.command });
|
|
47
|
+
const runner = getCliExecRunner();
|
|
48
|
+
let activeCorrelationId = null;
|
|
49
|
+
let stdoutBytes = 0;
|
|
50
|
+
let stderrBytes = 0;
|
|
51
|
+
let stdoutTruncated = false;
|
|
52
|
+
let stderrTruncated = false;
|
|
53
|
+
const handleEvent = (event) => {
|
|
54
|
+
if (!activeCorrelationId) {
|
|
55
|
+
activeCorrelationId = event.correlationId;
|
|
56
|
+
}
|
|
57
|
+
if (event.correlationId !== activeCorrelationId) {
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
hooks.onEvent?.(event);
|
|
61
|
+
streamEvent(writeEvent, event, {
|
|
62
|
+
onStdout: (bytes) => {
|
|
63
|
+
stdoutBytes += bytes;
|
|
64
|
+
stdoutTruncated = stdoutTruncated || stdoutBytes > MAX_BUFFERED_OUTPUT_BYTES;
|
|
65
|
+
},
|
|
66
|
+
onStderr: (bytes) => {
|
|
67
|
+
stderrBytes += bytes;
|
|
68
|
+
stderrTruncated = stderrTruncated || stderrBytes > MAX_BUFFERED_OUTPUT_BYTES;
|
|
69
|
+
}
|
|
70
|
+
});
|
|
71
|
+
switch (event.type) {
|
|
72
|
+
case 'exec:begin':
|
|
73
|
+
events?.toolCall({
|
|
74
|
+
stageId: stage.id,
|
|
75
|
+
stageIndex: index,
|
|
76
|
+
toolName: 'exec',
|
|
77
|
+
status: 'started',
|
|
78
|
+
message: stage.command,
|
|
79
|
+
attempt: event.attempt
|
|
80
|
+
});
|
|
81
|
+
break;
|
|
82
|
+
case 'exec:chunk':
|
|
83
|
+
events?.log({
|
|
84
|
+
stageId: stage.id,
|
|
85
|
+
stageIndex: index,
|
|
86
|
+
level: event.payload.stream === 'stderr' ? 'error' : 'info',
|
|
87
|
+
message: event.payload.data,
|
|
88
|
+
source: event.payload.stream
|
|
89
|
+
});
|
|
90
|
+
break;
|
|
91
|
+
case 'exec:retry':
|
|
92
|
+
events?.toolCall({
|
|
93
|
+
stageId: stage.id,
|
|
94
|
+
stageIndex: index,
|
|
95
|
+
toolName: 'exec',
|
|
96
|
+
status: 'retry',
|
|
97
|
+
message: event.payload.errorMessage,
|
|
98
|
+
attempt: event.attempt
|
|
99
|
+
});
|
|
100
|
+
break;
|
|
101
|
+
case 'exec:end':
|
|
102
|
+
events?.toolCall({
|
|
103
|
+
stageId: stage.id,
|
|
104
|
+
stageIndex: index,
|
|
105
|
+
toolName: 'exec',
|
|
106
|
+
status: event.payload.status,
|
|
107
|
+
message: `exit ${event.payload.exitCode ?? 'null'}`,
|
|
108
|
+
attempt: event.attempt
|
|
109
|
+
});
|
|
110
|
+
break;
|
|
111
|
+
default:
|
|
112
|
+
break;
|
|
113
|
+
}
|
|
114
|
+
};
|
|
115
|
+
const unsubscribe = runner.on(handleEvent);
|
|
116
|
+
try {
|
|
117
|
+
const sessionConfig = stage.session ?? {};
|
|
118
|
+
const sessionId = sessionConfig.id;
|
|
119
|
+
const wantsPersist = Boolean(sessionConfig.persist || sessionConfig.reuse);
|
|
120
|
+
const persistSession = Boolean(sessionId && wantsPersist);
|
|
121
|
+
const reuseSession = Boolean(sessionId && (sessionConfig.reuse ?? persistSession));
|
|
122
|
+
const baseEnv = {
|
|
123
|
+
...process.env,
|
|
124
|
+
...(envOverrides ?? {}),
|
|
125
|
+
CODEX_ORCHESTRATOR_TASK_ID: manifest.task_id,
|
|
126
|
+
CODEX_ORCHESTRATOR_RUN_ID: manifest.run_id,
|
|
127
|
+
CODEX_ORCHESTRATOR_PIPELINE_ID: manifest.pipeline_id,
|
|
128
|
+
CODEX_ORCHESTRATOR_MANIFEST_PATH: paths.manifestPath,
|
|
129
|
+
CODEX_ORCHESTRATOR_RUN_DIR: paths.runDir,
|
|
130
|
+
CODEX_ORCHESTRATOR_RUNS_DIR: env.runsRoot,
|
|
131
|
+
CODEX_ORCHESTRATOR_OUT_DIR: env.outRoot,
|
|
132
|
+
CODEX_ORCHESTRATOR_REPO_ROOT: env.repoRoot,
|
|
133
|
+
CODEX_ORCHESTRATOR_PACKAGE_ROOT: PACKAGE_ROOT
|
|
134
|
+
};
|
|
135
|
+
const execEnv = { ...baseEnv, ...stage.env };
|
|
136
|
+
const invocationId = `cli-command:${manifest.run_id}:${stage.id}:${Date.now()}`;
|
|
137
|
+
let result;
|
|
138
|
+
const eventCapture = MAX_CAPTURED_CHUNK_EVENTS > 0 ? { maxChunkEvents: MAX_CAPTURED_CHUNK_EVENTS } : undefined;
|
|
139
|
+
try {
|
|
140
|
+
result = await runner.run({
|
|
141
|
+
command: stage.command,
|
|
142
|
+
args: [],
|
|
143
|
+
cwd: stage.cwd ?? env.repoRoot,
|
|
144
|
+
env: execEnv,
|
|
145
|
+
sessionId: sessionId ?? undefined,
|
|
146
|
+
persistSession,
|
|
147
|
+
reuseSession,
|
|
148
|
+
invocationId,
|
|
149
|
+
toolId: 'cli:command',
|
|
150
|
+
description: stage.title,
|
|
151
|
+
eventCapture,
|
|
152
|
+
metadata: {
|
|
153
|
+
stageId: stage.id,
|
|
154
|
+
pipelineId: manifest.pipeline_id,
|
|
155
|
+
runId: manifest.run_id,
|
|
156
|
+
commandIndex: entry.index
|
|
157
|
+
}
|
|
158
|
+
});
|
|
159
|
+
hooks.onResult?.(result);
|
|
160
|
+
if (result.handle) {
|
|
161
|
+
recordHandle(manifest, result.handle, {
|
|
162
|
+
stageId: stage.id,
|
|
163
|
+
pipelineId: manifest.pipeline_id,
|
|
164
|
+
runId: manifest.run_id
|
|
165
|
+
});
|
|
166
|
+
const appendedPrivacyRecords = updatePrivacyManifest(manifest, {
|
|
167
|
+
env,
|
|
168
|
+
paths,
|
|
169
|
+
logPath: privacyLogPath
|
|
170
|
+
});
|
|
171
|
+
writePrivacyLog(privacyLog, appendedPrivacyRecords);
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
catch (error) {
|
|
175
|
+
if (error instanceof ToolInvocationFailedError) {
|
|
176
|
+
hooks.onError?.(error);
|
|
177
|
+
captureFailureHandle(manifest, stage, error);
|
|
178
|
+
const appendedPrivacyRecords = updatePrivacyManifest(manifest, {
|
|
179
|
+
env,
|
|
180
|
+
paths,
|
|
181
|
+
logPath: privacyLogPath
|
|
182
|
+
});
|
|
183
|
+
writePrivacyLog(privacyLog, appendedPrivacyRecords);
|
|
184
|
+
}
|
|
185
|
+
throw error;
|
|
186
|
+
}
|
|
187
|
+
const normalizedExitCode = result.exitCode ?? (result.signal ? 128 : 0);
|
|
188
|
+
const stdoutText = result.stdout.trim();
|
|
189
|
+
const stderrText = result.stderr.trim();
|
|
190
|
+
const summary = buildSummary(stage, normalizedExitCode, stdoutText, stderrText, result.signal);
|
|
191
|
+
entry.completed_at = isoTimestamp();
|
|
192
|
+
entry.exit_code = normalizedExitCode;
|
|
193
|
+
entry.summary = summary;
|
|
194
|
+
entry.status = result.status === 'succeeded' ? 'succeeded' : stage.allowFailure ? 'skipped' : 'failed';
|
|
195
|
+
if (entry.status === 'failed') {
|
|
196
|
+
const errorDetails = {
|
|
197
|
+
exit_code: normalizedExitCode,
|
|
198
|
+
sandbox_state: result.sandboxState,
|
|
199
|
+
stderr: stderrText
|
|
200
|
+
};
|
|
201
|
+
if (result.signal) {
|
|
202
|
+
errorDetails.signal = result.signal;
|
|
203
|
+
}
|
|
204
|
+
if (stdoutTruncated) {
|
|
205
|
+
errorDetails.stdout_truncated = true;
|
|
206
|
+
}
|
|
207
|
+
if (stderrTruncated) {
|
|
208
|
+
errorDetails.stderr_truncated = true;
|
|
209
|
+
}
|
|
210
|
+
entry.error_file = await appendCommandError(env, paths, manifest, entry, 'command-failed', errorDetails);
|
|
211
|
+
}
|
|
212
|
+
await persistManifest(paths, manifest, persister, { force: true });
|
|
213
|
+
events?.stageCompleted({
|
|
214
|
+
stageId: stage.id,
|
|
215
|
+
stageIndex: index,
|
|
216
|
+
title: stage.title,
|
|
217
|
+
kind: 'command',
|
|
218
|
+
status: entry.status,
|
|
219
|
+
exitCode: entry.exit_code,
|
|
220
|
+
summary: entry.summary,
|
|
221
|
+
logPath: entry.log_path
|
|
222
|
+
});
|
|
223
|
+
return { exitCode: normalizedExitCode, summary };
|
|
224
|
+
}
|
|
225
|
+
finally {
|
|
226
|
+
unsubscribe();
|
|
227
|
+
runnerLog.end();
|
|
228
|
+
commandLog.end();
|
|
229
|
+
privacyLog.end();
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
function recordHandle(manifest, descriptor, context) {
|
|
233
|
+
const handles = Array.isArray(manifest.handles) ? [...manifest.handles] : [];
|
|
234
|
+
const entry = {
|
|
235
|
+
handle_id: descriptor.id,
|
|
236
|
+
correlation_id: descriptor.correlationId,
|
|
237
|
+
stage_id: context.stageId,
|
|
238
|
+
pipeline_id: context.pipelineId,
|
|
239
|
+
status: descriptor.status,
|
|
240
|
+
frame_count: descriptor.frameCount,
|
|
241
|
+
latest_sequence: descriptor.latestSequence,
|
|
242
|
+
created_at: descriptor.createdAt,
|
|
243
|
+
metadata: {
|
|
244
|
+
run_id: context.runId
|
|
245
|
+
}
|
|
246
|
+
};
|
|
247
|
+
const existingIndex = handles.findIndex((candidate) => candidate.handle_id === entry.handle_id);
|
|
248
|
+
if (existingIndex >= 0) {
|
|
249
|
+
handles[existingIndex] = entry;
|
|
250
|
+
}
|
|
251
|
+
else {
|
|
252
|
+
handles.push(entry);
|
|
253
|
+
}
|
|
254
|
+
manifest.handles = handles;
|
|
255
|
+
}
|
|
256
|
+
function updatePrivacyManifest(manifest, context) {
|
|
257
|
+
const metrics = getPrivacyGuard().getMetrics();
|
|
258
|
+
const existingDecisions = manifest.privacy?.decisions ?? [];
|
|
259
|
+
const newMetricsDecisions = metrics.decisions.slice(existingDecisions.length);
|
|
260
|
+
const appended = newMetricsDecisions.map((decision) => ({
|
|
261
|
+
handle_id: decision.handleId,
|
|
262
|
+
sequence: decision.sequence,
|
|
263
|
+
action: decision.action,
|
|
264
|
+
rule: decision.rule ?? null,
|
|
265
|
+
reason: decision.reason ?? null,
|
|
266
|
+
timestamp: decision.timestamp,
|
|
267
|
+
stage_id: resolveHandleStage(manifest, decision.handleId)
|
|
268
|
+
}));
|
|
269
|
+
if (!manifest.privacy) {
|
|
270
|
+
manifest.privacy = {
|
|
271
|
+
mode: metrics.mode,
|
|
272
|
+
decisions: [...appended],
|
|
273
|
+
totals: {
|
|
274
|
+
total_frames: metrics.totalFrames,
|
|
275
|
+
redacted_frames: metrics.redactedFrames,
|
|
276
|
+
blocked_frames: metrics.blockedFrames,
|
|
277
|
+
allowed_frames: metrics.allowedFrames
|
|
278
|
+
},
|
|
279
|
+
log_path: relativeToRepo(context.env, context.logPath)
|
|
280
|
+
};
|
|
281
|
+
}
|
|
282
|
+
else {
|
|
283
|
+
manifest.privacy.mode = metrics.mode;
|
|
284
|
+
manifest.privacy.totals = {
|
|
285
|
+
total_frames: metrics.totalFrames,
|
|
286
|
+
redacted_frames: metrics.redactedFrames,
|
|
287
|
+
blocked_frames: metrics.blockedFrames,
|
|
288
|
+
allowed_frames: metrics.allowedFrames
|
|
289
|
+
};
|
|
290
|
+
if (appended.length > 0) {
|
|
291
|
+
manifest.privacy.decisions.push(...appended);
|
|
292
|
+
}
|
|
293
|
+
manifest.privacy.log_path = relativeToRepo(context.env, context.logPath);
|
|
294
|
+
}
|
|
295
|
+
return appended;
|
|
296
|
+
}
|
|
297
|
+
function resolveHandleStage(manifest, handleId) {
|
|
298
|
+
const record = manifest.handles?.find((entry) => entry.handle_id === handleId);
|
|
299
|
+
return record?.stage_id ?? null;
|
|
300
|
+
}
|
|
301
|
+
function captureFailureHandle(manifest, stage, error) {
|
|
302
|
+
const metadata = error.record?.metadata?.exec;
|
|
303
|
+
const handleId = metadata?.handleId;
|
|
304
|
+
if (!handleId) {
|
|
305
|
+
return;
|
|
306
|
+
}
|
|
307
|
+
try {
|
|
308
|
+
const descriptor = getExecHandleService().getDescriptor(handleId);
|
|
309
|
+
recordHandle(manifest, descriptor, {
|
|
310
|
+
stageId: stage.id,
|
|
311
|
+
pipelineId: manifest.pipeline_id,
|
|
312
|
+
runId: manifest.run_id
|
|
313
|
+
});
|
|
314
|
+
}
|
|
315
|
+
catch (lookupError) {
|
|
316
|
+
logger.warn(`Handle descriptor missing for failed stage ${stage.id ?? '<unknown>'}: ${lookupError.message}`);
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
function writePrivacyLog(stream, records) {
|
|
320
|
+
if (!records || records.length === 0) {
|
|
321
|
+
return;
|
|
322
|
+
}
|
|
323
|
+
for (const record of records) {
|
|
324
|
+
stream.write(`${JSON.stringify(record)}\n`);
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
function streamEvent(writeEvent, event, hooks) {
|
|
328
|
+
switch (event.type) {
|
|
329
|
+
case 'exec:begin':
|
|
330
|
+
writeEvent({
|
|
331
|
+
type: 'exec:begin',
|
|
332
|
+
correlation_id: event.correlationId,
|
|
333
|
+
attempt: event.attempt,
|
|
334
|
+
command: event.payload.command,
|
|
335
|
+
args: event.payload.args,
|
|
336
|
+
cwd: event.payload.cwd,
|
|
337
|
+
session_id: event.payload.sessionId,
|
|
338
|
+
sandbox_state: event.payload.sandboxState,
|
|
339
|
+
persisted: event.payload.persisted
|
|
340
|
+
});
|
|
341
|
+
break;
|
|
342
|
+
case 'exec:chunk': {
|
|
343
|
+
writeEvent({
|
|
344
|
+
type: 'exec:chunk',
|
|
345
|
+
correlation_id: event.correlationId,
|
|
346
|
+
attempt: event.attempt,
|
|
347
|
+
stream: event.payload.stream,
|
|
348
|
+
sequence: event.payload.sequence,
|
|
349
|
+
bytes: event.payload.bytes,
|
|
350
|
+
data: event.payload.data
|
|
351
|
+
});
|
|
352
|
+
if (EMIT_COMMAND_STREAM_MIRRORS) {
|
|
353
|
+
writeEvent({
|
|
354
|
+
type: event.payload.stream === 'stdout' ? 'command:stdout' : 'command:stderr',
|
|
355
|
+
data: event.payload.data
|
|
356
|
+
});
|
|
357
|
+
}
|
|
358
|
+
if (event.payload.stream === 'stdout') {
|
|
359
|
+
hooks.onStdout(event.payload.bytes);
|
|
360
|
+
}
|
|
361
|
+
else {
|
|
362
|
+
hooks.onStderr(event.payload.bytes);
|
|
363
|
+
}
|
|
364
|
+
break;
|
|
365
|
+
}
|
|
366
|
+
case 'exec:retry':
|
|
367
|
+
writeEvent({
|
|
368
|
+
type: 'exec:retry',
|
|
369
|
+
correlation_id: event.correlationId,
|
|
370
|
+
attempt: event.attempt,
|
|
371
|
+
delay_ms: event.payload.delayMs,
|
|
372
|
+
sandbox_state: event.payload.sandboxState,
|
|
373
|
+
error: event.payload.errorMessage
|
|
374
|
+
});
|
|
375
|
+
break;
|
|
376
|
+
case 'exec:end':
|
|
377
|
+
writeEvent({
|
|
378
|
+
type: 'exec:end',
|
|
379
|
+
correlation_id: event.correlationId,
|
|
380
|
+
attempt: event.attempt,
|
|
381
|
+
exit_code: event.payload.exitCode,
|
|
382
|
+
signal: event.payload.signal,
|
|
383
|
+
duration_ms: event.payload.durationMs,
|
|
384
|
+
status: event.payload.status
|
|
385
|
+
});
|
|
386
|
+
writeEvent({
|
|
387
|
+
type: 'command:end',
|
|
388
|
+
exit_code: event.payload.exitCode,
|
|
389
|
+
signal: event.payload.signal,
|
|
390
|
+
duration_ms: event.payload.durationMs
|
|
391
|
+
});
|
|
392
|
+
break;
|
|
393
|
+
default:
|
|
394
|
+
break;
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
function buildSummary(stage, exitCode, stdout, stderr, signal) {
|
|
398
|
+
if (stage.summaryHint) {
|
|
399
|
+
return stage.summaryHint;
|
|
400
|
+
}
|
|
401
|
+
if (signal) {
|
|
402
|
+
return `Terminated with signal ${signal}${stderr ? ` — ${truncate(stderr)}` : ''}`;
|
|
403
|
+
}
|
|
404
|
+
if (exitCode !== 0) {
|
|
405
|
+
return `Exited with code ${exitCode}${stderr ? ` — ${truncate(stderr)}` : ''}`;
|
|
406
|
+
}
|
|
407
|
+
if (stdout) {
|
|
408
|
+
return truncate(stdout);
|
|
409
|
+
}
|
|
410
|
+
if (stderr) {
|
|
411
|
+
return truncate(stderr);
|
|
412
|
+
}
|
|
413
|
+
return `Command completed with code ${exitCode}`;
|
|
414
|
+
}
|
|
415
|
+
function truncate(value, length = 240) {
|
|
416
|
+
if (value.length <= length) {
|
|
417
|
+
return value;
|
|
418
|
+
}
|
|
419
|
+
return `${value.slice(0, length)}…`;
|
|
420
|
+
}
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
import process from 'node:process';
|
|
2
|
+
import { buildRunRequestV2, ControlPlaneDriftReporter, ControlPlaneValidationError, ControlPlaneValidator } from '../../control-plane/index.js';
|
|
3
|
+
import { relativeToRepo } from '../run/runPaths.js';
|
|
4
|
+
import { appendSummary } from '../run/manifest.js';
|
|
5
|
+
import { persistManifest } from '../run/manifestPersister.js';
|
|
6
|
+
import { isoTimestamp } from '../utils/time.js';
|
|
7
|
+
import { resolveEnforcementMode } from '../utils/enforcementMode.js';
|
|
8
|
+
export class ControlPlaneService {
|
|
9
|
+
modeResolver;
|
|
10
|
+
now;
|
|
11
|
+
constructor(modeResolver = resolveControlPlaneMode, now = () => new Date()) {
|
|
12
|
+
this.modeResolver = modeResolver;
|
|
13
|
+
this.now = now;
|
|
14
|
+
}
|
|
15
|
+
async guard(options) {
|
|
16
|
+
const mode = this.modeResolver();
|
|
17
|
+
const driftReporter = new ControlPlaneDriftReporter({
|
|
18
|
+
repoRoot: options.env.repoRoot,
|
|
19
|
+
taskId: options.env.taskId
|
|
20
|
+
});
|
|
21
|
+
const validator = new ControlPlaneValidator({ mode, driftReporter, now: this.now });
|
|
22
|
+
const request = buildRunRequestV2({
|
|
23
|
+
runId: options.runId,
|
|
24
|
+
task: options.task,
|
|
25
|
+
pipeline: options.pipeline,
|
|
26
|
+
manifest: options.manifest,
|
|
27
|
+
env: options.env,
|
|
28
|
+
requestedBy: options.requestedBy,
|
|
29
|
+
now: this.now
|
|
30
|
+
});
|
|
31
|
+
try {
|
|
32
|
+
const result = await validator.validate(request);
|
|
33
|
+
this.attachControlPlaneToManifest(options.env, options.manifest, result);
|
|
34
|
+
await persistManifest(options.paths, options.manifest, options.persister, { force: true });
|
|
35
|
+
return result;
|
|
36
|
+
}
|
|
37
|
+
catch (error) {
|
|
38
|
+
if (error instanceof ControlPlaneValidationError) {
|
|
39
|
+
this.attachControlPlaneToManifest(options.env, options.manifest, error.result);
|
|
40
|
+
options.manifest.status = 'failed';
|
|
41
|
+
options.manifest.status_detail = 'control-plane-validation-failed';
|
|
42
|
+
options.manifest.completed_at = isoTimestamp();
|
|
43
|
+
appendSummary(options.manifest, `Control plane validation failed: ${error.message}`);
|
|
44
|
+
await persistManifest(options.paths, options.manifest, options.persister, { force: true });
|
|
45
|
+
}
|
|
46
|
+
throw error;
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
attachControlPlaneToManifest(env, manifest, result) {
|
|
50
|
+
const { request, outcome } = result;
|
|
51
|
+
const drift = outcome.drift
|
|
52
|
+
? {
|
|
53
|
+
report_path: relativeToRepo(env, outcome.drift.absoluteReportPath),
|
|
54
|
+
total_samples: outcome.drift.totalSamples,
|
|
55
|
+
invalid_samples: outcome.drift.invalidSamples,
|
|
56
|
+
invalid_rate: outcome.drift.invalidRate,
|
|
57
|
+
last_sampled_at: outcome.drift.lastSampledAt,
|
|
58
|
+
mode: outcome.drift.mode
|
|
59
|
+
}
|
|
60
|
+
: undefined;
|
|
61
|
+
manifest.control_plane = {
|
|
62
|
+
schema_id: request.schema,
|
|
63
|
+
schema_version: request.version,
|
|
64
|
+
request_id: request.requestId,
|
|
65
|
+
validation: {
|
|
66
|
+
mode: outcome.mode,
|
|
67
|
+
status: outcome.status,
|
|
68
|
+
timestamp: outcome.timestamp,
|
|
69
|
+
errors: outcome.errors.map((error) => ({ ...error }))
|
|
70
|
+
},
|
|
71
|
+
drift,
|
|
72
|
+
enforcement: {
|
|
73
|
+
enabled: outcome.mode === 'enforce',
|
|
74
|
+
activated_at: outcome.mode === 'enforce' ? outcome.timestamp : null
|
|
75
|
+
}
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
applyControlPlaneToRunSummary(runSummary, result) {
|
|
79
|
+
if (!result) {
|
|
80
|
+
return;
|
|
81
|
+
}
|
|
82
|
+
const { request, outcome } = result;
|
|
83
|
+
runSummary.controlPlane = {
|
|
84
|
+
schemaId: request.schema,
|
|
85
|
+
schemaVersion: request.version,
|
|
86
|
+
requestId: request.requestId,
|
|
87
|
+
validation: {
|
|
88
|
+
mode: outcome.mode,
|
|
89
|
+
status: outcome.status,
|
|
90
|
+
timestamp: outcome.timestamp,
|
|
91
|
+
errors: outcome.errors.map((error) => ({ ...error }))
|
|
92
|
+
},
|
|
93
|
+
drift: outcome.drift
|
|
94
|
+
? {
|
|
95
|
+
mode: outcome.drift.mode,
|
|
96
|
+
totalSamples: outcome.drift.totalSamples,
|
|
97
|
+
invalidSamples: outcome.drift.invalidSamples,
|
|
98
|
+
invalidRate: outcome.drift.invalidRate,
|
|
99
|
+
lastSampledAt: outcome.drift.lastSampledAt
|
|
100
|
+
}
|
|
101
|
+
: undefined
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
export function resolveControlPlaneMode() {
|
|
106
|
+
return resolveEnforcementMode(process.env.CODEX_CONTROL_PLANE_MODE ?? null, process.env.CODEX_CONTROL_PLANE_ENFORCE ?? null);
|
|
107
|
+
}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { spawn } from 'node:child_process';
|
|
2
|
+
import { ExecSessionManager, UnifiedExecRunner, ToolOrchestrator } from '../../../../packages/orchestrator/src/index.js';
|
|
3
|
+
import { RemoteExecHandleService } from '../../../../packages/orchestrator/src/exec/handle-service.js';
|
|
4
|
+
import { PrivacyGuard } from '../../privacy/guard.js';
|
|
5
|
+
import { resolveEnforcementMode } from '../utils/enforcementMode.js';
|
|
6
|
+
class CliExecSessionHandle {
|
|
7
|
+
id;
|
|
8
|
+
constructor(id) {
|
|
9
|
+
this.id = id;
|
|
10
|
+
}
|
|
11
|
+
async dispose() {
|
|
12
|
+
// CLI commands currently spawn per-invocation processes so there is no persistent resource to release.
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
const orchestrator = new ToolOrchestrator();
|
|
16
|
+
const sessionManager = new ExecSessionManager({
|
|
17
|
+
baseEnv: process.env,
|
|
18
|
+
factory: async ({ id }) => new CliExecSessionHandle(id)
|
|
19
|
+
});
|
|
20
|
+
const privacyGuard = new PrivacyGuard({ mode: resolvePrivacyGuardMode() });
|
|
21
|
+
const handleService = new RemoteExecHandleService({ guard: privacyGuard, now: () => new Date() });
|
|
22
|
+
const cliExecutor = async (request) => {
|
|
23
|
+
const child = spawn(request.command, request.args ?? [], {
|
|
24
|
+
cwd: request.cwd,
|
|
25
|
+
env: request.env,
|
|
26
|
+
shell: true,
|
|
27
|
+
stdio: ['ignore', 'pipe', 'pipe']
|
|
28
|
+
});
|
|
29
|
+
if (!child.stdout || !child.stderr) {
|
|
30
|
+
throw new Error('CLI exec commands require stdout/stderr streams.');
|
|
31
|
+
}
|
|
32
|
+
child.stdout.on('data', request.onStdout);
|
|
33
|
+
child.stderr.on('data', request.onStderr);
|
|
34
|
+
return await new Promise((resolve, reject) => {
|
|
35
|
+
const handleExit = (exitCode, signal) => {
|
|
36
|
+
cleanup();
|
|
37
|
+
resolve({ exitCode, signal });
|
|
38
|
+
};
|
|
39
|
+
const handleError = (error) => {
|
|
40
|
+
cleanup();
|
|
41
|
+
reject(error);
|
|
42
|
+
};
|
|
43
|
+
const cleanup = () => {
|
|
44
|
+
child.off('exit', handleExit);
|
|
45
|
+
child.off('error', handleError);
|
|
46
|
+
};
|
|
47
|
+
child.once('exit', handleExit);
|
|
48
|
+
child.once('error', handleError);
|
|
49
|
+
});
|
|
50
|
+
};
|
|
51
|
+
const runner = new UnifiedExecRunner({
|
|
52
|
+
orchestrator,
|
|
53
|
+
sessionManager,
|
|
54
|
+
executor: cliExecutor,
|
|
55
|
+
now: () => new Date(),
|
|
56
|
+
handleService
|
|
57
|
+
});
|
|
58
|
+
export function getCliExecRunner() {
|
|
59
|
+
return runner;
|
|
60
|
+
}
|
|
61
|
+
export function getExecHandleService() {
|
|
62
|
+
return handleService;
|
|
63
|
+
}
|
|
64
|
+
export function getPrivacyGuard() {
|
|
65
|
+
return privacyGuard;
|
|
66
|
+
}
|
|
67
|
+
function resolvePrivacyGuardMode() {
|
|
68
|
+
return resolveEnforcementMode(process.env.CODEX_PRIVACY_GUARD_MODE ?? null, process.env.CODEX_PRIVACY_GUARD_ENFORCE ?? null);
|
|
69
|
+
}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import process from 'node:process';
|
|
2
|
+
import { loadUserConfig } from '../config/userConfig.js';
|
|
3
|
+
import { resolvePipeline } from '../pipelines/index.js';
|
|
4
|
+
import { loadDesignConfig, shouldActivateDesignPipeline, designPipelineId } from '../../../../packages/shared/config/index.js';
|
|
5
|
+
import { logger } from '../../logger.js';
|
|
6
|
+
export class PipelineResolver {
|
|
7
|
+
async loadDesignConfig(rootDir) {
|
|
8
|
+
const designConfig = await loadDesignConfig({ rootDir });
|
|
9
|
+
if (designConfig.warnings.length > 0) {
|
|
10
|
+
for (const warning of designConfig.warnings) {
|
|
11
|
+
logger.warn(`[design-config] ${warning}`);
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
return designConfig;
|
|
15
|
+
}
|
|
16
|
+
async resolve(env, options) {
|
|
17
|
+
logger.info(`PipelineResolver.resolve start for ${options.pipelineId ?? '<default>'}`);
|
|
18
|
+
const designConfig = await this.loadDesignConfig(env.repoRoot);
|
|
19
|
+
logger.info(`PipelineResolver.resolve loaded design config from ${designConfig.path}`);
|
|
20
|
+
const userConfig = await loadUserConfig(env);
|
|
21
|
+
logger.info(`PipelineResolver.resolve loaded user config`);
|
|
22
|
+
const requestedPipelineId = options.pipelineId ??
|
|
23
|
+
(shouldActivateDesignPipeline(designConfig) ? designPipelineId(designConfig) : undefined);
|
|
24
|
+
const envOverrides = this.resolveDesignEnvOverrides(designConfig, requestedPipelineId);
|
|
25
|
+
try {
|
|
26
|
+
const { pipeline, source } = resolvePipeline(env, {
|
|
27
|
+
pipelineId: requestedPipelineId,
|
|
28
|
+
config: userConfig
|
|
29
|
+
});
|
|
30
|
+
logger.info(`PipelineResolver.resolve selected pipeline ${pipeline.id}`);
|
|
31
|
+
return { pipeline, userConfig, designConfig, source, envOverrides };
|
|
32
|
+
}
|
|
33
|
+
catch (error) {
|
|
34
|
+
logger.error(`PipelineResolver.resolve failed for ${requestedPipelineId ?? '<default>'}: ${error.message}`);
|
|
35
|
+
throw error;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
resolveDesignEnvOverrides(designConfig, pipelineId) {
|
|
39
|
+
const envOverrides = {
|
|
40
|
+
DESIGN_CONFIG_PATH: designConfig.path
|
|
41
|
+
};
|
|
42
|
+
if (pipelineId === designPipelineId(designConfig) && process.env.DESIGN_PIPELINE === undefined) {
|
|
43
|
+
envOverrides.DESIGN_PIPELINE = '1';
|
|
44
|
+
}
|
|
45
|
+
return envOverrides;
|
|
46
|
+
}
|
|
47
|
+
}
|