@kbediako/codex-orchestrator 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +7 -0
- package/README.md +238 -0
- package/dist/bin/codex-orchestrator.js +507 -0
- package/dist/orchestrator/src/agents/builder.js +16 -0
- package/dist/orchestrator/src/agents/index.js +4 -0
- package/dist/orchestrator/src/agents/planner.js +17 -0
- package/dist/orchestrator/src/agents/reviewer.js +13 -0
- package/dist/orchestrator/src/agents/tester.js +13 -0
- package/dist/orchestrator/src/cli/adapters/CommandBuilder.js +20 -0
- package/dist/orchestrator/src/cli/adapters/CommandPlanner.js +164 -0
- package/dist/orchestrator/src/cli/adapters/CommandReviewer.js +32 -0
- package/dist/orchestrator/src/cli/adapters/CommandTester.js +33 -0
- package/dist/orchestrator/src/cli/adapters/index.js +4 -0
- package/dist/orchestrator/src/cli/config/userConfig.js +28 -0
- package/dist/orchestrator/src/cli/doctor.js +48 -0
- package/dist/orchestrator/src/cli/events/runEvents.js +84 -0
- package/dist/orchestrator/src/cli/exec/command.js +56 -0
- package/dist/orchestrator/src/cli/exec/context.js +108 -0
- package/dist/orchestrator/src/cli/exec/experience.js +77 -0
- package/dist/orchestrator/src/cli/exec/finalization.js +140 -0
- package/dist/orchestrator/src/cli/exec/learning.js +62 -0
- package/dist/orchestrator/src/cli/exec/stageRunner.js +71 -0
- package/dist/orchestrator/src/cli/exec/summary.js +109 -0
- package/dist/orchestrator/src/cli/exec/telemetry.js +18 -0
- package/dist/orchestrator/src/cli/exec/tfgrpo.js +200 -0
- package/dist/orchestrator/src/cli/exec/tfgrpoArtifacts.js +19 -0
- package/dist/orchestrator/src/cli/exec/types.js +1 -0
- package/dist/orchestrator/src/cli/init.js +64 -0
- package/dist/orchestrator/src/cli/mcp.js +124 -0
- package/dist/orchestrator/src/cli/metrics/metricsAggregator.js +404 -0
- package/dist/orchestrator/src/cli/metrics/metricsRecorder.js +138 -0
- package/dist/orchestrator/src/cli/orchestrator.js +554 -0
- package/dist/orchestrator/src/cli/pipelines/defaultDiagnostics.js +32 -0
- package/dist/orchestrator/src/cli/pipelines/designReference.js +72 -0
- package/dist/orchestrator/src/cli/pipelines/hiFiDesignToolkit.js +71 -0
- package/dist/orchestrator/src/cli/pipelines/index.js +34 -0
- package/dist/orchestrator/src/cli/run/environment.js +24 -0
- package/dist/orchestrator/src/cli/run/manifest.js +367 -0
- package/dist/orchestrator/src/cli/run/manifestPersister.js +88 -0
- package/dist/orchestrator/src/cli/run/runPaths.js +30 -0
- package/dist/orchestrator/src/cli/selfCheck.js +12 -0
- package/dist/orchestrator/src/cli/services/commandRunner.js +420 -0
- package/dist/orchestrator/src/cli/services/controlPlaneService.js +107 -0
- package/dist/orchestrator/src/cli/services/execRuntime.js +69 -0
- package/dist/orchestrator/src/cli/services/pipelineResolver.js +47 -0
- package/dist/orchestrator/src/cli/services/runPreparation.js +82 -0
- package/dist/orchestrator/src/cli/services/runSummaryWriter.js +35 -0
- package/dist/orchestrator/src/cli/services/schedulerService.js +42 -0
- package/dist/orchestrator/src/cli/tasks/taskMetadata.js +19 -0
- package/dist/orchestrator/src/cli/telemetry/schema.js +8 -0
- package/dist/orchestrator/src/cli/types.js +1 -0
- package/dist/orchestrator/src/cli/ui/HudApp.js +112 -0
- package/dist/orchestrator/src/cli/ui/controller.js +26 -0
- package/dist/orchestrator/src/cli/ui/store.js +240 -0
- package/dist/orchestrator/src/cli/utils/enforcementMode.js +12 -0
- package/dist/orchestrator/src/cli/utils/fs.js +8 -0
- package/dist/orchestrator/src/cli/utils/interactive.js +25 -0
- package/dist/orchestrator/src/cli/utils/jsonlWriter.js +10 -0
- package/dist/orchestrator/src/cli/utils/optionalDeps.js +30 -0
- package/dist/orchestrator/src/cli/utils/packageInfo.js +25 -0
- package/dist/orchestrator/src/cli/utils/planFormatter.js +49 -0
- package/dist/orchestrator/src/cli/utils/runId.js +7 -0
- package/dist/orchestrator/src/cli/utils/specGuardRunner.js +26 -0
- package/dist/orchestrator/src/cli/utils/strings.js +8 -0
- package/dist/orchestrator/src/cli/utils/time.js +6 -0
- package/dist/orchestrator/src/control-plane/drift-reporter.js +109 -0
- package/dist/orchestrator/src/control-plane/index.js +3 -0
- package/dist/orchestrator/src/control-plane/request-builder.js +217 -0
- package/dist/orchestrator/src/control-plane/types.js +1 -0
- package/dist/orchestrator/src/control-plane/validator.js +50 -0
- package/dist/orchestrator/src/credentials/CredentialBroker.js +1 -0
- package/dist/orchestrator/src/events/EventBus.js +25 -0
- package/dist/orchestrator/src/learning/crystalizer.js +108 -0
- package/dist/orchestrator/src/learning/harvester.js +146 -0
- package/dist/orchestrator/src/learning/manifest.js +56 -0
- package/dist/orchestrator/src/learning/runner.js +177 -0
- package/dist/orchestrator/src/learning/validator.js +164 -0
- package/dist/orchestrator/src/logger.js +20 -0
- package/dist/orchestrator/src/manager.js +388 -0
- package/dist/orchestrator/src/persistence/ArtifactStager.js +95 -0
- package/dist/orchestrator/src/persistence/ExperienceStore.js +210 -0
- package/dist/orchestrator/src/persistence/PersistenceCoordinator.js +65 -0
- package/dist/orchestrator/src/persistence/RunManifestWriter.js +23 -0
- package/dist/orchestrator/src/persistence/TaskStateStore.js +172 -0
- package/dist/orchestrator/src/persistence/identifierGuards.js +1 -0
- package/dist/orchestrator/src/persistence/lockFile.js +26 -0
- package/dist/orchestrator/src/persistence/sanitizeIdentifier.js +26 -0
- package/dist/orchestrator/src/persistence/sanitizeRunId.js +8 -0
- package/dist/orchestrator/src/persistence/sanitizeTaskId.js +8 -0
- package/dist/orchestrator/src/persistence/writeAtomicFile.js +4 -0
- package/dist/orchestrator/src/privacy/guard.js +111 -0
- package/dist/orchestrator/src/scheduler/index.js +1 -0
- package/dist/orchestrator/src/scheduler/plan.js +171 -0
- package/dist/orchestrator/src/scheduler/types.js +1 -0
- package/dist/orchestrator/src/sync/CloudRunsClient.js +1 -0
- package/dist/orchestrator/src/sync/CloudRunsHttpClient.js +82 -0
- package/dist/orchestrator/src/sync/CloudSyncWorker.js +206 -0
- package/dist/orchestrator/src/sync/createCloudSyncWorker.js +15 -0
- package/dist/orchestrator/src/types.js +1 -0
- package/dist/orchestrator/src/utils/atomicWrite.js +15 -0
- package/dist/orchestrator/src/utils/errorMessage.js +14 -0
- package/dist/orchestrator/src/utils/executionMode.js +69 -0
- package/dist/packages/control-plane-schemas/src/index.js +1 -0
- package/dist/packages/control-plane-schemas/src/run-request.js +548 -0
- package/dist/packages/orchestrator/src/exec/handle-service.js +203 -0
- package/dist/packages/orchestrator/src/exec/session-manager.js +147 -0
- package/dist/packages/orchestrator/src/exec/unified-exec.js +432 -0
- package/dist/packages/orchestrator/src/index.js +3 -0
- package/dist/packages/orchestrator/src/instructions/loader.js +101 -0
- package/dist/packages/orchestrator/src/instructions/promptPacks.js +151 -0
- package/dist/packages/orchestrator/src/notifications/index.js +74 -0
- package/dist/packages/orchestrator/src/telemetry/otel-exporter.js +142 -0
- package/dist/packages/orchestrator/src/tool-orchestrator.js +161 -0
- package/dist/packages/sdk-node/src/orchestrator.js +195 -0
- package/dist/packages/shared/config/designConfig.js +495 -0
- package/dist/packages/shared/config/env.js +37 -0
- package/dist/packages/shared/config/index.js +2 -0
- package/dist/packages/shared/design-artifacts/writer.js +221 -0
- package/dist/packages/shared/events/serializer.js +84 -0
- package/dist/packages/shared/events/types.js +1 -0
- package/dist/packages/shared/manifest/artifactUtils.js +36 -0
- package/dist/packages/shared/manifest/designArtifacts.js +665 -0
- package/dist/packages/shared/manifest/fileIO.js +29 -0
- package/dist/packages/shared/manifest/toolRuns.js +78 -0
- package/dist/packages/shared/manifest/toolkitArtifacts.js +223 -0
- package/dist/packages/shared/manifest/types.js +5 -0
- package/dist/packages/shared/manifest/validator.js +73 -0
- package/dist/packages/shared/manifest/writer.js +2 -0
- package/dist/packages/shared/streams/stdio.js +112 -0
- package/dist/scripts/design/pipeline/advanced-assets.js +466 -0
- package/dist/scripts/design/pipeline/componentize.js +74 -0
- package/dist/scripts/design/pipeline/context.js +34 -0
- package/dist/scripts/design/pipeline/extract.js +249 -0
- package/dist/scripts/design/pipeline/optionalDeps.js +107 -0
- package/dist/scripts/design/pipeline/prepare.js +46 -0
- package/dist/scripts/design/pipeline/reference.js +94 -0
- package/dist/scripts/design/pipeline/state.js +206 -0
- package/dist/scripts/design/pipeline/toolkit/common.js +94 -0
- package/dist/scripts/design/pipeline/toolkit/extract.js +258 -0
- package/dist/scripts/design/pipeline/toolkit/publish.js +202 -0
- package/dist/scripts/design/pipeline/toolkit/publishActions.js +12 -0
- package/dist/scripts/design/pipeline/toolkit/reference.js +846 -0
- package/dist/scripts/design/pipeline/toolkit/snapshot.js +882 -0
- package/dist/scripts/design/pipeline/toolkit/tokens.js +456 -0
- package/dist/scripts/design/pipeline/visual-regression.js +137 -0
- package/dist/scripts/design/pipeline/write-artifacts.js +61 -0
- package/package.json +97 -0
- package/schemas/manifest.json +1064 -0
- package/templates/README.md +12 -0
- package/templates/codex/mcp-client.json +8 -0
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { existsSync, readFileSync } from 'node:fs';
|
|
2
|
+
import { dirname, join } from 'node:path';
|
|
3
|
+
import { fileURLToPath } from 'node:url';
|
|
4
|
+
export function findPackageRoot(fromUrl = import.meta.url) {
|
|
5
|
+
let current = dirname(fileURLToPath(fromUrl));
|
|
6
|
+
while (current) {
|
|
7
|
+
const candidate = join(current, 'package.json');
|
|
8
|
+
if (existsSync(candidate)) {
|
|
9
|
+
return current;
|
|
10
|
+
}
|
|
11
|
+
const parent = dirname(current);
|
|
12
|
+
if (parent === current) {
|
|
13
|
+
current = null;
|
|
14
|
+
continue;
|
|
15
|
+
}
|
|
16
|
+
current = parent;
|
|
17
|
+
}
|
|
18
|
+
throw new Error('Unable to locate package.json');
|
|
19
|
+
}
|
|
20
|
+
export function loadPackageInfo(fromUrl = import.meta.url) {
|
|
21
|
+
const root = findPackageRoot(fromUrl);
|
|
22
|
+
const pkgPath = join(root, 'package.json');
|
|
23
|
+
const raw = readFileSync(pkgPath, 'utf8');
|
|
24
|
+
return JSON.parse(raw);
|
|
25
|
+
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
export function formatPlanPreview(result) {
|
|
2
|
+
const lines = [];
|
|
3
|
+
lines.push(`Pipeline: ${result.pipeline.id} — ${result.pipeline.title} [${result.pipeline.source}]`);
|
|
4
|
+
if (result.pipeline.description) {
|
|
5
|
+
lines.push(`Description: ${result.pipeline.description}`);
|
|
6
|
+
}
|
|
7
|
+
lines.push('Stages:');
|
|
8
|
+
if (result.stages.length === 0) {
|
|
9
|
+
lines.push(' (none)');
|
|
10
|
+
}
|
|
11
|
+
else {
|
|
12
|
+
for (const stage of result.stages) {
|
|
13
|
+
lines.push(...formatStage(stage));
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
if (result.plan.notes) {
|
|
17
|
+
lines.push(`Notes: ${result.plan.notes}`);
|
|
18
|
+
}
|
|
19
|
+
return lines.join('\n');
|
|
20
|
+
}
|
|
21
|
+
function formatStage(stage) {
|
|
22
|
+
if (stage.kind === 'command') {
|
|
23
|
+
const lines = [
|
|
24
|
+
` ${stage.index}. [command] ${stage.title} (${stage.id})`,
|
|
25
|
+
` command: ${stage.command}`
|
|
26
|
+
];
|
|
27
|
+
if (stage.cwd) {
|
|
28
|
+
lines.push(` cwd: ${stage.cwd}`);
|
|
29
|
+
}
|
|
30
|
+
if (stage.env && Object.keys(stage.env).length > 0) {
|
|
31
|
+
lines.push(` env: ${Object.entries(stage.env).map(([key, value]) => `${key}=${value}`).join(', ')}`);
|
|
32
|
+
}
|
|
33
|
+
if (stage.allowFailure) {
|
|
34
|
+
lines.push(' allowFailure: true');
|
|
35
|
+
}
|
|
36
|
+
if (stage.summaryHint) {
|
|
37
|
+
lines.push(` summaryHint: ${stage.summaryHint}`);
|
|
38
|
+
}
|
|
39
|
+
return lines;
|
|
40
|
+
}
|
|
41
|
+
const lines = [
|
|
42
|
+
` ${stage.index}. [subpipeline] ${stage.title} (${stage.id})`,
|
|
43
|
+
` pipeline: ${stage.pipeline}`
|
|
44
|
+
];
|
|
45
|
+
if (stage.optional) {
|
|
46
|
+
lines.push(' optional: true');
|
|
47
|
+
}
|
|
48
|
+
return lines;
|
|
49
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { spawn } from 'node:child_process';
|
|
2
|
+
import { existsSync } from 'node:fs';
|
|
3
|
+
import { join } from 'node:path';
|
|
4
|
+
import process from 'node:process';
|
|
5
|
+
import { logger } from '../../logger.js';
|
|
6
|
+
const specGuardPath = join(process.cwd(), 'scripts', 'spec-guard.mjs');
|
|
7
|
+
if (!existsSync(specGuardPath)) {
|
|
8
|
+
logger.warn(`[spec-guard] skipped: ${specGuardPath} not found`);
|
|
9
|
+
process.exit(0);
|
|
10
|
+
}
|
|
11
|
+
const child = spawn(process.execPath, [specGuardPath, ...process.argv.slice(2)], {
|
|
12
|
+
stdio: 'inherit'
|
|
13
|
+
});
|
|
14
|
+
child.on('error', (error) => {
|
|
15
|
+
logger.error(`[spec-guard] failed: ${error.message}`);
|
|
16
|
+
process.exit(1);
|
|
17
|
+
});
|
|
18
|
+
child.on('exit', (code, signal) => {
|
|
19
|
+
if (typeof code === 'number') {
|
|
20
|
+
process.exit(code);
|
|
21
|
+
}
|
|
22
|
+
if (signal) {
|
|
23
|
+
logger.error(`[spec-guard] exited with signal ${signal}`);
|
|
24
|
+
}
|
|
25
|
+
process.exit(1);
|
|
26
|
+
});
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export function slugify(value, fallback = 'command') {
|
|
2
|
+
const cleaned = value.trim().replace(/[^a-zA-Z0-9]+/g, '-').replace(/-+/g, '-');
|
|
3
|
+
const normalized = cleaned.replace(/^-|-$/g, '');
|
|
4
|
+
if (!normalized) {
|
|
5
|
+
return fallback;
|
|
6
|
+
}
|
|
7
|
+
return normalized.slice(0, 80);
|
|
8
|
+
}
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
import { mkdir, readFile, writeFile } from 'node:fs/promises';
|
|
2
|
+
import { dirname, join } from 'node:path';
|
|
3
|
+
import { CONTROL_PLANE_RUN_REQUEST_SCHEMA, CONTROL_PLANE_RUN_REQUEST_VERSION } from '../../../packages/control-plane-schemas/src/index.js';
|
|
4
|
+
const DEFAULT_MAX_ENTRIES = 50;
|
|
5
|
+
export class ControlPlaneDriftReporter {
|
|
6
|
+
options;
|
|
7
|
+
reportPath;
|
|
8
|
+
read;
|
|
9
|
+
write;
|
|
10
|
+
ensureDir;
|
|
11
|
+
maxEntries;
|
|
12
|
+
constructor(options) {
|
|
13
|
+
this.options = options;
|
|
14
|
+
this.reportPath =
|
|
15
|
+
options.outputPath ??
|
|
16
|
+
join(options.repoRoot, 'out', options.taskId, 'control-plane', 'drift.json');
|
|
17
|
+
this.read = options.fs?.readFile ?? readFile;
|
|
18
|
+
this.write = options.fs?.writeFile ?? writeFile;
|
|
19
|
+
this.ensureDir = options.fs?.mkdir ?? mkdir;
|
|
20
|
+
this.maxEntries = options.maxEntries ?? DEFAULT_MAX_ENTRIES;
|
|
21
|
+
}
|
|
22
|
+
get path() {
|
|
23
|
+
return this.reportPath;
|
|
24
|
+
}
|
|
25
|
+
async record(entry) {
|
|
26
|
+
const state = await this.load();
|
|
27
|
+
state.total_samples += 1;
|
|
28
|
+
if (!entry.valid) {
|
|
29
|
+
state.invalid_samples += 1;
|
|
30
|
+
}
|
|
31
|
+
state.invalid_rate =
|
|
32
|
+
state.total_samples === 0 ? 0 : state.invalid_samples / state.total_samples;
|
|
33
|
+
state.last_sampled_at = entry.timestamp;
|
|
34
|
+
state.mode = entry.mode;
|
|
35
|
+
if (!entry.valid) {
|
|
36
|
+
state.entries.push({
|
|
37
|
+
request_id: entry.requestId,
|
|
38
|
+
timestamp: entry.timestamp,
|
|
39
|
+
status: 'failed',
|
|
40
|
+
mode: entry.mode,
|
|
41
|
+
errors: entry.errors.map((error) => ({ ...error }))
|
|
42
|
+
});
|
|
43
|
+
}
|
|
44
|
+
else if (state.entries.length === 0) {
|
|
45
|
+
// Keep at least one success sample for context when no failures are present.
|
|
46
|
+
state.entries.push({
|
|
47
|
+
request_id: entry.requestId,
|
|
48
|
+
timestamp: entry.timestamp,
|
|
49
|
+
status: 'passed',
|
|
50
|
+
mode: entry.mode
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
if (state.entries.length > this.maxEntries) {
|
|
54
|
+
state.entries.splice(0, state.entries.length - this.maxEntries);
|
|
55
|
+
}
|
|
56
|
+
await this.persist(state);
|
|
57
|
+
return {
|
|
58
|
+
mode: state.mode,
|
|
59
|
+
absoluteReportPath: this.reportPath,
|
|
60
|
+
totalSamples: state.total_samples,
|
|
61
|
+
invalidSamples: state.invalid_samples,
|
|
62
|
+
invalidRate: state.invalid_rate,
|
|
63
|
+
lastSampledAt: state.last_sampled_at
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
async load() {
|
|
67
|
+
try {
|
|
68
|
+
const raw = await this.read(this.reportPath, 'utf8');
|
|
69
|
+
const parsed = JSON.parse(raw);
|
|
70
|
+
if (!parsed || typeof parsed !== 'object') {
|
|
71
|
+
return this.createInitialState();
|
|
72
|
+
}
|
|
73
|
+
const entries = Array.isArray(parsed.entries) ? parsed.entries : [];
|
|
74
|
+
return {
|
|
75
|
+
schema_id: parsed.schema_id ?? CONTROL_PLANE_RUN_REQUEST_SCHEMA,
|
|
76
|
+
schema_version: parsed.schema_version ?? CONTROL_PLANE_RUN_REQUEST_VERSION,
|
|
77
|
+
total_samples: typeof parsed.total_samples === 'number' ? parsed.total_samples : 0,
|
|
78
|
+
invalid_samples: typeof parsed.invalid_samples === 'number' ? parsed.invalid_samples : 0,
|
|
79
|
+
invalid_rate: typeof parsed.invalid_rate === 'number' ? parsed.invalid_rate : 0,
|
|
80
|
+
last_sampled_at: typeof parsed.last_sampled_at === 'string' ? parsed.last_sampled_at : null,
|
|
81
|
+
mode: parsed.mode ?? 'shadow',
|
|
82
|
+
entries
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
catch (error) {
|
|
86
|
+
if (error?.code === 'ENOENT') {
|
|
87
|
+
return this.createInitialState();
|
|
88
|
+
}
|
|
89
|
+
throw error;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
async persist(state) {
|
|
93
|
+
await this.ensureDir(dirname(this.reportPath), { recursive: true });
|
|
94
|
+
const serialized = JSON.stringify(state, null, 2);
|
|
95
|
+
await this.write(this.reportPath, `${serialized}\n`, 'utf8');
|
|
96
|
+
}
|
|
97
|
+
createInitialState() {
|
|
98
|
+
return {
|
|
99
|
+
schema_id: CONTROL_PLANE_RUN_REQUEST_SCHEMA,
|
|
100
|
+
schema_version: CONTROL_PLANE_RUN_REQUEST_VERSION,
|
|
101
|
+
total_samples: 0,
|
|
102
|
+
invalid_samples: 0,
|
|
103
|
+
invalid_rate: 0,
|
|
104
|
+
last_sampled_at: null,
|
|
105
|
+
mode: 'shadow',
|
|
106
|
+
entries: []
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
}
|
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
import { CONTROL_PLANE_RUN_REQUEST_SCHEMA, CONTROL_PLANE_RUN_REQUEST_VERSION } from '../../../packages/control-plane-schemas/src/index.js';
|
|
2
|
+
import { logger } from '../logger.js';
|
|
3
|
+
const DEFAULT_HEARTBEAT_INTERVAL_SECONDS = 30;
|
|
4
|
+
const DEFAULT_HEARTBEAT_TIMEOUT_SECONDS = 120;
|
|
5
|
+
const DEFAULT_RECOVERY_RETRIES = 3;
|
|
6
|
+
const DEFAULT_MAX_SUBSCRIBERS = 8;
|
|
7
|
+
const DEFAULT_BACKPRESSURE_MS = 250;
|
|
8
|
+
const DEFAULT_POLICY_VERSION = '2025-10-01';
|
|
9
|
+
const DEFAULT_METRIC_INTERVAL_SECONDS = 30;
|
|
10
|
+
const DEFAULT_REQUIRED_DIMENSIONS = ['instanceId', 'phase', 'status'];
|
|
11
|
+
const DEFAULT_TFGRPO_EPOCHS = 3;
|
|
12
|
+
const DEFAULT_TFGRPO_SAMPLE_SIZE = 100;
|
|
13
|
+
const DEFAULT_TFGRPO_TRAIN_TEMP = 0.7;
|
|
14
|
+
const DEFAULT_TFGRPO_EVAL_TEMP = 0.3;
|
|
15
|
+
const DEFAULT_TFGRPO_GROUP_SIZE = 2;
|
|
16
|
+
export function buildRunRequestV2(options) {
|
|
17
|
+
const { runId, task, pipeline, manifest, env } = options;
|
|
18
|
+
const now = options.now ?? (() => new Date());
|
|
19
|
+
const capabilities = pipeline.tags && pipeline.tags.length > 0 ? [...pipeline.tags] : ['general'];
|
|
20
|
+
const stages = pipeline.stages.map((stage) => {
|
|
21
|
+
if (stage.kind === 'command') {
|
|
22
|
+
const entry = {
|
|
23
|
+
id: stage.id,
|
|
24
|
+
kind: 'command',
|
|
25
|
+
title: stage.title,
|
|
26
|
+
optional: Boolean(stage.allowFailure)
|
|
27
|
+
};
|
|
28
|
+
if (stage.session?.id) {
|
|
29
|
+
entry.capabilities = [`session:${stage.session.id}`];
|
|
30
|
+
}
|
|
31
|
+
return entry;
|
|
32
|
+
}
|
|
33
|
+
return {
|
|
34
|
+
id: stage.id,
|
|
35
|
+
kind: 'subpipeline',
|
|
36
|
+
title: stage.title,
|
|
37
|
+
optional: Boolean(stage.optional)
|
|
38
|
+
};
|
|
39
|
+
});
|
|
40
|
+
const fanOut = capabilities.map((capability) => ({
|
|
41
|
+
capability,
|
|
42
|
+
weight: 1,
|
|
43
|
+
maxConcurrency: capability === 'general' ? 2 : 1
|
|
44
|
+
}));
|
|
45
|
+
const fanOutCapacity = computeFanOutCapacity(fanOut);
|
|
46
|
+
const groupSize = resolveGroupSize(manifest);
|
|
47
|
+
const scheduleBounds = resolveScheduleBounds({
|
|
48
|
+
defaultMin: 1,
|
|
49
|
+
defaultMax: Math.max(1, fanOut.length),
|
|
50
|
+
fanOutCapacity,
|
|
51
|
+
groupSize
|
|
52
|
+
});
|
|
53
|
+
const constraints = {
|
|
54
|
+
privacyLevel: 'standard',
|
|
55
|
+
policyVersion: DEFAULT_POLICY_VERSION
|
|
56
|
+
};
|
|
57
|
+
const metadata = {
|
|
58
|
+
artifactRoot: manifest.artifact_root,
|
|
59
|
+
runsRoot: env.runsRoot,
|
|
60
|
+
outRoot: env.outRoot,
|
|
61
|
+
pipelineId: pipeline.id,
|
|
62
|
+
taskSlug: task.metadata?.slug ?? null
|
|
63
|
+
};
|
|
64
|
+
const tfgrpoMetadata = resolveTfgrpoMetadata(groupSize);
|
|
65
|
+
if (tfgrpoMetadata) {
|
|
66
|
+
metadata.tfgrpo = tfgrpoMetadata;
|
|
67
|
+
}
|
|
68
|
+
return {
|
|
69
|
+
schema: CONTROL_PLANE_RUN_REQUEST_SCHEMA,
|
|
70
|
+
version: CONTROL_PLANE_RUN_REQUEST_VERSION,
|
|
71
|
+
requestId: runId,
|
|
72
|
+
task: {
|
|
73
|
+
id: task.id,
|
|
74
|
+
slug: task.metadata?.slug ? String(task.metadata.slug) : undefined,
|
|
75
|
+
title: task.title,
|
|
76
|
+
description: task.description,
|
|
77
|
+
metadata: task.metadata,
|
|
78
|
+
tags: capabilities
|
|
79
|
+
},
|
|
80
|
+
pipeline: {
|
|
81
|
+
id: pipeline.id,
|
|
82
|
+
version: '1.0.0',
|
|
83
|
+
title: pipeline.title,
|
|
84
|
+
capabilities,
|
|
85
|
+
stages
|
|
86
|
+
},
|
|
87
|
+
schedule: {
|
|
88
|
+
strategy: 'auto',
|
|
89
|
+
minInstances: scheduleBounds.minInstances,
|
|
90
|
+
maxInstances: scheduleBounds.maxInstances,
|
|
91
|
+
fanOut,
|
|
92
|
+
recovery: {
|
|
93
|
+
heartbeatIntervalSeconds: DEFAULT_HEARTBEAT_INTERVAL_SECONDS,
|
|
94
|
+
missingHeartbeatTimeoutSeconds: DEFAULT_HEARTBEAT_TIMEOUT_SECONDS,
|
|
95
|
+
maxRetries: DEFAULT_RECOVERY_RETRIES
|
|
96
|
+
}
|
|
97
|
+
},
|
|
98
|
+
streaming: {
|
|
99
|
+
handles: true,
|
|
100
|
+
resumeSupported: true,
|
|
101
|
+
observers: {
|
|
102
|
+
maxSubscribers: DEFAULT_MAX_SUBSCRIBERS,
|
|
103
|
+
defaultBackpressureMs: DEFAULT_BACKPRESSURE_MS
|
|
104
|
+
}
|
|
105
|
+
},
|
|
106
|
+
constraints,
|
|
107
|
+
metrics: {
|
|
108
|
+
emitIntervalSeconds: DEFAULT_METRIC_INTERVAL_SECONDS,
|
|
109
|
+
requiredDimensions: DEFAULT_REQUIRED_DIMENSIONS
|
|
110
|
+
},
|
|
111
|
+
requestedAt: now().toISOString(),
|
|
112
|
+
requestedBy: options.requestedBy ?? {
|
|
113
|
+
actorId: 'codex-cli',
|
|
114
|
+
channel: 'cli',
|
|
115
|
+
name: 'Codex CLI'
|
|
116
|
+
},
|
|
117
|
+
metadata
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
function resolveTfgrpoMetadata(groupSize) {
|
|
121
|
+
const sampleSize = parsePositiveInteger(process.env.TFGRPO_SAMPLE_SIZE);
|
|
122
|
+
const epochs = parsePositiveInteger(process.env.TFGRPO_EPOCHS);
|
|
123
|
+
const trainTemp = parseFloatSafe(process.env.TFGRPO_TRAIN_TEMP);
|
|
124
|
+
const evalTemp = parseFloatSafe(process.env.TFGRPO_EVAL_TEMP);
|
|
125
|
+
const hasGroupSize = groupSize !== null;
|
|
126
|
+
if (!hasGroupSize && sampleSize === null && epochs === null && trainTemp === null && evalTemp === null) {
|
|
127
|
+
return null;
|
|
128
|
+
}
|
|
129
|
+
const metadata = {
|
|
130
|
+
sampleSize: sampleSize ?? DEFAULT_TFGRPO_SAMPLE_SIZE,
|
|
131
|
+
epochs: epochs ?? DEFAULT_TFGRPO_EPOCHS,
|
|
132
|
+
temperature: {
|
|
133
|
+
train: trainTemp ?? DEFAULT_TFGRPO_TRAIN_TEMP,
|
|
134
|
+
eval: evalTemp ?? DEFAULT_TFGRPO_EVAL_TEMP
|
|
135
|
+
}
|
|
136
|
+
};
|
|
137
|
+
if (hasGroupSize && groupSize !== null) {
|
|
138
|
+
metadata.groupSize = groupSize;
|
|
139
|
+
}
|
|
140
|
+
return metadata;
|
|
141
|
+
}
|
|
142
|
+
function parsePositiveInteger(value) {
|
|
143
|
+
if (!value) {
|
|
144
|
+
return null;
|
|
145
|
+
}
|
|
146
|
+
const parsed = Number.parseInt(value, 10);
|
|
147
|
+
if (Number.isNaN(parsed) || parsed <= 0) {
|
|
148
|
+
return null;
|
|
149
|
+
}
|
|
150
|
+
return parsed;
|
|
151
|
+
}
|
|
152
|
+
function parseFloatSafe(value) {
|
|
153
|
+
if (!value) {
|
|
154
|
+
return null;
|
|
155
|
+
}
|
|
156
|
+
const parsed = Number.parseFloat(value);
|
|
157
|
+
if (Number.isNaN(parsed)) {
|
|
158
|
+
return null;
|
|
159
|
+
}
|
|
160
|
+
return parsed;
|
|
161
|
+
}
|
|
162
|
+
function computeFanOutCapacity(fanOut) {
|
|
163
|
+
return fanOut.reduce((total, entry) => {
|
|
164
|
+
const capacity = entry?.maxConcurrency ?? 1;
|
|
165
|
+
return total + Math.max(1, capacity);
|
|
166
|
+
}, 0);
|
|
167
|
+
}
|
|
168
|
+
function resolveGroupSize(manifest, env = process.env) {
|
|
169
|
+
const envSize = parsePositiveInteger(env.TFGRPO_GROUP_SIZE);
|
|
170
|
+
if (envSize !== null) {
|
|
171
|
+
return envSize;
|
|
172
|
+
}
|
|
173
|
+
const manifestSize = typeof manifest.tfgrpo?.group_size === 'number' && Number.isFinite(manifest.tfgrpo.group_size)
|
|
174
|
+
? manifest.tfgrpo.group_size
|
|
175
|
+
: null;
|
|
176
|
+
if (manifestSize !== null && manifestSize > 0) {
|
|
177
|
+
return manifestSize;
|
|
178
|
+
}
|
|
179
|
+
if (isFeatureEnabled(env.FEATURE_TFGRPO_GROUP)) {
|
|
180
|
+
return DEFAULT_TFGRPO_GROUP_SIZE;
|
|
181
|
+
}
|
|
182
|
+
return null;
|
|
183
|
+
}
|
|
184
|
+
function resolveScheduleBounds(params) {
|
|
185
|
+
if (params.groupSize === null) {
|
|
186
|
+
const maxInstances = Math.min(params.defaultMax, params.fanOutCapacity);
|
|
187
|
+
return {
|
|
188
|
+
minInstances: params.defaultMin,
|
|
189
|
+
maxInstances: Math.max(params.defaultMin, maxInstances)
|
|
190
|
+
};
|
|
191
|
+
}
|
|
192
|
+
if (params.groupSize < 2) {
|
|
193
|
+
logGroupGuard(`TF-GRPO groupSize ${params.groupSize} violates guardrail (must be ≥ 2). ` +
|
|
194
|
+
'Set TFGRPO_GROUP_SIZE>=2 or disable FEATURE_TFGRPO_GROUP.');
|
|
195
|
+
throw new Error('TF-GRPO groupSize guardrail violated (expected ≥ 2).');
|
|
196
|
+
}
|
|
197
|
+
if (params.groupSize > params.fanOutCapacity) {
|
|
198
|
+
logGroupGuard(`TF-GRPO groupSize ${params.groupSize} exceeds available fan-out capacity (${params.fanOutCapacity}). ` +
|
|
199
|
+
'Increase pipeline fan-out or lower TFGRPO_GROUP_SIZE.');
|
|
200
|
+
throw new Error('TF-GRPO groupSize exceeds available scheduling capacity.');
|
|
201
|
+
}
|
|
202
|
+
const cappedMax = Math.min(params.fanOutCapacity, params.groupSize);
|
|
203
|
+
return {
|
|
204
|
+
minInstances: params.groupSize,
|
|
205
|
+
maxInstances: cappedMax
|
|
206
|
+
};
|
|
207
|
+
}
|
|
208
|
+
function isFeatureEnabled(value) {
|
|
209
|
+
if (!value) {
|
|
210
|
+
return false;
|
|
211
|
+
}
|
|
212
|
+
const normalized = value.trim().toLowerCase();
|
|
213
|
+
return ['1', 'true', 'on', 'yes'].includes(normalized);
|
|
214
|
+
}
|
|
215
|
+
function logGroupGuard(message) {
|
|
216
|
+
logger.error(`[control-plane.guard] ${message}`);
|
|
217
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import { validateRunRequestV2 } from '../../../packages/control-plane-schemas/src/index.js';
|
|
2
|
+
const DEFAULT_MODE = 'shadow';
|
|
3
|
+
export class ControlPlaneValidationError extends Error {
|
|
4
|
+
result;
|
|
5
|
+
constructor(message, result) {
|
|
6
|
+
super(message);
|
|
7
|
+
this.result = result;
|
|
8
|
+
this.name = 'ControlPlaneValidationError';
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
export class ControlPlaneValidator {
|
|
12
|
+
mode;
|
|
13
|
+
driftReporter;
|
|
14
|
+
now;
|
|
15
|
+
constructor(options = {}) {
|
|
16
|
+
this.mode = options.mode ?? DEFAULT_MODE;
|
|
17
|
+
this.driftReporter = options.driftReporter;
|
|
18
|
+
this.now = options.now ?? (() => new Date());
|
|
19
|
+
}
|
|
20
|
+
async validate(request) {
|
|
21
|
+
const timestamp = this.now().toISOString();
|
|
22
|
+
const evaluation = validateRunRequestV2(request);
|
|
23
|
+
const canonicalRequest = evaluation.valid ? evaluation.value : request;
|
|
24
|
+
const errors = evaluation.valid ? [] : evaluation.errors.map((error) => ({ ...error }));
|
|
25
|
+
const drift = this.driftReporter
|
|
26
|
+
? await this.driftReporter.record({
|
|
27
|
+
requestId: canonicalRequest.requestId,
|
|
28
|
+
timestamp,
|
|
29
|
+
mode: this.mode,
|
|
30
|
+
valid: evaluation.valid,
|
|
31
|
+
errors
|
|
32
|
+
})
|
|
33
|
+
: undefined;
|
|
34
|
+
const outcome = {
|
|
35
|
+
mode: this.mode,
|
|
36
|
+
status: evaluation.valid ? 'passed' : 'failed',
|
|
37
|
+
timestamp,
|
|
38
|
+
errors,
|
|
39
|
+
drift
|
|
40
|
+
};
|
|
41
|
+
const result = {
|
|
42
|
+
request: canonicalRequest,
|
|
43
|
+
outcome
|
|
44
|
+
};
|
|
45
|
+
if (!evaluation.valid && this.mode === 'enforce') {
|
|
46
|
+
throw new ControlPlaneValidationError(`Control plane validation failed (${errors.length} error${errors.length === 1 ? '' : 's'}).`, result);
|
|
47
|
+
}
|
|
48
|
+
return result;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { EventEmitter } from 'node:events';
|
|
2
|
+
/**
|
|
3
|
+
* Lightweight typed event bus so the manager can notify downstream subscribers
|
|
4
|
+
* (e.g. cloud-sync worker) whenever orchestration milestones complete.
|
|
5
|
+
*/
|
|
6
|
+
export class EventBus {
|
|
7
|
+
emitter = new EventEmitter({ captureRejections: false });
|
|
8
|
+
emit(event) {
|
|
9
|
+
this.emitter.emit(event.type, event);
|
|
10
|
+
}
|
|
11
|
+
on(event, listener) {
|
|
12
|
+
const wrapped = (payload) => listener(payload);
|
|
13
|
+
this.emitter.on(event, wrapped);
|
|
14
|
+
return () => {
|
|
15
|
+
this.emitter.off(event, wrapped);
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
once(event, listener) {
|
|
19
|
+
const wrapped = (payload) => listener(payload);
|
|
20
|
+
this.emitter.once(event, wrapped);
|
|
21
|
+
return () => {
|
|
22
|
+
this.emitter.off(event, wrapped);
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
}
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import { execFile } from 'node:child_process';
|
|
2
|
+
import { promisify } from 'node:util';
|
|
3
|
+
import { mkdtemp, readFile, writeFile, mkdir, rm } from 'node:fs/promises';
|
|
4
|
+
import { tmpdir } from 'node:os';
|
|
5
|
+
import { join, relative } from 'node:path';
|
|
6
|
+
import { isoTimestamp } from '../cli/utils/time.js';
|
|
7
|
+
import { slugify } from '../cli/utils/strings.js';
|
|
8
|
+
import { appendLearningAlert, ensureLearningSection } from './manifest.js';
|
|
9
|
+
import { computePromptPackStamp, loadPromptPacks } from '../../../packages/orchestrator/src/instructions/promptPacks.js';
|
|
10
|
+
const DEFAULT_PROMPT_PACK = 'crystalizer-v1';
|
|
11
|
+
const DEFAULT_OUTPUT_DIR = '.agent/patterns/candidates';
|
|
12
|
+
const DEFAULT_MODEL = process.env.CRYSTALIZER_MODEL || 'gpt-5.1-codex-max';
|
|
13
|
+
const DEFAULT_BUDGET = 0.5;
|
|
14
|
+
export async function runCrystalizer(options) {
|
|
15
|
+
const { manifest, client, problemStatement, validatedPatch, scenarioSummary = null, repoRoot = process.cwd(), outputDir = DEFAULT_OUTPUT_DIR, promptPackId = DEFAULT_PROMPT_PACK, model = DEFAULT_MODEL, budgetUsd = DEFAULT_BUDGET } = options;
|
|
16
|
+
if (outputDir.includes('docs/patterns')) {
|
|
17
|
+
throw new Error('Crystalizer output may not be written to docs/patterns; use .agent/patterns instead.');
|
|
18
|
+
}
|
|
19
|
+
const learning = ensureLearningSection(manifest);
|
|
20
|
+
const packs = await loadPromptPacks(repoRoot);
|
|
21
|
+
const pack = packs.find((entry) => entry.id === promptPackId);
|
|
22
|
+
if (!pack) {
|
|
23
|
+
throw new Error(`Prompt pack ${promptPackId} not found`);
|
|
24
|
+
}
|
|
25
|
+
const packStamp = computePromptPackStamp(pack.sources);
|
|
26
|
+
const promptSources = pack.sources.map((source) => source.content).join('\n\n');
|
|
27
|
+
const prompt = composePrompt(promptSources, packStamp, problemStatement, validatedPatch, scenarioSummary ?? '');
|
|
28
|
+
const response = await client.generate(prompt, { model });
|
|
29
|
+
const cost = typeof response.costUsd === 'number' ? response.costUsd : 0;
|
|
30
|
+
if (cost > budgetUsd) {
|
|
31
|
+
appendLearningAlert(manifest, {
|
|
32
|
+
type: 'budget_exceeded',
|
|
33
|
+
channel: 'slack',
|
|
34
|
+
target: '#learning-alerts',
|
|
35
|
+
message: `Crystalizer cost ${cost.toFixed(2)} exceeded budget ${budgetUsd.toFixed(2)}`
|
|
36
|
+
});
|
|
37
|
+
learning.crystalizer = {
|
|
38
|
+
candidate_path: null,
|
|
39
|
+
model,
|
|
40
|
+
prompt_pack: promptPackId,
|
|
41
|
+
prompt_pack_stamp: packStamp,
|
|
42
|
+
budget_usd: budgetUsd,
|
|
43
|
+
cost_usd: cost,
|
|
44
|
+
status: 'failed',
|
|
45
|
+
error: 'budget_exceeded',
|
|
46
|
+
created_at: isoTimestamp()
|
|
47
|
+
};
|
|
48
|
+
return { manifest, candidatePath: null };
|
|
49
|
+
}
|
|
50
|
+
const candidateDir = join(repoRoot, outputDir);
|
|
51
|
+
await mkdir(candidateDir, { recursive: true });
|
|
52
|
+
const baseSlug = slugify(problemStatement || 'learning');
|
|
53
|
+
const slug = baseSlug.slice(0, 60) || 'learning';
|
|
54
|
+
const candidatePath = join(candidateDir, `${slug}.md`);
|
|
55
|
+
const markdown = [
|
|
56
|
+
'# Problem',
|
|
57
|
+
problemStatement || 'Not provided',
|
|
58
|
+
'',
|
|
59
|
+
'## Solution',
|
|
60
|
+
response.content.trim(),
|
|
61
|
+
'',
|
|
62
|
+
'## Rationale',
|
|
63
|
+
scenarioSummary ? scenarioSummary : 'Captured from validation run.'
|
|
64
|
+
].join('\n');
|
|
65
|
+
await writeFile(candidatePath, markdown, 'utf8');
|
|
66
|
+
learning.crystalizer = {
|
|
67
|
+
candidate_path: relative(repoRoot, candidatePath),
|
|
68
|
+
model: response.model ?? model,
|
|
69
|
+
prompt_pack: promptPackId,
|
|
70
|
+
prompt_pack_stamp: packStamp,
|
|
71
|
+
budget_usd: budgetUsd,
|
|
72
|
+
cost_usd: cost,
|
|
73
|
+
status: 'succeeded',
|
|
74
|
+
created_at: isoTimestamp()
|
|
75
|
+
};
|
|
76
|
+
return { manifest, candidatePath };
|
|
77
|
+
}
|
|
78
|
+
function composePrompt(promptBody, packStamp, problem, patch, scenarioSummary) {
|
|
79
|
+
const segments = [
|
|
80
|
+
`Prompt-Pack: ${packStamp}`,
|
|
81
|
+
promptBody,
|
|
82
|
+
'You are the crystalizer. Produce a concise pattern with Problem, Solution, and Rationale.',
|
|
83
|
+
`Problem:\n${problem}`,
|
|
84
|
+
`Patch:\n${patch}`,
|
|
85
|
+
scenarioSummary ? `Scenario:\n${scenarioSummary}` : ''
|
|
86
|
+
];
|
|
87
|
+
return segments.filter(Boolean).join('\n\n');
|
|
88
|
+
}
|
|
89
|
+
export async function createCodexCliCrystalizerClient(binary = process.env.CODEX_CLI_BIN ?? 'codex') {
|
|
90
|
+
const execFileAsync = promisify(execFile);
|
|
91
|
+
return {
|
|
92
|
+
async generate(prompt, options) {
|
|
93
|
+
const workDir = await mkdtemp(join(tmpdir(), 'crystalizer-'));
|
|
94
|
+
try {
|
|
95
|
+
const promptPath = join(workDir, 'prompt.txt');
|
|
96
|
+
const outputPath = join(workDir, 'output.txt');
|
|
97
|
+
await writeFile(promptPath, prompt, 'utf8');
|
|
98
|
+
const args = ['chat', '--model', options.model, '--input-file', promptPath, '--output-file', outputPath];
|
|
99
|
+
await execFileAsync(binary, args, { env: { ...process.env } });
|
|
100
|
+
const content = await readFile(outputPath, 'utf8');
|
|
101
|
+
return { content: content.trim(), model: options.model };
|
|
102
|
+
}
|
|
103
|
+
finally {
|
|
104
|
+
await rm(workDir, { recursive: true, force: true });
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
};
|
|
108
|
+
}
|