@kbediako/codex-orchestrator 0.1.2 → 0.1.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/README.md +15 -8
  2. package/dist/bin/codex-orchestrator.js +252 -121
  3. package/dist/orchestrator/src/cli/config/delegationConfig.js +485 -0
  4. package/dist/orchestrator/src/cli/config/userConfig.js +86 -12
  5. package/dist/orchestrator/src/cli/control/confirmations.js +262 -0
  6. package/dist/orchestrator/src/cli/control/controlServer.js +1476 -0
  7. package/dist/orchestrator/src/cli/control/controlState.js +46 -0
  8. package/dist/orchestrator/src/cli/control/controlWatcher.js +222 -0
  9. package/dist/orchestrator/src/cli/control/delegationTokens.js +62 -0
  10. package/dist/orchestrator/src/cli/control/questions.js +106 -0
  11. package/dist/orchestrator/src/cli/delegationServer.js +1368 -0
  12. package/dist/orchestrator/src/cli/events/runEventStream.js +246 -0
  13. package/dist/orchestrator/src/cli/exec/context.js +9 -3
  14. package/dist/orchestrator/src/cli/exec/learning.js +5 -3
  15. package/dist/orchestrator/src/cli/exec/stageRunner.js +30 -5
  16. package/dist/orchestrator/src/cli/exec/summary.js +1 -1
  17. package/dist/orchestrator/src/cli/metrics/metricsAggregator.js +377 -147
  18. package/dist/orchestrator/src/cli/metrics/metricsRecorder.js +3 -5
  19. package/dist/orchestrator/src/cli/orchestrator.js +233 -47
  20. package/dist/orchestrator/src/cli/pipelines/index.js +13 -24
  21. package/dist/orchestrator/src/cli/rlm/prompt.js +31 -0
  22. package/dist/orchestrator/src/cli/rlm/runner.js +177 -0
  23. package/dist/orchestrator/src/cli/rlm/types.js +1 -0
  24. package/dist/orchestrator/src/cli/rlm/validator.js +159 -0
  25. package/dist/orchestrator/src/cli/rlmRunner.js +440 -0
  26. package/dist/orchestrator/src/cli/run/environment.js +4 -11
  27. package/dist/orchestrator/src/cli/run/manifest.js +7 -1
  28. package/dist/orchestrator/src/cli/run/manifestPersister.js +33 -3
  29. package/dist/orchestrator/src/cli/run/runPaths.js +14 -0
  30. package/dist/orchestrator/src/cli/services/commandRunner.js +2 -2
  31. package/dist/orchestrator/src/cli/services/controlPlaneService.js +3 -1
  32. package/dist/orchestrator/src/cli/services/execRuntime.js +1 -2
  33. package/dist/orchestrator/src/cli/services/pipelineResolver.js +33 -2
  34. package/dist/orchestrator/src/cli/services/runPreparation.js +7 -1
  35. package/dist/orchestrator/src/cli/services/schedulerService.js +1 -1
  36. package/dist/orchestrator/src/cli/utils/devtools.js +33 -2
  37. package/dist/orchestrator/src/cli/utils/specGuardRunner.js +3 -1
  38. package/dist/orchestrator/src/cli/utils/strings.js +8 -6
  39. package/dist/orchestrator/src/persistence/ExperienceStore.js +115 -58
  40. package/dist/orchestrator/src/persistence/PersistenceCoordinator.js +8 -8
  41. package/dist/orchestrator/src/persistence/TaskStateStore.js +3 -2
  42. package/dist/orchestrator/src/persistence/lockFile.js +26 -1
  43. package/dist/orchestrator/src/persistence/sanitizeIdentifier.js +1 -1
  44. package/dist/orchestrator/src/sync/CloudSyncWorker.js +17 -4
  45. package/dist/packages/orchestrator/src/exec/stdio.js +112 -0
  46. package/dist/packages/orchestrator/src/exec/unified-exec.js +1 -1
  47. package/dist/packages/orchestrator/src/index.js +1 -0
  48. package/dist/packages/orchestrator/src/telemetry/otel-exporter.js +21 -0
  49. package/dist/packages/shared/design-artifacts/writer.js +4 -14
  50. package/dist/packages/shared/streams/stdio.js +2 -112
  51. package/dist/packages/shared/utils/strings.js +17 -0
  52. package/dist/scripts/design/pipeline/advanced-assets.js +1 -1
  53. package/dist/scripts/design/pipeline/context.js +5 -5
  54. package/dist/scripts/design/pipeline/extract.js +9 -6
  55. package/dist/scripts/design/pipeline/{optionalDeps.js → optional-deps.js} +49 -38
  56. package/dist/scripts/design/pipeline/permit.js +59 -0
  57. package/dist/scripts/design/pipeline/toolkit/common.js +18 -32
  58. package/dist/scripts/design/pipeline/toolkit/reference.js +1 -1
  59. package/dist/scripts/design/pipeline/toolkit/snapshot.js +1 -1
  60. package/dist/scripts/design/pipeline/visual-regression.js +2 -11
  61. package/dist/scripts/lib/cli-args.js +53 -0
  62. package/dist/scripts/lib/docs-helpers.js +111 -0
  63. package/dist/scripts/lib/npm-pack.js +20 -0
  64. package/dist/scripts/lib/run-manifests.js +160 -0
  65. package/package.json +7 -2
  66. package/dist/orchestrator/src/cli/pipelines/defaultDiagnostics.js +0 -32
  67. package/dist/orchestrator/src/cli/pipelines/designReference.js +0 -72
  68. package/dist/orchestrator/src/cli/pipelines/hiFiDesignToolkit.js +0 -71
  69. package/dist/orchestrator/src/cli/utils/jsonlWriter.js +0 -10
  70. package/dist/orchestrator/src/control-plane/index.js +0 -3
  71. package/dist/orchestrator/src/persistence/identifierGuards.js +0 -1
  72. package/dist/orchestrator/src/persistence/writeAtomicFile.js +0 -4
  73. package/dist/orchestrator/src/scheduler/index.js +0 -1
@@ -7,6 +7,13 @@ export function resolveRunPaths(env, runId) {
7
7
  const heartbeatPath = join(runDir, '.heartbeat');
8
8
  const resumeTokenPath = join(runDir, '.resume-token');
9
9
  const logPath = join(runDir, 'runner.ndjson');
10
+ const eventsPath = join(runDir, 'events.jsonl');
11
+ const controlPath = join(runDir, 'control.json');
12
+ const controlAuthPath = join(runDir, 'control_auth.json');
13
+ const controlEndpointPath = join(runDir, 'control_endpoint.json');
14
+ const confirmationsPath = join(runDir, 'confirmations.json');
15
+ const questionsPath = join(runDir, 'questions.json');
16
+ const delegationTokensPath = join(runDir, 'delegation_tokens.json');
10
17
  const commandsDir = join(runDir, 'commands');
11
18
  const errorsDir = join(runDir, 'errors');
12
19
  const compatDir = join(env.runsRoot, env.taskId, 'mcp', safeRunId);
@@ -18,6 +25,13 @@ export function resolveRunPaths(env, runId) {
18
25
  heartbeatPath,
19
26
  resumeTokenPath,
20
27
  logPath,
28
+ eventsPath,
29
+ controlPath,
30
+ controlAuthPath,
31
+ controlEndpointPath,
32
+ confirmationsPath,
33
+ questionsPath,
34
+ delegationTokensPath,
21
35
  commandsDir,
22
36
  errorsDir,
23
37
  compatDir,
@@ -12,7 +12,7 @@ import { EnvUtils } from '../../../../packages/shared/config/index.js';
12
12
  import { findPackageRoot } from '../utils/packageInfo.js';
13
13
  const MAX_BUFFERED_OUTPUT_BYTES = 64 * 1024;
14
14
  const EMIT_COMMAND_STREAM_MIRRORS = EnvUtils.getBoolean('CODEX_ORCHESTRATOR_EMIT_COMMAND_STREAMS', false);
15
- const MAX_CAPTURED_CHUNK_EVENTS = EnvUtils.getInt('CODEX_ORCHESTRATOR_EXEC_EVENT_MAX_CHUNKS', 0);
15
+ export const MAX_CAPTURED_CHUNK_EVENTS = EnvUtils.getInt('CODEX_ORCHESTRATOR_EXEC_EVENT_MAX_CHUNKS', 500);
16
16
  const PACKAGE_ROOT = findPackageRoot();
17
17
  export async function runCommandStage(context, hooks = {}) {
18
18
  const { env, paths, manifest, stage, index, events, persister, envOverrides } = context;
@@ -129,7 +129,7 @@ export async function runCommandStage(context, hooks = {}) {
129
129
  CODEX_ORCHESTRATOR_RUN_DIR: paths.runDir,
130
130
  CODEX_ORCHESTRATOR_RUNS_DIR: env.runsRoot,
131
131
  CODEX_ORCHESTRATOR_OUT_DIR: env.outRoot,
132
- CODEX_ORCHESTRATOR_REPO_ROOT: env.repoRoot,
132
+ CODEX_ORCHESTRATOR_ROOT: env.repoRoot,
133
133
  CODEX_ORCHESTRATOR_PACKAGE_ROOT: PACKAGE_ROOT
134
134
  };
135
135
  const execEnv = { ...baseEnv, ...stage.env };
@@ -1,5 +1,7 @@
1
1
  import process from 'node:process';
2
- import { buildRunRequestV2, ControlPlaneDriftReporter, ControlPlaneValidationError, ControlPlaneValidator } from '../../control-plane/index.js';
2
+ import { ControlPlaneDriftReporter } from '../../control-plane/drift-reporter.js';
3
+ import { buildRunRequestV2 } from '../../control-plane/request-builder.js';
4
+ import { ControlPlaneValidationError, ControlPlaneValidator } from '../../control-plane/validator.js';
3
5
  import { relativeToRepo } from '../run/runPaths.js';
4
6
  import { appendSummary } from '../run/manifest.js';
5
7
  import { persistManifest } from '../run/manifestPersister.js';
@@ -1,6 +1,5 @@
1
1
  import { spawn } from 'node:child_process';
2
- import { ExecSessionManager, UnifiedExecRunner, ToolOrchestrator } from '../../../../packages/orchestrator/src/index.js';
3
- import { RemoteExecHandleService } from '../../../../packages/orchestrator/src/exec/handle-service.js';
2
+ import { ExecSessionManager, UnifiedExecRunner, ToolOrchestrator, RemoteExecHandleService } from '../../../../packages/orchestrator/src/index.js';
4
3
  import { PrivacyGuard } from '../../privacy/guard.js';
5
4
  import { resolveEnforcementMode } from '../utils/enforcementMode.js';
6
5
  class CliExecSessionHandle {
@@ -1,8 +1,12 @@
1
1
  import process from 'node:process';
2
- import { loadUserConfig } from '../config/userConfig.js';
2
+ import { loadPackageConfig, loadUserConfig } from '../config/userConfig.js';
3
3
  import { resolvePipeline } from '../pipelines/index.js';
4
4
  import { loadDesignConfig, shouldActivateDesignPipeline, designPipelineId } from '../../../../packages/shared/config/index.js';
5
5
  import { logger } from '../../logger.js';
6
+ const DEVTOOLS_PIPELINE_ALIASES = new Map([
7
+ ['implementation-gate-devtools', 'implementation-gate'],
8
+ ['frontend-testing-devtools', 'frontend-testing']
9
+ ]);
6
10
  export class PipelineResolver {
7
11
  async loadDesignConfig(rootDir) {
8
12
  const designConfig = await loadDesignConfig({ rootDir });
@@ -19,9 +23,15 @@ export class PipelineResolver {
19
23
  logger.info(`PipelineResolver.resolve loaded design config from ${designConfig.path}`);
20
24
  const userConfig = await loadUserConfig(env);
21
25
  logger.info(`PipelineResolver.resolve loaded user config`);
22
- const requestedPipelineId = options.pipelineId ??
26
+ const pipelineCandidate = options.pipelineId ??
23
27
  (shouldActivateDesignPipeline(designConfig) ? designPipelineId(designConfig) : undefined);
28
+ const resolvedAlias = this.resolvePipelineAlias(pipelineCandidate);
29
+ const requestedPipelineId = resolvedAlias.pipelineId;
24
30
  const envOverrides = this.resolveDesignEnvOverrides(designConfig, requestedPipelineId);
31
+ if (resolvedAlias.devtoolsRequested) {
32
+ envOverrides.CODEX_REVIEW_DEVTOOLS = '1';
33
+ logger.warn(`[pipeline] ${resolvedAlias.aliasId} is deprecated; use ${requestedPipelineId} with CODEX_REVIEW_DEVTOOLS=1.`);
34
+ }
25
35
  try {
26
36
  const { pipeline, source } = resolvePipeline(env, {
27
37
  pipelineId: requestedPipelineId,
@@ -31,6 +41,17 @@ export class PipelineResolver {
31
41
  return { pipeline, userConfig, designConfig, source, envOverrides };
32
42
  }
33
43
  catch (error) {
44
+ if (requestedPipelineId === 'rlm' && userConfig?.source === 'repo') {
45
+ const packageConfig = await loadPackageConfig(env);
46
+ if (packageConfig) {
47
+ const { pipeline, source } = resolvePipeline(env, {
48
+ pipelineId: requestedPipelineId,
49
+ config: packageConfig
50
+ });
51
+ logger.info(`PipelineResolver.resolve selected package pipeline ${pipeline.id}`);
52
+ return { pipeline, userConfig: packageConfig, designConfig, source, envOverrides };
53
+ }
54
+ }
34
55
  logger.error(`PipelineResolver.resolve failed for ${requestedPipelineId ?? '<default>'}: ${error.message}`);
35
56
  throw error;
36
57
  }
@@ -44,4 +65,14 @@ export class PipelineResolver {
44
65
  }
45
66
  return envOverrides;
46
67
  }
68
+ resolvePipelineAlias(pipelineId) {
69
+ if (!pipelineId) {
70
+ return { pipelineId, devtoolsRequested: false };
71
+ }
72
+ const target = DEVTOOLS_PIPELINE_ALIASES.get(pipelineId);
73
+ if (!target) {
74
+ return { pipelineId, devtoolsRequested: false };
75
+ }
76
+ return { pipelineId: target, devtoolsRequested: true, aliasId: pipelineId };
77
+ }
47
78
  }
@@ -62,11 +62,17 @@ export async function prepareRun(options) {
62
62
  planPreview
63
63
  };
64
64
  }
65
- export function resolvePipelineForResume(env, manifest, config) {
65
+ export function resolvePipelineForResume(env, manifest, config, fallbackConfig = null) {
66
66
  const existing = findPipeline(config ?? null, manifest.pipeline_id);
67
67
  if (existing) {
68
68
  return existing;
69
69
  }
70
+ if (manifest.pipeline_id === 'rlm' && fallbackConfig) {
71
+ const fallback = findPipeline(fallbackConfig, manifest.pipeline_id);
72
+ if (fallback) {
73
+ return fallback;
74
+ }
75
+ }
70
76
  const { pipeline } = resolvePipeline(env, { pipelineId: manifest.pipeline_id, config });
71
77
  return pipeline;
72
78
  }
@@ -1,6 +1,6 @@
1
1
  import { sanitizeTaskId } from '../run/environment.js';
2
2
  import { persistManifest } from '../run/manifestPersister.js';
3
- import { buildSchedulerRunSummary, createSchedulerPlan, finalizeSchedulerPlan, serializeSchedulerPlan } from '../../scheduler/index.js';
3
+ import { buildSchedulerRunSummary, createSchedulerPlan, finalizeSchedulerPlan, serializeSchedulerPlan } from '../../scheduler/plan.js';
4
4
  import { isoTimestamp } from '../utils/time.js';
5
5
  export class SchedulerService {
6
6
  now;
@@ -5,6 +5,7 @@ import process from 'node:process';
5
5
  import { EnvUtils } from '../../../../packages/shared/config/env.js';
6
6
  export const DEVTOOLS_SKILL_NAME = 'chrome-devtools';
7
7
  export const DEVTOOLS_CONFIG_OVERRIDE = 'mcp_servers.chrome-devtools.enabled=true';
8
+ const CONFIG_OVERRIDE_ENV_KEYS = ['CODEX_MCP_CONFIG_OVERRIDES', 'CODEX_CONFIG_OVERRIDES'];
8
9
  const DEVTOOLS_CONFIG_FILENAME = 'config.toml';
9
10
  const DEVTOOLS_MCP_COMMAND = [
10
11
  'mcp',
@@ -86,16 +87,18 @@ export function buildDevtoolsSetupPlan(env = process.env) {
86
87
  };
87
88
  }
88
89
  export function resolveCodexCommand(args, env = process.env) {
90
+ const overrides = parseConfigOverrides(env);
89
91
  if (!isDevtoolsEnabled(env)) {
90
- return { command: 'codex', args };
92
+ return { command: 'codex', args: applyConfigOverrides(overrides, args) };
91
93
  }
92
94
  const readiness = resolveDevtoolsReadiness(env);
93
95
  if (readiness.status !== 'ok') {
94
96
  throw new Error(formatDevtoolsPreflightError(readiness));
95
97
  }
98
+ const mergedOverrides = dedupeOverrides([DEVTOOLS_CONFIG_OVERRIDE, ...overrides]);
96
99
  return {
97
100
  command: 'codex',
98
- args: ['-c', DEVTOOLS_CONFIG_OVERRIDE, ...args]
101
+ args: applyConfigOverrides(mergedOverrides, args)
99
102
  };
100
103
  }
101
104
  export function formatDevtoolsPreflightError(readiness) {
@@ -187,6 +190,34 @@ function hasDevtoolsConfigEntry(raw) {
187
190
  }
188
191
  return false;
189
192
  }
193
+ function parseConfigOverrides(env) {
194
+ const overrides = [];
195
+ for (const key of CONFIG_OVERRIDE_ENV_KEYS) {
196
+ const raw = env[key];
197
+ if (!raw) {
198
+ continue;
199
+ }
200
+ const parts = raw
201
+ .split(/[,;\n]/)
202
+ .map((part) => part.trim())
203
+ .filter((part) => part.length > 0);
204
+ overrides.push(...parts);
205
+ }
206
+ return dedupeOverrides(overrides);
207
+ }
208
+ function applyConfigOverrides(overrides, args) {
209
+ if (overrides.length === 0) {
210
+ return args;
211
+ }
212
+ const configArgs = [];
213
+ for (const override of overrides) {
214
+ configArgs.push('-c', override);
215
+ }
216
+ return [...configArgs, ...args];
217
+ }
218
+ function dedupeOverrides(overrides) {
219
+ return Array.from(new Set(overrides.filter((override) => override.trim().length > 0)));
220
+ }
190
221
  function stripTomlComment(line) {
191
222
  const index = line.indexOf('#');
192
223
  if (index === -1) {
@@ -3,7 +3,9 @@ import { existsSync } from 'node:fs';
3
3
  import { join } from 'node:path';
4
4
  import process from 'node:process';
5
5
  import { logger } from '../../logger.js';
6
- const specGuardPath = join(process.cwd(), 'scripts', 'spec-guard.mjs');
6
+ import { resolveEnvironmentPaths } from '../../../../scripts/lib/run-manifests.js';
7
+ const { repoRoot } = resolveEnvironmentPaths();
8
+ const specGuardPath = join(repoRoot, 'scripts', 'spec-guard.mjs');
7
9
  if (!existsSync(specGuardPath)) {
8
10
  logger.warn(`[spec-guard] skipped: ${specGuardPath} not found`);
9
11
  process.exit(0);
@@ -1,8 +1,10 @@
1
+ import { slugify as sharedSlugify } from '../../../../packages/shared/utils/strings.js';
1
2
  export function slugify(value, fallback = 'command') {
2
- const cleaned = value.trim().replace(/[^a-zA-Z0-9]+/g, '-').replace(/-+/g, '-');
3
- const normalized = cleaned.replace(/^-|-$/g, '');
4
- if (!normalized) {
5
- return fallback;
6
- }
7
- return normalized.slice(0, 80);
3
+ return sharedSlugify(value, {
4
+ fallback,
5
+ maxLength: 80,
6
+ lowercase: false,
7
+ pattern: /[^a-zA-Z0-9]+/g,
8
+ collapseDashes: true
9
+ });
8
10
  }
@@ -1,9 +1,11 @@
1
1
  import { randomBytes } from 'node:crypto';
2
- import { readFile, mkdir, rm, readdir } from 'node:fs/promises';
2
+ import { createReadStream } from 'node:fs';
3
+ import { appendFile, mkdir, open, rm } from 'node:fs/promises';
3
4
  import { join } from 'node:path';
5
+ import { createInterface } from 'node:readline';
6
+ import { listDirectories, resolveEnvironmentPaths } from '../../../scripts/lib/run-manifests.js';
4
7
  import { acquireLockWithRetry } from './lockFile.js';
5
8
  import { sanitizeTaskId } from './sanitizeTaskId.js';
6
- import { writeAtomicFile } from './writeAtomicFile.js';
7
9
  export class ExperienceStoreLockError extends Error {
8
10
  taskId;
9
11
  constructor(message, taskId) {
@@ -21,14 +23,16 @@ export class ExperienceStore {
21
23
  lockRetry;
22
24
  now;
23
25
  constructor(options = {}) {
24
- this.outDir = options.outDir ?? join(process.cwd(), 'out');
25
- this.runsDir = options.runsDir ?? join(process.cwd(), '.runs');
26
+ const envPaths = resolveEnvironmentPaths();
27
+ this.outDir = options.outDir ?? envPaths.outRoot;
28
+ this.runsDir = options.runsDir ?? envPaths.runsRoot;
26
29
  this.maxWords = Math.max(1, options.maxSummaryWords ?? DEFAULT_MAX_WORDS);
27
30
  const defaults = {
28
31
  maxAttempts: 5,
29
32
  initialDelayMs: 100,
30
33
  backoffFactor: 2,
31
- maxDelayMs: 1000
34
+ maxDelayMs: 1000,
35
+ staleMs: 5 * 60 * 1000
32
36
  };
33
37
  const overrides = options.lockRetry ?? {};
34
38
  const sanitizedOverrides = Object.fromEntries(Object.entries(overrides).filter(([, value]) => value !== undefined));
@@ -50,10 +54,10 @@ export class ExperienceStore {
50
54
  const targetDir = join(this.outDir, taskId);
51
55
  await mkdir(targetDir, { recursive: true });
52
56
  const filePath = join(targetDir, 'experiences.jsonl');
53
- const existing = await this.readRecords(filePath);
54
57
  const nextRecords = inputs.map((input) => this.prepareRecord(input, manifestPath));
55
- const serialized = [...existing, ...nextRecords].map((record) => JSON.stringify(record)).join('\n');
56
- await writeAtomicFile(filePath, `${serialized}\n`);
58
+ await this.ensureTrailingNewline(filePath);
59
+ const payload = nextRecords.map((record) => JSON.stringify(record)).join('\n');
60
+ await appendFile(filePath, `${payload}\n`, 'utf8');
57
61
  return nextRecords;
58
62
  }
59
63
  finally {
@@ -70,36 +74,26 @@ export class ExperienceStore {
70
74
  if (limit === 0) {
71
75
  return [];
72
76
  }
73
- const allRecords = taskFilter
74
- ? await this.readRecords(join(this.outDir, taskFilter, 'experiences.jsonl'))
75
- : await this.readAllRecords();
76
- const filtered = allRecords.filter((record) => {
77
+ const collector = createTopKCollector(limit, params.minReward);
78
+ const applyRecord = (record) => {
77
79
  if (record.domain !== safeDomain) {
78
- return false;
80
+ return;
79
81
  }
80
82
  if (taskFilter && record.taskId !== taskFilter) {
81
- return false;
83
+ return;
82
84
  }
83
- return true;
84
- });
85
- const scored = filtered
86
- .map((record) => ({
87
- record,
88
- score: record.reward.gtScore + record.reward.relativeRank
89
- }))
90
- .filter((entry) => {
91
- if (params.minReward === undefined) {
92
- return true;
93
- }
94
- return entry.score >= params.minReward;
95
- })
96
- .sort((a, b) => {
97
- if (b.score === a.score) {
98
- return b.record.createdAt.localeCompare(a.record.createdAt);
85
+ collector.add(record);
86
+ };
87
+ if (taskFilter) {
88
+ await this.scanRecords(join(this.outDir, taskFilter, 'experiences.jsonl'), applyRecord);
89
+ }
90
+ else {
91
+ const directories = await listDirectories(this.outDir);
92
+ for (const dir of directories) {
93
+ await this.scanRecords(join(this.outDir, dir, 'experiences.jsonl'), applyRecord);
99
94
  }
100
- return b.score - a.score;
101
- });
102
- return scored.slice(0, limit).map((entry) => entry.record);
95
+ }
96
+ return collector.finalize();
103
97
  }
104
98
  verifyStamp(record) {
105
99
  return HEX_STAMP_PATTERN.test(record.stampSignature);
@@ -154,46 +148,109 @@ export class ExperienceStore {
154
148
  async releaseLock(lockPath) {
155
149
  await rm(lockPath, { force: true });
156
150
  }
157
- async readRecords(filePath) {
151
+ async scanRecords(filePath, onRecord) {
158
152
  try {
159
- const raw = await readFile(filePath, 'utf8');
160
- return raw
161
- .split('\n')
162
- .filter(Boolean)
163
- .map((line) => JSON.parse(line));
153
+ const stream = createReadStream(filePath, { encoding: 'utf8' });
154
+ const reader = createInterface({ input: stream, crlfDelay: Infinity });
155
+ for await (const line of reader) {
156
+ const trimmed = line.trim();
157
+ if (!trimmed) {
158
+ continue;
159
+ }
160
+ try {
161
+ const record = JSON.parse(trimmed);
162
+ onRecord(record);
163
+ }
164
+ catch {
165
+ continue;
166
+ }
167
+ }
164
168
  }
165
169
  catch (error) {
166
170
  if (error.code === 'ENOENT') {
167
- return [];
171
+ return;
168
172
  }
169
173
  throw error;
170
174
  }
171
175
  }
172
- async readAllRecords() {
173
- const directories = await listDirectories(this.outDir);
174
- const all = [];
175
- for (const dir of directories) {
176
- const filePath = join(this.outDir, dir, 'experiences.jsonl');
177
- all.push(...(await this.readRecords(filePath)));
178
- }
179
- return all;
176
+ async ensureTrailingNewline(filePath) {
177
+ try {
178
+ const handle = await open(filePath, 'r');
179
+ let needsNewline = false;
180
+ try {
181
+ const { size } = await handle.stat();
182
+ if (size === 0) {
183
+ return;
184
+ }
185
+ const buffer = Buffer.alloc(1);
186
+ await handle.read(buffer, 0, 1, size - 1);
187
+ needsNewline = buffer[0] !== 0x0a;
188
+ }
189
+ finally {
190
+ await handle.close();
191
+ }
192
+ if (needsNewline) {
193
+ await appendFile(filePath, '\n', 'utf8');
194
+ }
195
+ }
196
+ catch (error) {
197
+ if (error.code === 'ENOENT') {
198
+ return;
199
+ }
200
+ throw error;
201
+ }
180
202
  }
181
203
  generateId() {
182
204
  const suffix = randomBytes(3).toString('hex');
183
205
  return `exp-${Date.now().toString(36)}-${suffix}`;
184
206
  }
185
207
  }
186
- async function listDirectories(path) {
187
- try {
188
- const entries = await readdir(path, { withFileTypes: true });
189
- return entries.filter((entry) => entry.isDirectory()).map((entry) => entry.name);
190
- }
191
- catch (error) {
192
- if (error.code === 'ENOENT') {
193
- return [];
208
+ function createTopKCollector(limit, minReward) {
209
+ const entries = [];
210
+ const threshold = typeof minReward === 'number' ? minReward : null;
211
+ const compare = (a, b) => {
212
+ if (a.score !== b.score) {
213
+ return a.score - b.score;
194
214
  }
195
- throw error;
196
- }
215
+ const aTime = a.record.createdAt ?? '';
216
+ const bTime = b.record.createdAt ?? '';
217
+ return aTime.localeCompare(bTime);
218
+ };
219
+ const add = (record) => {
220
+ const score = record.reward.gtScore + record.reward.relativeRank;
221
+ if (threshold !== null && score < threshold) {
222
+ return;
223
+ }
224
+ const entry = { record, score };
225
+ if (entries.length === 0) {
226
+ entries.push(entry);
227
+ return;
228
+ }
229
+ const worst = entries[0];
230
+ if (entries.length >= limit && worst && compare(entry, worst) <= 0) {
231
+ return;
232
+ }
233
+ let index = 0;
234
+ while (index < entries.length && compare(entries[index], entry) <= 0) {
235
+ index += 1;
236
+ }
237
+ entries.splice(index, 0, entry);
238
+ if (entries.length > limit) {
239
+ entries.shift();
240
+ }
241
+ };
242
+ const finalize = () => entries
243
+ .slice()
244
+ .sort((a, b) => {
245
+ if (a.score !== b.score) {
246
+ return b.score - a.score;
247
+ }
248
+ const aTime = a.record.createdAt ?? '';
249
+ const bTime = b.record.createdAt ?? '';
250
+ return bTime.localeCompare(aTime);
251
+ })
252
+ .map((entry) => entry.record);
253
+ return { add, finalize };
197
254
  }
198
255
  function truncateSummary(value, maxWords) {
199
256
  const tokens = value.trim().split(/\s+/u).filter(Boolean);
@@ -33,10 +33,12 @@ export class PersistenceCoordinator {
33
33
  }
34
34
  async handleRunCompleted(summary) {
35
35
  let stateStoreError = null;
36
- try {
37
- await this.stateStore.recordRun(summary);
38
- }
39
- catch (error) {
36
+ const [stateResult, manifestResult] = await Promise.allSettled([
37
+ this.stateStore.recordRun(summary),
38
+ this.manifestWriter.write(summary)
39
+ ]);
40
+ if (stateResult.status === 'rejected') {
41
+ const error = stateResult.reason;
40
42
  stateStoreError = error;
41
43
  if (error instanceof TaskStateStoreLockError) {
42
44
  logger.warn(`Task state snapshot skipped for task ${summary.taskId} (run ${summary.runId}): ${error.message}`);
@@ -45,10 +47,8 @@ export class PersistenceCoordinator {
45
47
  logger.error(`Task state snapshot failed for task ${summary.taskId} (run ${summary.runId})`, error);
46
48
  }
47
49
  }
48
- try {
49
- await this.manifestWriter.write(summary);
50
- }
51
- catch (error) {
50
+ if (manifestResult.status === 'rejected') {
51
+ const error = manifestResult.reason;
52
52
  this.options.onError?.(error, summary);
53
53
  if (!this.options.onError) {
54
54
  logger.error(`PersistenceCoordinator manifest write error for task ${summary.taskId} (run ${summary.runId})`, error);
@@ -2,7 +2,7 @@ import { mkdir, readFile, rm } from 'node:fs/promises';
2
2
  import { dirname, join } from 'node:path';
3
3
  import { acquireLockWithRetry } from './lockFile.js';
4
4
  import { sanitizeTaskId } from './sanitizeTaskId.js';
5
- import { writeAtomicFile } from './writeAtomicFile.js';
5
+ import { writeAtomicFile } from '../utils/atomicWrite.js';
6
6
  export class TaskStateStoreLockError extends Error {
7
7
  taskId;
8
8
  constructor(message, taskId) {
@@ -27,7 +27,8 @@ export class TaskStateStore {
27
27
  maxAttempts: 5,
28
28
  initialDelayMs: 100,
29
29
  backoffFactor: 2,
30
- maxDelayMs: 1000
30
+ maxDelayMs: 1000,
31
+ staleMs: 5 * 60 * 1000
31
32
  };
32
33
  const overrides = options.lockRetry ?? {};
33
34
  const sanitizedOverrides = Object.fromEntries(Object.entries(overrides).filter(([, value]) => value !== undefined));
@@ -1,8 +1,9 @@
1
- import { open } from 'node:fs/promises';
1
+ import { open, rm, stat } from 'node:fs/promises';
2
2
  import { setTimeout as delay } from 'node:timers/promises';
3
3
  export async function acquireLockWithRetry(params) {
4
4
  await params.ensureDirectory();
5
5
  const { maxAttempts, initialDelayMs, backoffFactor, maxDelayMs } = params.retry;
6
+ const staleMs = params.retry.staleMs ?? 0;
6
7
  let attempt = 0;
7
8
  let delayMs = initialDelayMs;
8
9
  while (attempt < maxAttempts) {
@@ -16,6 +17,13 @@ export async function acquireLockWithRetry(params) {
16
17
  if (error.code !== 'EEXIST') {
17
18
  throw error;
18
19
  }
20
+ if (staleMs > 0) {
21
+ const cleared = await clearStaleLock(params.lockPath, staleMs);
22
+ if (cleared) {
23
+ attempt -= 1;
24
+ continue;
25
+ }
26
+ }
19
27
  if (attempt >= maxAttempts) {
20
28
  throw params.createError(params.taskId, attempt);
21
29
  }
@@ -24,3 +32,20 @@ export async function acquireLockWithRetry(params) {
24
32
  }
25
33
  }
26
34
  }
35
+ async function clearStaleLock(lockPath, staleMs) {
36
+ try {
37
+ const stats = await stat(lockPath);
38
+ const ageMs = Date.now() - stats.mtimeMs;
39
+ if (!Number.isFinite(ageMs) || ageMs <= staleMs) {
40
+ return false;
41
+ }
42
+ await rm(lockPath, { force: true });
43
+ return true;
44
+ }
45
+ catch (error) {
46
+ if (error.code === 'ENOENT') {
47
+ return false;
48
+ }
49
+ return false;
50
+ }
51
+ }
@@ -1,4 +1,4 @@
1
- import { WINDOWS_FORBIDDEN_CHARACTERS } from './identifierGuards.js';
1
+ const WINDOWS_FORBIDDEN_CHARACTERS = new Set(['<', '>', ':', '"', '|', '?', '*']);
2
2
  export function sanitizeIdentifier(kind, value) {
3
3
  const label = kind === 'task' ? 'task' : 'run';
4
4
  if (!value) {
@@ -52,7 +52,7 @@ export class CloudSyncWorker {
52
52
  }
53
53
  catch (error) {
54
54
  this.options.onError?.(error, summary, 0);
55
- await this.appendAuditLog({
55
+ await this.safeAppendAuditLog({
56
56
  level: 'error',
57
57
  message: 'Failed to read manifest before sync',
58
58
  summary,
@@ -74,7 +74,7 @@ export class CloudSyncWorker {
74
74
  idempotencyKey
75
75
  });
76
76
  this.options.onSuccess?.(result, summary);
77
- await this.appendAuditLog({
77
+ await this.safeAppendAuditLog({
78
78
  level: 'info',
79
79
  message: 'Cloud sync completed',
80
80
  summary,
@@ -84,7 +84,7 @@ export class CloudSyncWorker {
84
84
  }
85
85
  catch (error) {
86
86
  this.options.onError?.(error, summary, attempt);
87
- await this.appendAuditLog({
87
+ await this.safeAppendAuditLog({
88
88
  level: 'error',
89
89
  message: 'Cloud sync attempt failed',
90
90
  summary,
@@ -120,6 +120,19 @@ export class CloudSyncWorker {
120
120
  });
121
121
  await appendFile(logPath, `${line}\n`, 'utf-8');
122
122
  }
123
+ async safeAppendAuditLog(entry) {
124
+ try {
125
+ await this.appendAuditLog(entry);
126
+ }
127
+ catch (error) {
128
+ try {
129
+ this.options.onError?.(error, entry.summary, 0);
130
+ }
131
+ catch {
132
+ // swallow audit log failures
133
+ }
134
+ }
135
+ }
123
136
  shouldRetry(error) {
124
137
  if (this.options.retryDecider) {
125
138
  return this.options.retryDecider(error);
@@ -160,7 +173,7 @@ export class CloudSyncWorker {
160
173
  if (repaired) {
161
174
  try {
162
175
  const parsed = JSON.parse(repaired);
163
- await this.appendAuditLog({
176
+ await this.safeAppendAuditLog({
164
177
  level: 'info',
165
178
  message: 'Recovered manifest from partial JSON',
166
179
  summary,