@kbediako/codex-orchestrator 0.1.37 → 0.2.0

package/dist/orchestrator/src/persistence/ExperienceStore.js

@@ -1,6 +1,6 @@
 import { randomBytes } from 'node:crypto';
 import { createReadStream } from 'node:fs';
-import { appendFile, mkdir, open, rm } from 'node:fs/promises';
+import { appendFile, mkdir, open } from 'node:fs/promises';
 import { join } from 'node:path';
 import { createInterface } from 'node:readline';
 import { listDirectories, resolveEnvironmentPaths } from '../../../scripts/lib/run-manifests.js';
@@ -49,7 +49,7 @@ export class ExperienceStore {
         }
         const taskId = sanitizeTaskId(inputs[0].taskId);
         const lockPath = this.buildLockPath(taskId);
-        await this.acquireLock(taskId, lockPath);
+        const lock = await this.acquireLock(taskId, lockPath);
         try {
             const targetDir = join(this.outDir, taskId);
             await mkdir(targetDir, { recursive: true });
@@ -61,20 +61,34 @@ export class ExperienceStore {
             return nextRecords;
         }
         finally {
-            await this.releaseLock(lockPath);
+            await this.releaseLock(lock);
         }
     }
     async fetchTop(params) {
+        const result = await this.selectTop({
+            domain: params.domain,
+            limit: params.limit,
+            minScore: params.minReward,
+            taskId: params.taskId
+        });
+        return result.records;
+    }
+    async selectTop(params) {
         const safeDomain = params.domain.trim();
+        const policy = resolveExperienceSelectionPolicy(params.policy, params.minScore);
         if (!safeDomain) {
-            return [];
+            return emptySelection(policy);
         }
         const taskFilter = params.taskId ? sanitizeTaskId(params.taskId) : null;
         const limit = Math.max(0, params.limit);
         if (limit === 0) {
-            return [];
+            return emptySelection(policy);
         }
-        const collector = createTopKCollector(limit, params.minReward);
+        const entries = [];
+        const selectedRecordEntriesBySource = new Map();
+        const selectedRecordLookup = new Map();
+        const sourceCandidateCounts = new Map();
+        let candidateCount = 0;
         const applyRecord = (record) => {
             if (record.domain !== safeDomain) {
                 return;
@@ -82,7 +96,17 @@ export class ExperienceStore {
             if (taskFilter && record.taskId !== taskFilter) {
                 return;
             }
-            collector.add(record);
+            const entry = createSelectionEntry(record, policy);
+            entries.push(entry);
+            candidateCount += 1;
+            sourceCandidateCounts.set(entry.sourceBucketKey, (sourceCandidateCounts.get(entry.sourceBucketKey) ?? 0) + 1);
+            // Keep full diagnostics for every candidate while retaining only bounded records for materialization.
+            const retainedEntry = { ...entry, record };
+            selectedRecordLookup.set(retainedEntry.id, record);
+            const prunedEntry = insertSourceCandidate(selectedRecordEntriesBySource, retainedEntry, limit);
+            if (prunedEntry?.record) {
+                selectedRecordLookup.delete(prunedEntry.id);
+            }
         };
         if (taskFilter) {
             await this.scanRecords(join(this.outDir, taskFilter, 'experiences.jsonl'), applyRecord);
@@ -93,7 +117,7 @@ export class ExperienceStore {
                 await this.scanRecords(join(this.outDir, dir, 'experiences.jsonl'), applyRecord);
             }
         }
-        return collector.finalize();
+        return selectCompetitiveTop(entries, limit, policy, candidateCount, sourceCandidateCounts, selectedRecordLookup);
     }
     verifyStamp(record) {
         return HEX_STAMP_PATTERN.test(record.stampSignature);
@@ -135,7 +159,7 @@ export class ExperienceStore {
         return join(this.runsDir, `${taskId}.experiences.lock`);
     }
     async acquireLock(taskId, lockPath) {
-        await acquireLockWithRetry({
+        return await acquireLockWithRetry({
             taskId,
             lockPath,
             retry: this.lockRetry,
@@ -145,8 +169,8 @@ export class ExperienceStore {
             createError: (id, attempts) => new ExperienceStoreLockError(`Failed to acquire experience lock for ${id} after ${attempts} attempts`, id)
         });
     }
-    async releaseLock(lockPath) {
-        await rm(lockPath, { force: true });
+    async releaseLock(lock) {
+        await lock.release();
     }
     async scanRecords(filePath, onRecord) {
         try {
@@ -205,52 +229,212 @@ export class ExperienceStore {
         return `exp-${Date.now().toString(36)}-${suffix}`;
     }
 }
-function createTopKCollector(limit, minReward) {
-    const entries = [];
-    const threshold = typeof minReward === 'number' ? minReward : null;
-    const compare = (a, b) => {
-        if (a.score !== b.score) {
-            return a.score - b.score;
+function emptySelection(policy) {
+    return {
+        records: [],
+        diagnostics: {
+            policy,
+            candidate_count: 0,
+            selected_count: 0,
+            selected_ids: [],
+            suppressed_source_keys: [],
+            selected: [],
+            candidates: []
         }
-        const aTime = a.record.createdAt ?? '';
-        const bTime = b.record.createdAt ?? '';
-        return aTime.localeCompare(bTime);
     };
-    const add = (record) => {
-        const score = record.reward.gtScore + record.reward.relativeRank;
-        if (threshold !== null && score < threshold) {
-            return;
+}
+function resolveExperienceSelectionPolicy(policy, minScoreFallback) {
+    return {
+        kind: 'competitive_scoring_v1',
+        minScore: normalizeOptionalScore(policy?.minScore ?? minScoreFallback, 'selection.policy.minScore', null),
+        scoreWeights: {
+            gtScore: normalizeNonNegativeScore(policy?.scoreWeights?.gtScore, 'selection.policy.scoreWeights.gtScore', 1),
+            relativeRank: normalizeNonNegativeScore(policy?.scoreWeights?.relativeRank, 'selection.policy.scoreWeights.relativeRank', 1)
+        },
+        antiDominanceNormalization: {
+            enabled: normalizeBoolean(policy?.antiDominanceNormalization?.enabled, 'selection.policy.antiDominanceNormalization.enabled', true),
+            strength: normalizeNonNegativeScore(policy?.antiDominanceNormalization?.strength, 'selection.policy.antiDominanceNormalization.strength', 0.5),
+            sourceGrouping: 'provenance_fallback_v1'
         }
-        const entry = { record, score };
-        if (entries.length === 0) {
-            entries.push(entry);
-            return;
+    };
+}
+function createSelectionEntry(record, policy) {
+    const source = deriveExperienceSource(record);
+    const rawScore = roundScore(record.reward.gtScore * policy.scoreWeights.gtScore +
+        record.reward.relativeRank * policy.scoreWeights.relativeRank);
+    return {
+        id: record.id,
+        createdAt: record.createdAt,
+        sourceKey: source.key,
+        sourceKind: source.kind,
+        sourceBucketKey: formatExperienceSourceBucketKey(source.kind, source.key),
+        rawScore,
+        competitiveScore: rawScore,
+        dominancePenalty: 0,
+        selected: false,
+        selectedSlot: null,
+        exclusionReason: policy.minScore !== null && rawScore < policy.minScore ? 'raw_score_below_min_score' : null
+    };
+}
+function insertSourceCandidate(entriesBySource, entry, limit) {
+    const bucket = entriesBySource.get(entry.sourceBucketKey) ?? [];
+    let insertAt = 0;
+    while (insertAt < bucket.length &&
+        compareSelectionEntriesByRaw(bucket[insertAt], entry) <= 0) {
+        insertAt += 1;
+    }
+    bucket.splice(insertAt, 0, entry);
+    let prunedEntry = null;
+    if (bucket.length > limit) {
+        prunedEntry = bucket.pop() ?? null;
+    }
+    entriesBySource.set(entry.sourceBucketKey, bucket);
+    return prunedEntry;
+}
+function selectCompetitiveTop(entries, limit, policy, candidateCount = entries.length, sourceCandidateCounts = new Map(), selectedRecordLookup = new Map()) {
+    const remaining = entries.filter((entry) => entry.exclusionReason === null);
+    const selectedCounts = new Map();
+    const selectedEntries = [];
+    for (let slot = 1; slot <= limit && remaining.length > 0; slot += 1) {
+        for (const entry of remaining) {
+            const priorSelections = selectedCounts.get(entry.sourceBucketKey) ?? 0;
+            const dominancePenalty = policy.antiDominanceNormalization.enabled
+                ? roundScore(policy.antiDominanceNormalization.strength * priorSelections)
+                : 0;
+            entry.dominancePenalty = dominancePenalty;
+            entry.competitiveScore = roundScore(entry.rawScore - dominancePenalty);
         }
-        const worst = entries[0];
-        if (entries.length >= limit && worst && compare(entry, worst) <= 0) {
-            return;
+        remaining.sort(compareSelectionEntries);
+        const best = remaining[0];
+        if (!best || (policy.minScore !== null && best.competitiveScore < policy.minScore)) {
+            for (const entry of remaining) {
+                entry.exclusionReason = 'competitive_score_below_min_score';
+            }
+            break;
         }
-        let index = 0;
-        while (index < entries.length && compare(entries[index], entry) <= 0) {
-            index += 1;
+        best.selected = true;
+        best.selectedSlot = slot;
+        best.exclusionReason = null;
+        selectedEntries.push(best);
+        selectedCounts.set(best.sourceBucketKey, (selectedCounts.get(best.sourceBucketKey) ?? 0) + 1);
+        remaining.shift();
+    }
+    for (const entry of remaining) {
+        if (entry.exclusionReason === null) {
+            entry.exclusionReason = 'outcompeted';
         }
-        entries.splice(index, 0, entry);
-        if (entries.length > limit) {
-            entries.shift();
+    }
+    if (sourceCandidateCounts.size === 0) {
+        for (const entry of entries) {
+            sourceCandidateCounts.set(entry.sourceBucketKey, (sourceCandidateCounts.get(entry.sourceBucketKey) ?? 0) + 1);
         }
-    };
-    const finalize = () => entries
-        .slice()
-        .sort((a, b) => {
-        if (a.score !== b.score) {
-            return b.score - a.score;
+    }
+    const suppressedSourceKeys = [...sourceCandidateCounts.entries()]
+        .filter(([sourceBucketKey, count]) => count > (selectedCounts.get(sourceBucketKey) ?? 0))
+        .map(([sourceBucketKey]) => sourceBucketKey)
+        .sort((a, b) => a.localeCompare(b));
+    return {
+        records: selectedEntries.map((entry) => {
+            const record = entry.record ?? selectedRecordLookup.get(entry.id);
+            if (!record) {
+                throw new Error(`Selected experience ${entry.id} is missing a retained record`);
+            }
+            return record;
+        }),
+        diagnostics: {
+            policy,
+            candidate_count: candidateCount,
+            selected_count: selectedEntries.length,
+            selected_ids: selectedEntries.map((entry) => entry.id),
+            suppressed_source_keys: suppressedSourceKeys,
+            selected: selectedEntries.map((entry) => ({
+                id: entry.id,
+                source_key: entry.sourceKey,
+                source_kind: entry.sourceKind,
+                raw_score: entry.rawScore,
+                competitive_score: entry.competitiveScore,
+                dominance_penalty: entry.dominancePenalty
+            })),
+            candidates: entries
+                .slice()
+                .sort((a, b) => compareSelectionEntries(a, b))
+                .map((entry) => ({
+                id: entry.id,
+                source_key: entry.sourceKey,
+                source_kind: entry.sourceKind,
+                raw_score: entry.rawScore,
+                competitive_score: entry.competitiveScore,
+                dominance_penalty: entry.dominancePenalty,
+                selected: entry.selected,
+                selected_slot: entry.selectedSlot,
+                exclusion_reason: entry.exclusionReason
+            }))
         }
-        const aTime = a.record.createdAt ?? '';
-        const bTime = b.record.createdAt ?? '';
-        return bTime.localeCompare(aTime);
-    })
-        .map((entry) => entry.record);
-    return { add, finalize };
+    };
+}
+function deriveExperienceSource(record) {
+    if (record.groupId && record.groupId.trim()) {
+        return { kind: 'group_id', key: record.groupId.trim() };
+    }
+    if (record.runId && record.runId.trim()) {
+        return { kind: 'run_id', key: record.runId.trim() };
+    }
+    if (record.manifestPath && record.manifestPath.trim()) {
+        return { kind: 'manifest_path', key: record.manifestPath.trim() };
+    }
+    return { kind: 'stamp_signature', key: record.stampSignature };
+}
+function compareSelectionEntries(a, b) {
+    if (a.competitiveScore !== b.competitiveScore) {
+        return b.competitiveScore - a.competitiveScore;
+    }
+    return compareSelectionEntriesByRaw(a, b);
+}
+function compareSelectionEntriesByRaw(a, b) {
+    if (a.rawScore !== b.rawScore) {
+        return b.rawScore - a.rawScore;
+    }
+    const aTime = a.createdAt ?? '';
+    const bTime = b.createdAt ?? '';
+    const timeCompare = bTime.localeCompare(aTime);
+    if (timeCompare !== 0) {
+        return timeCompare;
+    }
+    return a.id.localeCompare(b.id);
+}
+function formatExperienceSourceBucketKey(kind, key) {
+    return `${kind}:${key}`;
+}
+function roundScore(value) {
+    if (!Number.isFinite(value)) {
+        return 0;
+    }
+    return Math.round(value * 1_000_000) / 1_000_000;
+}
+function normalizeOptionalScore(value, field, defaultValue) {
+    if (value === null || value === undefined) {
+        return defaultValue;
+    }
+    if (!Number.isFinite(value) || value < 0) {
+        throw new Error(`${field} must be a finite non-negative number.`);
+    }
+    return value;
+}
+function normalizeNonNegativeScore(value, field, defaultValue) {
+    const normalized = normalizeOptionalScore(value, field, defaultValue);
+    if (normalized === null) {
+        throw new Error(`${field} must be a finite non-negative number.`);
+    }
+    return normalized;
+}
+function normalizeBoolean(value, field, defaultValue) {
+    if (value === undefined) {
+        return defaultValue;
+    }
+    if (typeof value !== 'boolean') {
+        throw new Error(`${field} must be a boolean.`);
+    }
+    return value;
 }
 function truncateSummary(value, maxWords) {
     const tokens = value.trim().split(/\s+/u).filter(Boolean);

package/dist/orchestrator/src/persistence/TaskStateStore.js

@@ -1,4 +1,4 @@
-import { mkdir, readFile, rm } from 'node:fs/promises';
+import { mkdir, readFile } from 'node:fs/promises';
 import { dirname, join } from 'node:path';
 import { acquireLockWithRetry } from './lockFile.js';
 import { sanitizeTaskId } from './sanitizeTaskId.js';
@@ -37,7 +37,7 @@ export class TaskStateStore {
     async recordRun(summary) {
         const safeTaskId = sanitizeTaskId(summary.taskId);
         const lockPath = this.buildLockPath(safeTaskId);
-        await this.acquireLock(safeTaskId, lockPath);
+        const lock = await this.acquireLock(safeTaskId, lockPath);
         try {
             await this.ensureDirectory(this.outDir);
             const taskOutDir = join(this.outDir, safeTaskId);
@@ -48,7 +48,7 @@ export class TaskStateStore {
             await writeAtomicFile(snapshotPath, JSON.stringify(updated, null, 2));
         }
         finally {
-            await this.releaseLock(lockPath);
+            await this.releaseLock(lock);
         }
     }
     buildLockPath(taskId) {
@@ -56,7 +56,7 @@ export class TaskStateStore {
         return join(this.runsDir, `${safeTaskId}.lock`);
     }
     async acquireLock(taskId, lockPath) {
-        await acquireLockWithRetry({
+        return await acquireLockWithRetry({
             taskId,
             lockPath,
             retry: this.lockRetry,
@@ -64,8 +64,8 @@ export class TaskStateStore {
             createError: (id, attempts) => new TaskStateStoreLockError(`Failed to acquire task state lock for ${id} after ${attempts} attempts`, id)
         });
     }
-    async releaseLock(lockPath) {
-        await rm(lockPath, { force: true });
+    async releaseLock(lock) {
+        await lock.release();
     }
     async ensureDirectory(path) {
         await mkdir(path, { recursive: true });

package/dist/orchestrator/src/persistence/lockFile.js

@@ -1,4 +1,5 @@
-import { open, rm, stat } from 'node:fs/promises';
+import { randomUUID } from 'node:crypto';
+import { open, readFile, rm, stat } from 'node:fs/promises';
 import { setTimeout as delay } from 'node:timers/promises';
 export async function acquireLockWithRetry(params) {
     await params.ensureDirectory();
@@ -9,9 +10,17 @@ export async function acquireLockWithRetry(params) {
     while (attempt < maxAttempts) {
         attempt += 1;
         try {
-            const handle = await open(params.lockPath, 'wx');
-            await handle.close();
-            return;
+            const ownerToken = randomUUID();
+            const handle = await open(params.lockPath, 'wx+');
+            try {
+                await handle.writeFile(ownerToken, 'utf8');
+            }
+            catch (error) {
+                await handle.close();
+                await rm(params.lockPath, { force: true });
+                throw error;
+            }
+            return createAcquiredLock(params.lockPath, ownerToken, handle, staleMs);
         }
         catch (error) {
             if (error.code !== 'EEXIST') {
@@ -31,6 +40,63 @@ export async function acquireLockWithRetry(params) {
             delayMs = Math.min(delayMs * backoffFactor, maxDelayMs);
         }
     }
+    throw params.createError(params.taskId, attempt);
+}
+function createAcquiredLock(lockPath, ownerToken, handle, staleMs) {
+    let released = false;
+    const keepalive = createLockKeepalive(handle, staleMs);
+    return {
+        lockPath,
+        ownerToken,
+        async release() {
+            if (released) {
+                return;
+            }
+            released = true;
+            keepalive.stop();
+            await handle.close();
+            await releaseLockIfOwned(lockPath, ownerToken);
+        }
+    };
+}
+function createLockKeepalive(handle, staleMs) {
+    if (staleMs <= 0) {
+        return { stop() { } };
+    }
+    const intervalMs = Math.max(10, Math.floor(staleMs / 2));
+    const timer = setInterval(() => {
+        void touchLockHandle(handle);
+    }, intervalMs);
+    timer.unref?.();
+    return {
+        stop() {
+            clearInterval(timer);
+        }
+    };
+}
+async function touchLockHandle(handle) {
+    try {
+        const now = new Date();
+        await handle.utimes(now, now);
+    }
+    catch {
+        // Ignore keepalive failures; acquisition/release paths still surface real errors.
+    }
+}
+async function releaseLockIfOwned(lockPath, ownerToken) {
+    try {
+        const currentOwner = (await readFile(lockPath, 'utf8')).trim();
+        if (currentOwner !== ownerToken) {
+            return;
+        }
+        await rm(lockPath, { force: true });
+    }
+    catch (error) {
+        if (error.code === 'ENOENT') {
+            return;
+        }
+        throw error;
+    }
 }
 async function clearStaleLock(lockPath, staleMs) {
     try {

package/dist/orchestrator/src/persistence/sanitizeIdentifier.js

@@ -1,4 +1,37 @@
 const WINDOWS_FORBIDDEN_CHARACTERS = new Set(['<', '>', ':', '"', '|', '?', '*']);
+const WINDOWS_RESERVED_DEVICE_NAMES = new Set([
+    'CON',
+    'PRN',
+    'AUX',
+    'NUL',
+    'COM1',
+    'COM2',
+    'COM3',
+    'COM4',
+    'COM5',
+    'COM6',
+    'COM7',
+    'COM8',
+    'COM9',
+    'LPT1',
+    'LPT2',
+    'LPT3',
+    'LPT4',
+    'LPT5',
+    'LPT6',
+    'LPT7',
+    'LPT8',
+    'LPT9',
+    'CONIN$',
+    'CONOUT$'
+]);
+function isWindowsReservedDeviceName(value) {
+    const baseName = value.split('.')[0]?.trimEnd();
+    if (!baseName) {
+        return false;
+    }
+    return WINDOWS_RESERVED_DEVICE_NAMES.has(baseName.toUpperCase());
+}
 export function sanitizeIdentifier(kind, value) {
     const label = kind === 'task' ? 'task' : 'run';
     if (!value) {
@@ -22,5 +55,11 @@ export function sanitizeIdentifier(kind, value) {
     if (value.includes('/') || value.includes('\\')) {
         throw new Error(`Invalid ${label} ID "${value}": slashes are not allowed.`);
     }
+    if (value.endsWith('.') || value.endsWith(' ')) {
+        throw new Error(`Invalid ${label} ID "${value}": trailing dots or spaces are not allowed.`);
+    }
+    if (isWindowsReservedDeviceName(value)) {
+        throw new Error(`Invalid ${label} ID "${value}": Windows reserved device names are not allowed.`);
+    }
     return value;
 }

package/dist/orchestrator/src/sync/createCloudSyncWorker.js

@@ -2,8 +2,9 @@ import { CloudRunsHttpClient } from './CloudRunsHttpClient.js';
 import { CloudSyncWorker } from './CloudSyncWorker.js';
 /**
  * Convenience factory that wires the CloudRuns HTTP client to the CloudSync worker using
- * the credential broker defined in the architecture spec. Callers can override client or
- * worker options as needed.
+ * the credential broker defined in the architecture spec. This remains opt-in integration
+ * glue: the default CLI does not construct or start a CloudSyncWorker unless a caller wires
+ * it explicitly. Callers can override client or worker options as needed.
  */
 export function createCloudSyncWorker(params) {
     const client = new CloudRunsHttpClient({

package/dist/orchestrator/src/utils/atomicWrite.js

@@ -1,4 +1,4 @@
-import { mkdir, rename, writeFile } from 'node:fs/promises';
+import { chmod, mkdir, rename, stat, writeFile } from 'node:fs/promises';
 import { dirname } from 'node:path';
 let atomicWriteCounter = 0;
 export function buildAtomicTempPath(destination, now = Date.now, pid = process.pid) {
@@ -10,6 +10,21 @@ export async function writeAtomicFile(destination, contents, options = {}) {
     if (options.ensureDir) {
         await mkdir(dirname(destination), { recursive: true });
     }
-    await writeFile(tmpPath, contents, options.encoding ?? 'utf8');
+    let existingMode = null;
+    try {
+        existingMode = (await stat(destination)).mode & 0o777;
+    }
+    catch (error) {
+        if (error.code !== 'ENOENT') {
+            throw error;
+        }
+    }
+    await writeFile(tmpPath, contents, {
+        encoding: options.encoding ?? 'utf8',
+        mode: existingMode ?? undefined
+    });
+    if (existingMode !== null) {
+        await chmod(tmpPath, existingMode);
+    }
     await rename(tmpPath, destination);
 }