@on-par/factory-cli 0.9.7-next.5 → 0.9.7-next.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/doctor-config.d.ts +14 -0
- package/dist/cli/doctor-config.d.ts.map +1 -0
- package/dist/cli/doctor-config.js +126 -0
- package/dist/cli/doctor-config.js.map +1 -0
- package/dist/cli/doctor-health.d.ts +73 -0
- package/dist/cli/doctor-health.d.ts.map +1 -0
- package/dist/cli/doctor-health.js +156 -0
- package/dist/cli/doctor-health.js.map +1 -0
- package/dist/cli/doctor.d.ts +26 -81
- package/dist/cli/doctor.d.ts.map +1 -1
- package/dist/cli/doctor.js +213 -267
- package/dist/cli/doctor.js.map +1 -1
- package/dist/cli/index.d.ts +14 -65
- package/dist/cli/index.d.ts.map +1 -1
- package/dist/cli/index.js +58 -641
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/run-lane.d.ts +52 -0
- package/dist/cli/run-lane.d.ts.map +1 -0
- package/dist/cli/run-lane.js +239 -0
- package/dist/cli/run-lane.js.map +1 -0
- package/dist/cli/run.d.ts +52 -0
- package/dist/cli/run.d.ts.map +1 -0
- package/dist/cli/run.js +200 -0
- package/dist/cli/run.js.map +1 -0
- package/package.json +3 -3
package/dist/cli/index.js
CHANGED
|
@@ -1,21 +1,21 @@
|
|
|
1
1
|
import { fileURLToPath } from 'node:url';
|
|
2
2
|
// packages/cli/src/cli/index.ts — CLI entry point: factory <command> [options]
|
|
3
3
|
import { exec as execCb, execSync } from 'node:child_process';
|
|
4
|
-
import { randomUUID } from 'node:crypto';
|
|
5
4
|
import { appendFileSync, existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, unlinkSync, writeFileSync, } from 'node:fs';
|
|
6
5
|
import { createRequire } from 'node:module';
|
|
7
6
|
import { arch, hostname as osHostname, release, platform, userInfo } from 'node:os';
|
|
8
7
|
import { basename, dirname, isAbsolute, join, relative, resolve } from 'node:path';
|
|
9
8
|
import { promisify, styleText } from 'node:util';
|
|
10
9
|
import { INVALID_BRANCH_PREFIX_MESSAGE, resolveBranchPrefixOption } from './branch-prefix.js';
|
|
11
|
-
import { acquirePortLease, aggregateCosts, appendKpiHistoryLine, applyRepoConfig, buildPhase, checkPhase, clearProxyState, computeHealthKpis, ConstitutionLoader, createDefaultWorkSourceRegistry, createLogger, createLaneProxy, defaultFindPortListeners, describeEffectiveConfig, detectPostMergeDefects, diagnoseModels, probeDeclaredHarnesses, estimateTrailingSpend, fetchDefectSources, fetchHumanEventSources, fetchSubscriptionUsage, formatUsageReport, getConstitutionsDir, getFactoryPaths, GITHUB_ISSUE_SOURCE, hasUnresolvedPark, InvalidArtifactsDirError, InvalidWorkRequestInputError, InvalidWorkspaceError,
|
|
12
|
-
import { acquirePidFile, createCheckResumeStore, findSlicePlanComment,
|
|
10
|
+
import { acquirePortLease, aggregateCosts, appendKpiHistoryLine, applyRepoConfig, buildPhase, checkPhase, clearProxyState, computeHealthKpis, ConstitutionLoader, createDefaultWorkSourceRegistry, createLogger, createLaneProxy, defaultFindPortListeners, describeEffectiveConfig, detectPostMergeDefects, diagnoseModels, probeDeclaredHarnesses, estimateTrailingSpend, fetchDefectSources, fetchHumanEventSources, fetchSubscriptionUsage, formatUsageReport, getConstitutionsDir, getFactoryPaths, GITHUB_ISSUE_SOURCE, hasUnresolvedPark, InvalidArtifactsDirError, InvalidWorkRequestInputError, InvalidWorkspaceError, isCommandAvailable, isProxyRunning, kpiBranchPrefixes, kpisToHistoryRecord, laneBaseUrl, laneEnv, laneHostLabel, loadFactoryConfigForRepo, loadModelsConfig, loadRepoConfig, loadRoutesConfig, LOCAL_BRIEF_SOURCE, joinClassifierOutcomes, mergeClassifierOutcomes, parseClassifierOutcomes, summarizeClassifierOutcomes, formatClassifierReport, mergedPrRefs, ModelRegistry, ModelRouter, parkReasonFor, parseKpiHistory, parseQueue, planPhase, readEvents, readPortLeases, readQueue, readUsage, reapOrphanProcesses, reconstructKpiEvents, recordLeasePgid, releasePortLease, renderKpiReport, renderKpiTrend, resolveDefectWindowDays, parseSizeGateMode, parseMaxSlices, SIZE_GATE_MODES, resolveEffectiveModelPins, resolveEnvironmentProxy, resolveArtifactsDir, resolveLocalOnlyPolicy, resolveMergePolicy, resolveSandboxPolicy, resolveSkipCI, resolveTimeouts, resolveUsageCap, resolveWatchdogPolicy, runAutoIngest, shipPhase, formatCostTotal, validateQueue, watchUsage, writeLocalRunReport, writeProxyState, } from '@on-par/factory-core';
|
|
11
|
+
import { acquirePidFile, createCheckResumeStore, findSlicePlanComment, upsertSlicePlanComment, withSliceState, DEFAULT_GARDEN_MAX_CLUSTERS, DEFAULT_GARDEN_SINCE, capGardenClusters, clusterGarden, dedupGardenClusters, filterGardenOnlyNew, filterGardenWindow, markGardenTrackingUnknown, parseGardenDuration, readHarvestEvents, renderGardenJson, renderGardenReport, laneWorktreePath, migrateRepoConfigToYaml, REPO_CONFIG_YAML_HEADER, claudeConfigDirOverride, claudeKeychainService, cleanupWorktree, createDaemonLogSink, factoryBranchIssue, factoryBranchPrefixes, createFactorydServer, createRunRuntime, createShipExecutor, createGithubQueue, createLocalSmallDryRun, createOctokitFilingClient, createOctokitGreenPrClient, createOctokitQueueClient, daemonRuntimePaths, defaultRegistryPath, ensureDir, FACTORY_RUNTIME_CONFIG_KEYS, findUnmergedGreenPrs, formatGcReport, gitFetch, listMicroVms, listOrphanContainers, logCost, logEvent, planQueueMigration, readCosts, readGithubQueueSnapshot, reapLaneWorktree, reapOrphanContainers, reapOrphanMicroVm, releaseRuntimeFiles, releaseStaleClaims, resolveBranchPrefix, resolveEffectiveConfig, worktreeSandboxFor, resolveExperimental, resolveLocalOnly, resolvePrClassifierPolicy, resolveReviewFloorRules, RunLockHeldError, runOvernightQueue, parseRemoteSlug, setupWorktree, shellEscape, listBuiltInCheckers, sweepWorktrees, watchChecks, withFileLock, withGitLock, withRunLock, writePortFile, buildTriagePrompt, normalizeTriageQueue, parseTriageCandidates, recordLocalAuthFailure, } from '@on-par/factory-core/internal';
|
|
13
12
|
import { runTui } from '@on-par/factory-tui';
|
|
14
13
|
import { admissionStateReaderFor, runQueueReconcile } from './admission.js';
|
|
15
14
|
import { cmdClassifierBacktest, ClassifierBacktestInputError, } from './classifier-backtest.js';
|
|
16
|
-
import { findOpenPRForIssue, landIssue
|
|
15
|
+
import { findOpenPRForIssue, landIssue } from './land.js';
|
|
17
16
|
import { AwaitingReviewError, CiFailedError, CiUnverifiedError, getPullRequestLandState, LandConflictError, LandFailureError, MergeGatedError, } from './land-pr.js';
|
|
18
17
|
import { setLandRuntime } from './land-runtime.js';
|
|
18
|
+
import { cmdRun } from './run.js';
|
|
19
19
|
import { shipIssue } from './ship.js';
|
|
20
20
|
import { cmdStatus } from './status.js';
|
|
21
21
|
import { cmdSupervise } from './supervise.js';
|
|
@@ -34,7 +34,7 @@ import { buildUsageJson } from './usage-json.js';
|
|
|
34
34
|
import { parseResetIssues, runReset } from './reset.js';
|
|
35
35
|
import { cmdDaemonLogs, cmdDaemonStart, cmdDaemonStatus, cmdDaemonStop, createAutoUpdateDeps, DaemonCtlError, detectInstall, loadDaemonFileConfig, recoverStaleDrain, resolveFactorydConfig, runAutoUpdateLoop, } from './daemon.js';
|
|
36
36
|
import { checkForUpdate, cmdSelfUpdateCheck } from './self-update.js';
|
|
37
|
-
import {
|
|
37
|
+
import { cmdDoctor } from './doctor.js';
|
|
38
38
|
import { DOCS_URL, formatOverview, availableAgentClis, missingAgentCliMessage, missingTokenMessage, notInitializedMessage, } from './first-run.js';
|
|
39
39
|
import { cmdHostedSmoke } from './hosted.js';
|
|
40
40
|
import { cmdHostedQueue } from './hosted-queue.js';
|
|
@@ -48,11 +48,8 @@ import { selectOctokit } from './github-stub.js';
|
|
|
48
48
|
import { createFactoryOctokit } from './octokit.js';
|
|
49
49
|
import { DeepCheckError, createDeepModelRunner, runIssueCheck } from './ready-check.js';
|
|
50
50
|
import { childRunSucceeded, formatChildRunSummary, runChildrenInOrder } from './run-children.js';
|
|
51
|
-
import {
|
|
52
|
-
import {
|
|
53
|
-
import { distFreshnessProbe, runStalenessGuard } from './staleness.js';
|
|
54
|
-
import { readStopFileStatus, stopSentinelCheck, stopSentinelRunSkipMessage } from './stop-sentinel.js';
|
|
55
|
-
import { checkSweepHeartbeat, defaultSweepHeartbeatDeps, sweepHeartbeatCheck } from './sweep-heartbeat.js';
|
|
51
|
+
import { readRunFlagOverrides } from './run-flags.js';
|
|
52
|
+
import { runStalenessGuard } from './staleness.js';
|
|
56
53
|
const DEFAULT_CONFIG_LOADERS = {
|
|
57
54
|
loadModelsConfig,
|
|
58
55
|
loadRoutesConfig,
|
|
@@ -2518,194 +2515,9 @@ function usageFlagOverrides(opts, threshold) {
|
|
|
2518
2515
|
pollSeconds: opts.usagePoll === undefined ? undefined : Number(opts.usagePoll),
|
|
2519
2516
|
};
|
|
2520
2517
|
}
|
|
2521
|
-
async function cmdRun(opts = {}) {
|
|
2522
|
-
const repoRoot = await getRepoRoot();
|
|
2523
|
-
const paths = getFactoryPaths(repoRoot);
|
|
2524
|
-
const branchPrefix = branchPrefixOrExit(opts.branchPrefix);
|
|
2525
|
-
return withRepoRunLock(paths, 'factory run', async () => {
|
|
2526
|
-
const stopStatus = readStopFileStatus(paths);
|
|
2527
|
-
if (stopStatus.present) {
|
|
2528
|
-
const msg = stopSentinelRunSkipMessage(stopStatus);
|
|
2529
|
-
console.log(styleText('yellow', `!! ${msg}`));
|
|
2530
|
-
logEvent(paths.events, 'stopped', 'all', msg);
|
|
2531
|
-
return;
|
|
2532
|
-
}
|
|
2533
|
-
writeRunFlagOverrides(paths.runFlags, { autoMerge: opts.autoMerge, prClassifier: opts.prClassifier });
|
|
2534
|
-
const ghRepo = await getGitHubRepo();
|
|
2535
|
-
const factoryConfig = configLoaders.loadFactoryConfigForRepo(paths.config);
|
|
2536
|
-
const keychainErr = keychainPreflightError(probeClaudeKeychain());
|
|
2537
|
-
if (keychainErr) {
|
|
2538
|
-
logEvent(paths.events, 'environment_warning', 'all', keychainErr);
|
|
2539
|
-
throw new Error(keychainErr);
|
|
2540
|
-
}
|
|
2541
|
-
await runAutoWorktreeGc(repoRoot, paths, ghRepo, 'all');
|
|
2542
|
-
const { lanes, diagnostics } = await planRunLanes({
|
|
2543
|
-
localQueue: opts.localQueue === true,
|
|
2544
|
-
readLocalQueue: () => (existsSync(paths.queue) ? readFileSync(paths.queue, 'utf-8') : null),
|
|
2545
|
-
queue: () => {
|
|
2546
|
-
const [owner, repo] = ghRepo.split('/');
|
|
2547
|
-
const octokit = getOctokit();
|
|
2548
|
-
return createGithubQueue({
|
|
2549
|
-
client: createOctokitQueueClient(octokit),
|
|
2550
|
-
owner,
|
|
2551
|
-
repo,
|
|
2552
|
-
preflight: withAdmissionGuard({
|
|
2553
|
-
reader: admissionStateReaderFor(paths.state),
|
|
2554
|
-
preflight: (issue) => preflightQueuedIssue(issue, createQueuePreflightOps(octokit, owner, repo, {
|
|
2555
|
-
autoMergeStack: resolveSizeGateMode(factoryConfig) === 'stack' &&
|
|
2556
|
-
resolveMergePolicy(factoryConfig, process.env, { auto: opts.autoMerge }).auto,
|
|
2557
|
-
})),
|
|
2558
|
-
onConflict: (issue, verdict) => {
|
|
2559
|
-
const line = `#${issue} refused — ${verdict.reason} | repair: ${verdict.repair}`;
|
|
2560
|
-
console.error(styleText('yellow', `factory: ${line}`));
|
|
2561
|
-
logEvent(paths.events, 'queue_admission_conflict', String(issue), line);
|
|
2562
|
-
},
|
|
2563
|
-
}),
|
|
2564
|
-
});
|
|
2565
|
-
},
|
|
2566
|
-
});
|
|
2567
|
-
warnQueueDiagnostics(diagnostics);
|
|
2568
|
-
let knobs;
|
|
2569
|
-
let laneBreakerThreshold;
|
|
2570
|
-
try {
|
|
2571
|
-
knobs = resolveUsageKnobs(process.env, configLoaders.loadRepoConfig(repoRoot), usageFlagOverrides(opts, 'stopAt'));
|
|
2572
|
-
laneBreakerThreshold = resolveLaneBreakerThreshold(configLoaders.loadRepoConfig(repoRoot));
|
|
2573
|
-
}
|
|
2574
|
-
catch (err) {
|
|
2575
|
-
throw new CliExitError(`factory: ${err.message}`, 2);
|
|
2576
|
-
}
|
|
2577
|
-
if (knobs.watchSource === 'flag') {
|
|
2578
|
-
const detail = `usage watchdog ${knobs.watch ? 'enabled' : 'disabled'} by ${usageWatchSourceLabel(knobs.watchSource, knobs.watch)}`;
|
|
2579
|
-
console.log(`[factory] run: ${detail}`);
|
|
2580
|
-
logEvent(paths.events, 'watchdog', 'all', detail);
|
|
2581
|
-
}
|
|
2582
|
-
if (knobs.stopAtSource === 'flag') {
|
|
2583
|
-
const detail = 'usage watchdog threshold set by flag: --usage-threshold';
|
|
2584
|
-
console.log(`[factory] run: ${detail}`);
|
|
2585
|
-
logEvent(paths.events, 'watchdog', 'all', detail);
|
|
2586
|
-
}
|
|
2587
|
-
if (knobs.pollSource === 'flag') {
|
|
2588
|
-
const detail = 'usage watchdog poll set by flag: --usage-poll';
|
|
2589
|
-
console.log(`[factory] run: ${detail}`);
|
|
2590
|
-
logEvent(paths.events, 'watchdog', 'all', detail);
|
|
2591
|
-
}
|
|
2592
|
-
let watchdogHarness;
|
|
2593
|
-
if (knobs.watch) {
|
|
2594
|
-
try {
|
|
2595
|
-
const repoConfig = configLoaders.loadRepoConfig(repoRoot);
|
|
2596
|
-
const registry = new coreOps.ModelRegistry(applyRepoConfig(configLoaders.loadModelsConfig(), repoConfig));
|
|
2597
|
-
const pins = configLoaders.resolveEffectiveModelPins(registry, repoConfig, process.env);
|
|
2598
|
-
const build = resolveEffectiveBuildRoute(registry, repoConfig, pins);
|
|
2599
|
-
watchdogHarness = resolveRunUsageHarness({
|
|
2600
|
-
buildRoute: build.route,
|
|
2601
|
-
buildRouteSource: build.source,
|
|
2602
|
-
planHarnesses: [pins.plan, pins.planFallback]
|
|
2603
|
-
.filter((m) => !!m)
|
|
2604
|
-
.map((m) => registry.getHarnessId(m)),
|
|
2605
|
-
codexDisabled: resolveCodexDisabled(repoConfig),
|
|
2606
|
-
localOnly: resolveLocalOnly(),
|
|
2607
|
-
});
|
|
2608
|
-
}
|
|
2609
|
-
catch {
|
|
2610
|
-
watchdogHarness = undefined; // unresolvable config keeps today's Claude gating
|
|
2611
|
-
}
|
|
2612
|
-
}
|
|
2613
|
-
const controller = new AbortController();
|
|
2614
|
-
const watchdog = knobs.watch
|
|
2615
|
-
? coreOps
|
|
2616
|
-
.watchUsage({
|
|
2617
|
-
cap: knobs.cap,
|
|
2618
|
-
stopAt: knobs.stopAt,
|
|
2619
|
-
pollMs: knobs.pollMs,
|
|
2620
|
-
stopFile: paths.stop,
|
|
2621
|
-
eventsFile: paths.events,
|
|
2622
|
-
signal: controller.signal,
|
|
2623
|
-
estimator: knobs.estimator,
|
|
2624
|
-
...(watchdogHarness ? { harness: watchdogHarness } : {}),
|
|
2625
|
-
})
|
|
2626
|
-
.catch((err) => {
|
|
2627
|
-
// a watchdog crash must never take down the run
|
|
2628
|
-
logEvent(paths.events, 'warn', '-', `usage watchdog crashed: ${err.message}`);
|
|
2629
|
-
})
|
|
2630
|
-
: Promise.resolve();
|
|
2631
|
-
const { proxy } = await startLaneProxy(paths, resolveEnvironmentProxy(factoryConfig));
|
|
2632
|
-
const workspaceBackend = factoryConfig.workspace?.backend ?? 'host';
|
|
2633
|
-
const runId = randomUUID();
|
|
2634
|
-
const containerEngine = workspaceBackend === 'disposable-docker' ? createDockerEngine({}) : undefined;
|
|
2635
|
-
try {
|
|
2636
|
-
// Run lanes in parallel
|
|
2637
|
-
const pids = [];
|
|
2638
|
-
for (const planned of lanes) {
|
|
2639
|
-
logEvent(paths.events, 'lane-start', '-', `lane '${planned.lane}' started${planned.issues.length ? ` (${planned.issues.length} issues)` : ''}`, { lane: planned.lane });
|
|
2640
|
-
if (containerEngine) {
|
|
2641
|
-
const provision = await provisionLaneContainer(containerEngine, workspaceBackend, runId, slugify(planned.lane), ghRepo);
|
|
2642
|
-
if (provision.attempted && !provision.created) {
|
|
2643
|
-
logEvent(paths.events, 'warn', '-', `disposable-docker container creation failed for lane '${planned.lane}': ${provision.error}`, { lane: planned.lane });
|
|
2644
|
-
}
|
|
2645
|
-
else if (provision.attempted && provision.created && provision.workspaceCloned === false) {
|
|
2646
|
-
logEvent(paths.events, 'warn', '-', `disposable-docker workspace clone failed for lane '${planned.lane}': ${provision.workspaceError}`, { lane: planned.lane });
|
|
2647
|
-
}
|
|
2648
|
-
}
|
|
2649
|
-
pids.push(runLane(planned.lane, planned.issues, repoRoot, ghRepo, paths, {
|
|
2650
|
-
...planned.deps,
|
|
2651
|
-
laneBreakerThreshold,
|
|
2652
|
-
baseHoldGate: opts.localQueue === true
|
|
2653
|
-
? undefined
|
|
2654
|
-
: createBaseHoldGate({
|
|
2655
|
-
file: paths.baseHold,
|
|
2656
|
-
repo: ghRepo,
|
|
2657
|
-
onCleared: (c) => logEvent(paths.events, 'base-hold-cleared', planned.lane, formatBaseHoldCleared(c), {
|
|
2658
|
-
lane: planned.lane,
|
|
2659
|
-
}),
|
|
2660
|
-
git: (cmd) => shellOps.exec(cmd, { cwd: repoRoot }).then((r) => r.stdout),
|
|
2661
|
-
listCheckRuns: (ref) => {
|
|
2662
|
-
const [owner, repoName] = ghRepo.split('/');
|
|
2663
|
-
return createOctokitGreenPrClient(getOctokit()).listCheckRuns({ owner, repo: repoName, ref });
|
|
2664
|
-
},
|
|
2665
|
-
}),
|
|
2666
|
-
commentIssue: async (issue, body) => {
|
|
2667
|
-
const [owner, repoName] = ghRepo.split('/');
|
|
2668
|
-
await getOctokit().rest.issues.createComment({ owner, repo: repoName, issue_number: issue, body });
|
|
2669
|
-
},
|
|
2670
|
-
ship: (issue, o, c) => shipIssue(issue, { ...o, branchPrefix, prClassifier: opts.prClassifier }, c),
|
|
2671
|
-
sweepFinished: (issue) => sweepBetweenLaneIssues(issue, planned.lane, repoRoot, ghRepo, paths).then(() => { }),
|
|
2672
|
-
reapWorktree: (issue) => reapParkedLaneWorktree(issue, repoRoot, ghRepo, paths, opts.branchPrefix === undefined ? undefined : branchPrefix),
|
|
2673
|
-
recordStackMerge: (issue, prNumber) => recordSliceMerged({ octokit: getOctokit(), repo: ghRepo, issue, prNumber }).then(() => { }),
|
|
2674
|
-
waitMerge: (issue, branch, root, gh, p, d = {}) => waitForMerge(issue, branch, root, gh, p, { ...d, mergeOverrides: { auto: opts.autoMerge } }),
|
|
2675
|
-
}));
|
|
2676
|
-
}
|
|
2677
|
-
await Promise.allSettled(pids);
|
|
2678
|
-
controller.abort();
|
|
2679
|
-
await watchdog;
|
|
2680
|
-
logEvent(paths.events, 'run-done', 'all', 'all lanes finished');
|
|
2681
|
-
}
|
|
2682
|
-
finally {
|
|
2683
|
-
if (proxy) {
|
|
2684
|
-
await proxy.stop();
|
|
2685
|
-
clearProxyState(paths.proxyState);
|
|
2686
|
-
}
|
|
2687
|
-
}
|
|
2688
|
-
if (lanes.length > 0) {
|
|
2689
|
-
try {
|
|
2690
|
-
const events = existsSync(paths.events) ? readEvents(paths.events) : [];
|
|
2691
|
-
const costs = existsSync(paths.costs) ? internalOps.readCosts(paths.costs) : [];
|
|
2692
|
-
const recon = await reconstructHumanEventsForKpis(events, branchPrefix);
|
|
2693
|
-
const kpis = computeHealthKpis(recon.events, costs);
|
|
2694
|
-
await appendKpiSnapshot(paths, repoRoot, kpis, { partial: recon.partial });
|
|
2695
|
-
if (recon.partial) {
|
|
2696
|
-
logEvent(paths.events, 'warn', 'all', `kpi snapshot partial: GitHub human-event reconstruction failed: ${recon.error}`);
|
|
2697
|
-
}
|
|
2698
|
-
logEvent(paths.events, 'kpi-snapshot', 'all', `KPI snapshot appended to ${paths.kpiHistory}`);
|
|
2699
|
-
}
|
|
2700
|
-
catch (err) {
|
|
2701
|
-
logEvent(paths.events, 'warn', 'all', `kpi snapshot failed: ${err.message}`);
|
|
2702
|
-
}
|
|
2703
|
-
}
|
|
2704
|
-
});
|
|
2705
|
-
}
|
|
2706
2518
|
/** Wraps cmdRun so an empty queue is a no-op (not a throw) while auto-ingest is enabled and watching. */
|
|
2707
2519
|
export function createSuperviseRunQueue(paths, ingestCfg, deps = {}) {
|
|
2708
|
-
const { cmdRunFn =
|
|
2520
|
+
const { cmdRunFn = runRun, readQueueFile = () => (existsSync(paths.queue) ? readFileSync(paths.queue, 'utf-8') : ''), pendingCount = async () => parseQueue(readQueueFile()).entries.length, emitEvent = logEvent, } = deps;
|
|
2709
2521
|
return async () => {
|
|
2710
2522
|
if ((await pendingCount()) === 0 && ingestCfg.enabled) {
|
|
2711
2523
|
emitEvent(paths.events, 'idle', 'auto', 'queue empty — waiting for ready issues');
|
|
@@ -2938,234 +2750,6 @@ export async function planRunLanes(input) {
|
|
|
2938
2750
|
diagnostics: [],
|
|
2939
2751
|
};
|
|
2940
2752
|
}
|
|
2941
|
-
/** How often an in-flight issue's claim lease is refreshed (#1500). Well inside the
|
|
2942
|
-
* 15-minute default lease (github-queue.ts's DEFAULT_CLAIM_LEASE_MS) so a normal poll
|
|
2943
|
-
* cadence never lets the lease lapse mid-ship. */
|
|
2944
|
-
const HEARTBEAT_INTERVAL_MS = 5 * 60 * 1000;
|
|
2945
|
-
/** Runs `work` while periodically calling `heartbeat(issue)` so a long-running ship or
|
|
2946
|
-
* merge wait keeps this claimant's lease fresh. A heartbeat failure is swallowed — losing
|
|
2947
|
-
* one refresh must never abort the work it is protecting. */
|
|
2948
|
-
async function withHeartbeat(issue, heartbeat, work) {
|
|
2949
|
-
if (!heartbeat)
|
|
2950
|
-
return work();
|
|
2951
|
-
const timer = setInterval(() => {
|
|
2952
|
-
heartbeat(issue).catch(() => { });
|
|
2953
|
-
}, HEARTBEAT_INTERVAL_MS);
|
|
2954
|
-
try {
|
|
2955
|
-
return await work();
|
|
2956
|
-
}
|
|
2957
|
-
finally {
|
|
2958
|
-
clearInterval(timer);
|
|
2959
|
-
}
|
|
2960
|
-
}
|
|
2961
|
-
export function renderLanePausedComment(payload) {
|
|
2962
|
-
const checks = payload.failingChecks.length
|
|
2963
|
-
? payload.failingChecks.map((c) => `\`${c}\``).join(', ')
|
|
2964
|
-
: 'none recorded';
|
|
2965
|
-
return [
|
|
2966
|
-
`Lane '${payload.lane}' is paused: this issue and #${payload.firstIssue} parked with the same failure signature, so the factory stopped claiming issues on this lane.`,
|
|
2967
|
-
`Shared failing checks: ${checks}`,
|
|
2968
|
-
`To resume: fix the shared failure (often on the base branch), then re-run \`factory run\` to resume lane '${payload.lane}'.`,
|
|
2969
|
-
`Failure signature: \`${payload.signature}\``,
|
|
2970
|
-
].join('\n\n');
|
|
2971
|
-
}
|
|
2972
|
-
export async function runLane(lane, issues, repoRoot, ghRepo, paths, deps = {}) {
|
|
2973
|
-
const { ship = shipIssue, waitMerge = waitForMerge, recordStackMerge = async () => { }, pathExists = existsSync, emitEvent = logEvent, claimNext = async () => null, releaseIssue = async () => { }, heartbeat, reapWorktree = async () => { }, sweepFinished = async () => { }, countRemaining, baseHoldGate, laneBreakerThreshold = 0, commentIssue = async () => { }, } = deps;
|
|
2974
|
-
let streak = null;
|
|
2975
|
-
let merged = 0;
|
|
2976
|
-
let awaitingReview = 0;
|
|
2977
|
-
let skipped = 0;
|
|
2978
|
-
let decomposed = 0;
|
|
2979
|
-
let parked = 0;
|
|
2980
|
-
const buildClaim = (issue) => ({ issue, decision: { kind: 'build' } });
|
|
2981
|
-
const pending = issues.map(buildClaim);
|
|
2982
|
-
const seen = new Set(issues);
|
|
2983
|
-
// Truthful park-time remaining count (#1222): local pending plus the lane's still-queued
|
|
2984
|
-
// remote issues. Fail-closed — a count lookup may never change the message beyond the
|
|
2985
|
-
// local truth, and never lane flow.
|
|
2986
|
-
const remainingAfter = async (index) => {
|
|
2987
|
-
const local = pending.length - index - 1;
|
|
2988
|
-
if (!countRemaining)
|
|
2989
|
-
return local;
|
|
2990
|
-
try {
|
|
2991
|
-
return local + (await countRemaining());
|
|
2992
|
-
}
|
|
2993
|
-
catch {
|
|
2994
|
-
return local;
|
|
2995
|
-
}
|
|
2996
|
-
};
|
|
2997
|
-
// Every terminal outcome releases the claim, then gives finished worktrees a GC pass before
|
|
2998
|
-
// the lane moves on (#1757). The pass is fail-closed and must never change lane flow.
|
|
2999
|
-
const settle = async (issue, outcome) => {
|
|
3000
|
-
await releaseIssue(issue, outcome);
|
|
3001
|
-
await sweepFinished(issue).catch(() => { });
|
|
3002
|
-
};
|
|
3003
|
-
for (let i = 0;; i++) {
|
|
3004
|
-
if (i >= pending.length) {
|
|
3005
|
-
// A persisted base hold on the unchanged origin/main SHA means the base is still broken
|
|
3006
|
-
// (#2349): claiming would just cycle an issue again, so stop this lane without claiming.
|
|
3007
|
-
const gate = baseHoldGate ? await baseHoldGate().catch(() => ({ kind: 'claim' })) : null;
|
|
3008
|
-
if (gate?.kind === 'hold') {
|
|
3009
|
-
emitEvent(paths.events, 'warn', lane, `lane '${lane}' not claiming: base hold on ${gate.hold.baseSha.slice(0, 8)} (${gate.hold.failingChecks.join(', ')}) — origin/main unchanged and its CI is not green`, { lane });
|
|
3010
|
-
return;
|
|
3011
|
-
}
|
|
3012
|
-
const claimed = await claimNext();
|
|
3013
|
-
if (claimed === null)
|
|
3014
|
-
break;
|
|
3015
|
-
seen.add(claimed.issue);
|
|
3016
|
-
pending.push(claimed);
|
|
3017
|
-
}
|
|
3018
|
-
const { issue, decision } = pending[i];
|
|
3019
|
-
if (pathExists(paths.stop)) {
|
|
3020
|
-
emitEvent(paths.events, 'stopped', issue, 'STOP file present', { lane });
|
|
3021
|
-
await releaseIssue(issue, 'queued');
|
|
3022
|
-
return;
|
|
3023
|
-
}
|
|
3024
|
-
if (decision.kind === 'park') {
|
|
3025
|
-
emitEvent(paths.events, 'escalate', issue, decision.reason, { lane });
|
|
3026
|
-
emitEvent(paths.events, 'parked', issue, `issue #${issue} parked (${decision.reason}); lane '${lane}' continuing, ${await remainingAfter(i)} issue(s) remaining`, { lane });
|
|
3027
|
-
// A preflight-parked issue may still have a stale worktree from a prior run; the reap
|
|
3028
|
-
// is fail-closed and must never change lane flow.
|
|
3029
|
-
await reapWorktree(issue).catch(() => { });
|
|
3030
|
-
await settle(issue, 'parked');
|
|
3031
|
-
parked++;
|
|
3032
|
-
streak = null;
|
|
3033
|
-
continue;
|
|
3034
|
-
}
|
|
3035
|
-
const layer = decision.kind === 'adopt' ? decision.stackLayer : undefined;
|
|
3036
|
-
try {
|
|
3037
|
-
const sliceOutcome = {};
|
|
3038
|
-
const branch = decision.kind === 'adopt'
|
|
3039
|
-
? decision.branch
|
|
3040
|
-
: await withHeartbeat(issue, heartbeat, () => ship(issue, {}, {
|
|
3041
|
-
repoRoot,
|
|
3042
|
-
ghRepo,
|
|
3043
|
-
paths,
|
|
3044
|
-
lane,
|
|
3045
|
-
onOutcome: (o) => {
|
|
3046
|
-
sliceOutcome.value = o;
|
|
3047
|
-
},
|
|
3048
|
-
}));
|
|
3049
|
-
const shipped = sliceOutcome.value;
|
|
3050
|
-
if (shipped?.state === 'ready' && shipped.slice && shipped.slice.index < shipped.slice.count) {
|
|
3051
|
-
// ADR-0147: a non-final slice's PR is open; hand the issue back for the next slice
|
|
3052
|
-
// instead of waiting for the merge.
|
|
3053
|
-
awaitingReview++;
|
|
3054
|
-
streak = null;
|
|
3055
|
-
await settle(issue, 'queued');
|
|
3056
|
-
continue;
|
|
3057
|
-
}
|
|
3058
|
-
if (layer) {
|
|
3059
|
-
// ADR-0161: land exactly this layer's branch, record it, and re-queue for the next layer.
|
|
3060
|
-
await withHeartbeat(issue, heartbeat, () => waitMerge(issue, branch, repoRoot, ghRepo, paths, { landBranch: branch }));
|
|
3061
|
-
await recordStackMerge(issue, layer.prNumber).catch((err) => emitEvent(paths.events, 'warn', issue, `slice ${layer.index}/${layer.count} merged but recording it failed: ${errorDetail(err)}`, { lane }));
|
|
3062
|
-
merged++;
|
|
3063
|
-
streak = null;
|
|
3064
|
-
await settle(issue, layer.index < layer.count ? 'queued' : 'done');
|
|
3065
|
-
continue;
|
|
3066
|
-
}
|
|
3067
|
-
await withHeartbeat(issue, heartbeat, () => waitMerge(issue, branch, repoRoot, ghRepo, paths));
|
|
3068
|
-
merged++;
|
|
3069
|
-
streak = null;
|
|
3070
|
-
await settle(issue, 'done');
|
|
3071
|
-
}
|
|
3072
|
-
catch (err) {
|
|
3073
|
-
if (err instanceof AwaitingReviewError) {
|
|
3074
|
-
// The land path already emitted the awaiting-review event and cleaned the
|
|
3075
|
-
// worktree — this is a clean outcome, not a park; move to the next issue.
|
|
3076
|
-
awaitingReview++;
|
|
3077
|
-
streak = null;
|
|
3078
|
-
await settle(issue, layer && layer.index < layer.count ? 'queued' : 'done');
|
|
3079
|
-
continue;
|
|
3080
|
-
}
|
|
3081
|
-
if (err instanceof IssueSkippedError) {
|
|
3082
|
-
// shipIssue already emitted skipped-already-closed; nothing was attempted.
|
|
3083
|
-
skipped++;
|
|
3084
|
-
await settle(issue, 'done');
|
|
3085
|
-
continue;
|
|
3086
|
-
}
|
|
3087
|
-
if (err instanceof IssueDecomposedError) {
|
|
3088
|
-
// shipIssue already rewrote the queue — splice the children in after the
|
|
3089
|
-
// current issue so this same lane run ships them, deduping ones already queued.
|
|
3090
|
-
const fresh = err.childIssues.filter((n) => !seen.has(n));
|
|
3091
|
-
for (const n of fresh)
|
|
3092
|
-
seen.add(n);
|
|
3093
|
-
pending.splice(i + 1, 0, ...fresh.map(buildClaim));
|
|
3094
|
-
decomposed++;
|
|
3095
|
-
emitEvent(paths.events, 'decompose_filed', issue, `lane '${lane}' continuing with ${fresh.length} child issue(s) in place of #${issue}`, { lane });
|
|
3096
|
-
await settle(issue, 'done');
|
|
3097
|
-
continue;
|
|
3098
|
-
}
|
|
3099
|
-
if (err instanceof LaneEnvironmentPauseError) {
|
|
3100
|
-
// The base is broken, not the issue (#1928): release to queued (never parked),
|
|
3101
|
-
// pause the lane whatever laneBreakerThreshold is, and stop claiming.
|
|
3102
|
-
streak = null;
|
|
3103
|
-
// A preserved build is kept for `factory resume --from check` (#2564).
|
|
3104
|
-
if (!err.buildPreserved)
|
|
3105
|
-
await reapWorktree(issue).catch(() => { });
|
|
3106
|
-
await settle(issue, 'queued');
|
|
3107
|
-
// Persist the hold so a restarted supervisor remembers the broken base (#2348);
|
|
3108
|
-
// best-effort, never allowed to change lane flow.
|
|
3109
|
-
try {
|
|
3110
|
-
writeBaseHold(paths.baseHold, {
|
|
3111
|
-
repo: ghRepo,
|
|
3112
|
-
baseSha: err.baseSha,
|
|
3113
|
-
failingChecks: err.failingChecks,
|
|
3114
|
-
since: new Date().toISOString(),
|
|
3115
|
-
});
|
|
3116
|
-
}
|
|
3117
|
-
catch { }
|
|
3118
|
-
const payload = {
|
|
3119
|
-
lane,
|
|
3120
|
-
signature: err.failureSignature ?? '',
|
|
3121
|
-
failingChecks: err.failingChecks,
|
|
3122
|
-
firstIssue: issue,
|
|
3123
|
-
secondIssue: issue,
|
|
3124
|
-
cause: 'environment',
|
|
3125
|
-
baseSha: err.baseSha,
|
|
3126
|
-
};
|
|
3127
|
-
emitEvent(paths.events, 'lane-paused', lane, `lane '${lane}' paused: #${issue} failed only on checkers that also fail on base ${err.baseSha.slice(0, 8)} (${err.failingChecks.join(', ')}); #${issue} released to queued`, { lane, lanePaused: payload });
|
|
3128
|
-
return;
|
|
3129
|
-
}
|
|
3130
|
-
const reason = parkReasonFor(err);
|
|
3131
|
-
// Terminal reason events (escalate/timeout/fail/conflict) are emitted exactly
|
|
3132
|
-
// once by the layer that detects the failure — shipIssue for pipeline failures,
|
|
3133
|
-
// the land path for merge failures. runLane owns only lane-lifecycle events
|
|
3134
|
-
// (stopped/parked/lane-done), so injected ship functions never change ownership.
|
|
3135
|
-
const signature = err instanceof LaneParkError ? err.failureSignature : undefined;
|
|
3136
|
-
const checks = err instanceof LaneParkError ? (err.failingChecks ?? []) : [];
|
|
3137
|
-
if (!signature)
|
|
3138
|
-
streak = null;
|
|
3139
|
-
else if (streak && streak.signature === signature)
|
|
3140
|
-
streak.count++;
|
|
3141
|
-
else
|
|
3142
|
-
streak = { signature, firstIssue: issue, failingChecks: checks, count: 1 };
|
|
3143
|
-
const tripped = laneBreakerThreshold > 0 && streak !== null && streak.count >= laneBreakerThreshold;
|
|
3144
|
-
emitEvent(paths.events, 'parked', issue, `issue #${issue} parked (${reason}); ${tripped ? `lane '${lane}' paused (lane breaker)` : `lane '${lane}' continuing`}, ${await remainingAfter(i)} issue(s) remaining`, { lane });
|
|
3145
|
-
// Remove the parked lane's own worktree so a parked issue stops leaving a sibling
|
|
3146
|
-
// checkout behind (#1007); fail-closed and never allowed to change lane flow.
|
|
3147
|
-
await reapWorktree(issue).catch(() => { });
|
|
3148
|
-
await settle(issue, 'parked');
|
|
3149
|
-
parked++;
|
|
3150
|
-
if (tripped && streak) {
|
|
3151
|
-
const { signature: tripSignature, firstIssue, failingChecks: firstChecks } = streak;
|
|
3152
|
-
const payload = {
|
|
3153
|
-
lane,
|
|
3154
|
-
signature: tripSignature,
|
|
3155
|
-
failingChecks: checks.filter((c) => firstChecks.includes(c)),
|
|
3156
|
-
firstIssue,
|
|
3157
|
-
secondIssue: issue,
|
|
3158
|
-
};
|
|
3159
|
-
emitEvent(paths.events, 'lane-paused', lane, `lane '${lane}' paused: #${payload.firstIssue} and #${payload.secondIssue} parked with the same failure signature (${payload.failingChecks.join(', ') || 'no failing checks'})`, { lane, lanePaused: payload });
|
|
3160
|
-
// Tell the developer on GitHub why the lane stopped (#1919); fail-closed, never changes lane flow.
|
|
3161
|
-
await commentIssue(payload.secondIssue, renderLanePausedComment(payload)).catch(() => { });
|
|
3162
|
-
return;
|
|
3163
|
-
}
|
|
3164
|
-
continue;
|
|
3165
|
-
}
|
|
3166
|
-
}
|
|
3167
|
-
emitEvent(paths.events, 'lane-done', lane, `lane complete (${merged} merged, ${awaitingReview} awaiting review, ${skipped} skipped, ${decomposed} decomposed, ${parked} parked)`, { lane });
|
|
3168
|
-
}
|
|
3169
2753
|
/** The target issue was already resolved before the run started — a clean skip,
|
|
3170
2754
|
* not a park. Callers advance to the next queue entry. (#681) */
|
|
3171
2755
|
export class IssueSkippedError extends Error {
|
|
@@ -3388,220 +2972,6 @@ function probeClaudeKeychain() {
|
|
|
3388
2972
|
return { status: 'unreadable', inTmux: !!process.env.TMUX, ...profile };
|
|
3389
2973
|
}
|
|
3390
2974
|
}
|
|
3391
|
-
async function cmdDoctor(opts = {}) {
|
|
3392
|
-
let repoRoot;
|
|
3393
|
-
try {
|
|
3394
|
-
repoRoot = shellOps
|
|
3395
|
-
.execSync('git rev-parse --show-toplevel', {
|
|
3396
|
-
timeout: 10_000,
|
|
3397
|
-
stdio: ['ignore', 'pipe', 'ignore'],
|
|
3398
|
-
})
|
|
3399
|
-
.trim();
|
|
3400
|
-
}
|
|
3401
|
-
catch {
|
|
3402
|
-
repoRoot = null;
|
|
3403
|
-
}
|
|
3404
|
-
const checks = runDoctorChecks({
|
|
3405
|
-
commandAvailable: coreOps.isCommandAvailable,
|
|
3406
|
-
envPresent: (key) => !!process.env[key],
|
|
3407
|
-
tryExec: (cmd, opts) => {
|
|
3408
|
-
try {
|
|
3409
|
-
return shellOps
|
|
3410
|
-
.execSync(cmd, {
|
|
3411
|
-
timeout: opts?.timeoutMs ?? 10_000,
|
|
3412
|
-
stdio: ['ignore', 'pipe', 'ignore'],
|
|
3413
|
-
...(opts?.env ? { env: { ...process.env, ...opts.env } } : {}),
|
|
3414
|
-
})
|
|
3415
|
-
.trim();
|
|
3416
|
-
}
|
|
3417
|
-
catch {
|
|
3418
|
-
return null;
|
|
3419
|
-
}
|
|
3420
|
-
},
|
|
3421
|
-
pathExists: existsSync,
|
|
3422
|
-
distFreshness: distFreshnessProbe(import.meta.url),
|
|
3423
|
-
codexSessionState: () => codexChatgptSessionState(),
|
|
3424
|
-
claudeLoggedIn: () => hasClaudeLoginEvidence({
|
|
3425
|
-
exec: (file, args) => {
|
|
3426
|
-
shellOps.execSync([file, ...args].map(shellEscape).join(' '), { timeout: 10_000, stdio: 'ignore' });
|
|
3427
|
-
},
|
|
3428
|
-
fileExists: existsSync,
|
|
3429
|
-
}),
|
|
3430
|
-
claudeSessionState: () => claudeStoredSessionState(),
|
|
3431
|
-
localAuthFailure: (agent) => localAuthFailureSinceLogin(agent),
|
|
3432
|
-
opencodeLoggedIn: () => hasOpencodeLoginEvidence(),
|
|
3433
|
-
commandHarnesses: () => repoRoot === null
|
|
3434
|
-
? []
|
|
3435
|
-
: repoCommandHarnesses(configLoaders.loadModelsConfig().harnesses, configLoaders.loadRepoConfig(repoRoot)?.models?.registry),
|
|
3436
|
-
});
|
|
3437
|
-
// Only with --probe-inference: these send a real prompt (uses quota). Plain doctor makes no inference call.
|
|
3438
|
-
if (opts.probeInference) {
|
|
3439
|
-
const CLAUDE_AUTH_PROBE = 'claude -p "reply with exactly: ok"';
|
|
3440
|
-
const probeExec = (cmd) => {
|
|
3441
|
-
try {
|
|
3442
|
-
return shellOps.execSync(cmd, { timeout: 120_000, stdio: ['ignore', 'pipe', 'ignore'] }).trim();
|
|
3443
|
-
}
|
|
3444
|
-
catch {
|
|
3445
|
-
return null;
|
|
3446
|
-
}
|
|
3447
|
-
};
|
|
3448
|
-
let host = 'skipped';
|
|
3449
|
-
let hostDetail = 'skipped: claude CLI not on PATH';
|
|
3450
|
-
if (coreOps.isCommandAvailable('claude')) {
|
|
3451
|
-
host = probeExec(CLAUDE_AUTH_PROBE) === null ? 'failed' : 'ok';
|
|
3452
|
-
hostDetail = host === 'ok' ? 'claude -p succeeded on the host' : 'claude -p failed on the host';
|
|
3453
|
-
}
|
|
3454
|
-
let sandboxed = 'skipped';
|
|
3455
|
-
let sandboxDetail = 'skipped: host claude auth was not verified';
|
|
3456
|
-
const runtimeAvailable = hasSandboxRuntime(process.platform, (cmd) => isCommandAvailableViaShell(shellOps, cmd));
|
|
3457
|
-
const skip = sandboxAuthSkipDetail({ runtimeAvailable, hostOk: host === 'ok', inRepo: repoRoot !== null });
|
|
3458
|
-
if (skip !== null) {
|
|
3459
|
-
sandboxDetail = skip;
|
|
3460
|
-
}
|
|
3461
|
-
else if (repoRoot !== null) {
|
|
3462
|
-
const policy = resolveSandboxPolicy(configLoaders.loadFactoryConfigForRepo(getFactoryPaths(repoRoot).config).sandbox, {
|
|
3463
|
-
worktree: repoRoot,
|
|
3464
|
-
repoRoot,
|
|
3465
|
-
isAvailable: (cmd) => isCommandAvailableViaShell(shellOps, cmd),
|
|
3466
|
-
});
|
|
3467
|
-
if (!policy) {
|
|
3468
|
-
sandboxDetail = 'skipped: sandbox disabled by config or FACTORY_SANDBOX';
|
|
3469
|
-
}
|
|
3470
|
-
else if (policy.runtime === 'none') {
|
|
3471
|
-
sandboxDetail = 'skipped: no sandbox runtime (sandbox-exec/firejail) on this host';
|
|
3472
|
-
}
|
|
3473
|
-
else if (policy.runtime === 'docker-sandbox') {
|
|
3474
|
-
sandboxDetail = 'skipped: docker-sandbox containment is the microVM, not command wrapping (#653)';
|
|
3475
|
-
}
|
|
3476
|
-
else {
|
|
3477
|
-
sandboxed = probeExec(wrapCommandInSandbox(CLAUDE_AUTH_PROBE, policy)) === null ? 'failed' : 'ok';
|
|
3478
|
-
sandboxDetail =
|
|
3479
|
-
sandboxed === 'ok'
|
|
3480
|
-
? `claude -p succeeded under ${policy.runtime}`
|
|
3481
|
-
: `claude -p failed under ${policy.runtime} (local_auth)`;
|
|
3482
|
-
}
|
|
3483
|
-
}
|
|
3484
|
-
checks.push(...sandboxClaudeAuthChecks({ host, sandboxed, hostDetail, sandboxDetail }));
|
|
3485
|
-
}
|
|
3486
|
-
checks.push(claudeKeychainCheck(probeClaudeKeychain()));
|
|
3487
|
-
checks.splice(0, checks.length, ...reconcileAgentAuthChecks(checks));
|
|
3488
|
-
if (repoRoot !== null) {
|
|
3489
|
-
const paths = getFactoryPaths(repoRoot);
|
|
3490
|
-
let factoryConfig;
|
|
3491
|
-
try {
|
|
3492
|
-
factoryConfig = configLoaders.loadFactoryConfigForRepo(paths.config);
|
|
3493
|
-
}
|
|
3494
|
-
catch {
|
|
3495
|
-
factoryConfig = loadFactoryConfig(); // schema errors are reported by the config row (#2611)
|
|
3496
|
-
}
|
|
3497
|
-
const health = await inspectPortLeases({ registryFile: paths.ports });
|
|
3498
|
-
checks.push(...leaseChecks(health.map(toLeaseRow)));
|
|
3499
|
-
const dockerAvailable = coreOps.isCommandAvailable('docker');
|
|
3500
|
-
const orphanContainers = dockerAvailable ? await internalOps.listOrphanContainers().catch(() => []) : [];
|
|
3501
|
-
checks.push(...orphanContainerChecks(orphanContainers.map((c) => c.name)));
|
|
3502
|
-
const sbxAvailable = coreOps.isCommandAvailable('sbx');
|
|
3503
|
-
const activeVmNames = new Set(health.filter((h) => h.alive).map((h) => microVmName(h.lease.worktreeId)));
|
|
3504
|
-
const orphanVmNames = sbxAvailable ? (await internalOps.listMicroVms()).filter((n) => !activeVmNames.has(n)) : [];
|
|
3505
|
-
checks.push(...orphanMicroVmChecks(orphanVmNames));
|
|
3506
|
-
const sweepStatus = checkSweepHeartbeat(factoryConfig.sweep, process.env, defaultSweepHeartbeatDeps());
|
|
3507
|
-
const sweepCheck = sweepHeartbeatCheck(sweepStatus);
|
|
3508
|
-
if (sweepCheck)
|
|
3509
|
-
checks.push(sweepCheck);
|
|
3510
|
-
checks.push(stopSentinelCheck(readStopFileStatus(paths)));
|
|
3511
|
-
const eventsContent = existsSync(paths.events) ? readFileSync(paths.events, 'utf-8') : null;
|
|
3512
|
-
checks.push(eventLogCheck(eventsContent === null ? null : analyzeEventLog(eventsContent)));
|
|
3513
|
-
let constitution = null;
|
|
3514
|
-
let loadError;
|
|
3515
|
-
try {
|
|
3516
|
-
constitution = new ConstitutionLoader().loadRepoConstitution(repoRoot, factoryConfig.paths?.constitution);
|
|
3517
|
-
}
|
|
3518
|
-
catch (err) {
|
|
3519
|
-
loadError = `cannot parse constitution: ${err instanceof Error ? err.message : String(err)}`;
|
|
3520
|
-
}
|
|
3521
|
-
checks.push(...configLintChecks({ constitution, configFile: repoConfigFileFor(repoRoot, paths.config) }, loadError));
|
|
3522
|
-
try {
|
|
3523
|
-
const declared = Object.keys(configLoaders.loadModelsConfig().harnesses ?? {});
|
|
3524
|
-
checks.push(...modelsRegistryChecks(paths.config, configLoaders.loadRepoConfig(repoRoot)?.models?.registry, [
|
|
3525
|
-
...KNOWN_HARNESS_IDS,
|
|
3526
|
-
...declared,
|
|
3527
|
-
]));
|
|
3528
|
-
}
|
|
3529
|
-
catch {
|
|
3530
|
-
// malformed config is reported by the load paths / `factory models --doctor`; doctor keeps going
|
|
3531
|
-
}
|
|
3532
|
-
let harnessProbes = [];
|
|
3533
|
-
try {
|
|
3534
|
-
const registry = new coreOps.ModelRegistry(applyRepoConfig(configLoaders.loadModelsConfig(), configLoaders.loadRepoConfig(repoRoot)));
|
|
3535
|
-
harnessProbes = coreOps.probeDeclaredHarnesses(registry);
|
|
3536
|
-
}
|
|
3537
|
-
catch {
|
|
3538
|
-
// models config problems are reported by `factory models --doctor`; doctor keeps going
|
|
3539
|
-
}
|
|
3540
|
-
checks.push(...declaredHarnessChecks(harnessProbes));
|
|
3541
|
-
// Full run-lifecycle reap (#998): dead-run worktrees and their branches. Uses the same
|
|
3542
|
-
// GitHub evidence and the same lock pairing as `factory worktree gc` (cmdWorktreeGc) —
|
|
3543
|
-
// a merged or closed PR is what makes a worktree reapable.
|
|
3544
|
-
const ghRepo = await getGitHubRepo().catch(() => undefined);
|
|
3545
|
-
const octokit = ghRepo && hasGitHubToken() ? getOctokit() : undefined;
|
|
3546
|
-
// Green-and-ready PRs with no merge (#1000). Report only — doctor never lands a PR.
|
|
3547
|
-
checks.push(...unmergedGreenPrChecks(await scanGreenPrs(ghRepo, octokit)));
|
|
3548
|
-
if (opts.reconcile) {
|
|
3549
|
-
if (orphanContainers.length > 0) {
|
|
3550
|
-
const reapedContainers = await internalOps.reapOrphanContainers(orphanContainers);
|
|
3551
|
-
console.log(formatContainerReconcileReport(reapedContainers));
|
|
3552
|
-
}
|
|
3553
|
-
if (orphanVmNames.length > 0) {
|
|
3554
|
-
const reapedVms = await Promise.all(orphanVmNames.map((name) => internalOps.reapOrphanMicroVm(name)));
|
|
3555
|
-
console.log(formatMicroVmReconcileReport(reapedVms));
|
|
3556
|
-
}
|
|
3557
|
-
const reaped = await reapStalePortLeases({ registryFile: paths.ports, lockDir: paths.portsLock });
|
|
3558
|
-
console.log(formatReconcileReport(reaped));
|
|
3559
|
-
if (reaped.length > 0) {
|
|
3560
|
-
const graceMs = resolveProcessGroupGraceMs(factoryConfig);
|
|
3561
|
-
const orphanEvents = await coreOps.reapOrphanProcesses({ reaped, graceMs });
|
|
3562
|
-
for (const e of orphanEvents) {
|
|
3563
|
-
console.log(`reconcile: ${e.action === 'killed' ? 'killed' : 'found'} pid ${e.pid} (pgid ${e.pgid}, ${e.command}) squatting port ${e.port} of dead lane ${e.worktreeId}${e.action === 'reported' ? '; not factory-started, left running' : ''}`);
|
|
3564
|
-
}
|
|
3565
|
-
}
|
|
3566
|
-
try {
|
|
3567
|
-
const gcLog = (type, msg) => logEvent(paths.events, type, '-', msg);
|
|
3568
|
-
const gcSandbox = gcWorktreeSandbox(factoryConfig.sandbox, repoRoot);
|
|
3569
|
-
const report = await withGitLock(repoRoot, () => withFileLock(paths.gitLock, () => internalOps.sweepWorktrees({
|
|
3570
|
-
repoRoot,
|
|
3571
|
-
ttlDays: factoryConfig.worktree.gcTtlDays,
|
|
3572
|
-
repo: ghRepo,
|
|
3573
|
-
branchPrefix: resolveBranchPrefix(),
|
|
3574
|
-
}, { log: gcLog, octokit, sandbox: gcSandbox })));
|
|
3575
|
-
console.log(formatWorktreeReconcileReport(report.removed));
|
|
3576
|
-
}
|
|
3577
|
-
catch (err) {
|
|
3578
|
-
console.log(`reconcile: worktree sweep failed: ${err?.message ?? String(err)}`);
|
|
3579
|
-
}
|
|
3580
|
-
// Stale claim reap (#999): factory:in-progress claims naming a dead local pid go back to
|
|
3581
|
-
// factory:queued so the next run can claim them with no manual label surgery.
|
|
3582
|
-
try {
|
|
3583
|
-
if (ghRepo && octokit) {
|
|
3584
|
-
const [owner, repoName] = ghRepo.split('/');
|
|
3585
|
-
const released = await internalOps.releaseStaleClaims({
|
|
3586
|
-
client: createOctokitQueueClient(octokit),
|
|
3587
|
-
owner,
|
|
3588
|
-
repo: repoName,
|
|
3589
|
-
});
|
|
3590
|
-
console.log(formatClaimReconcileReport(released));
|
|
3591
|
-
}
|
|
3592
|
-
else {
|
|
3593
|
-
console.log('reconcile: skipping stale claims: no GitHub repo or token');
|
|
3594
|
-
}
|
|
3595
|
-
}
|
|
3596
|
-
catch (err) {
|
|
3597
|
-
console.log(`reconcile: stale claim release failed: ${err?.message ?? String(err)}`);
|
|
3598
|
-
}
|
|
3599
|
-
}
|
|
3600
|
-
}
|
|
3601
|
-
console.log(formatDoctorChecks(checks, { inRepo: repoRoot !== null }));
|
|
3602
|
-
if (doctorFailed(checks))
|
|
3603
|
-
process.exit(1);
|
|
3604
|
-
}
|
|
3605
2975
|
/** Appends one event to the events log without echoing it to stdout (stop/resume --json must print only JSON). */
|
|
3606
2976
|
function logEventQuietly(eventsFile, type, msg) {
|
|
3607
2977
|
createLogger(eventsFile, { issue: '-' }, { out: { write: () => true } }).warn(type, msg);
|
|
@@ -3726,7 +3096,7 @@ function runSupervise(opts) {
|
|
|
3726
3096
|
superviseLoop,
|
|
3727
3097
|
createSuperviseRunQueue,
|
|
3728
3098
|
createIngestHook,
|
|
3729
|
-
cmdRun: (o) =>
|
|
3099
|
+
cmdRun: (o) => runRun(o),
|
|
3730
3100
|
CliExitError,
|
|
3731
3101
|
});
|
|
3732
3102
|
}
|
|
@@ -3746,12 +3116,59 @@ function runStatus(opts = {}) {
|
|
|
3746
3116
|
CliExitError,
|
|
3747
3117
|
});
|
|
3748
3118
|
}
|
|
3119
|
+
/** `factory run` (#2800): the handler lives in ./run.ts; this binds its runtime. */
|
|
3120
|
+
function runRun(opts = {}) {
|
|
3121
|
+
return cmdRun(opts, {
|
|
3122
|
+
configLoaders,
|
|
3123
|
+
coreOps,
|
|
3124
|
+
shellOps,
|
|
3125
|
+
readCosts: internalOps.readCosts,
|
|
3126
|
+
getRepoRoot: () => getRepoRoot(),
|
|
3127
|
+
getGitHubRepo: () => getGitHubRepo(),
|
|
3128
|
+
getOctokit: () => getOctokit(),
|
|
3129
|
+
branchPrefixOrExit,
|
|
3130
|
+
withRepoRunLock,
|
|
3131
|
+
runAutoWorktreeGc,
|
|
3132
|
+
planRunLanes,
|
|
3133
|
+
preflightQueuedIssue,
|
|
3134
|
+
createQueuePreflightOps,
|
|
3135
|
+
warnQueueDiagnostics,
|
|
3136
|
+
resolveUsageKnobs,
|
|
3137
|
+
usageFlagOverrides,
|
|
3138
|
+
usageWatchSourceLabel,
|
|
3139
|
+
probeClaudeKeychain,
|
|
3140
|
+
startLaneProxy,
|
|
3141
|
+
sweepBetweenLaneIssues,
|
|
3142
|
+
reapParkedLaneWorktree,
|
|
3143
|
+
reconstructHumanEventsForKpis,
|
|
3144
|
+
appendKpiSnapshot,
|
|
3145
|
+
CliExitError,
|
|
3146
|
+
});
|
|
3147
|
+
}
|
|
3148
|
+
/** `factory doctor` (#2799): the handler lives in ./doctor.ts; this binds its runtime. */
|
|
3149
|
+
function runDoctor(opts = {}) {
|
|
3150
|
+
return cmdDoctor(opts, {
|
|
3151
|
+
shellOps,
|
|
3152
|
+
coreOps,
|
|
3153
|
+
internalOps,
|
|
3154
|
+
configLoaders,
|
|
3155
|
+
getGitHubRepo: () => getGitHubRepo(),
|
|
3156
|
+
getOctokit: () => getOctokit(),
|
|
3157
|
+
hasGitHubToken: () => hasGitHubToken(),
|
|
3158
|
+
scanGreenPrs,
|
|
3159
|
+
probeClaudeKeychain,
|
|
3160
|
+
toLeaseRow,
|
|
3161
|
+
gcWorktreeSandbox,
|
|
3162
|
+
repoConfigFileFor,
|
|
3163
|
+
isCommandAvailableViaShell,
|
|
3164
|
+
});
|
|
3165
|
+
}
|
|
3749
3166
|
export const PROGRAM_HANDLERS = {
|
|
3750
3167
|
cmdInit,
|
|
3751
3168
|
cmdMigrate,
|
|
3752
3169
|
cmdConstitution,
|
|
3753
3170
|
cmdModels,
|
|
3754
|
-
cmdDoctor,
|
|
3171
|
+
cmdDoctor: runDoctor,
|
|
3755
3172
|
cmdCost,
|
|
3756
3173
|
cmdUsage,
|
|
3757
3174
|
cmdStatus: runStatus,
|
|
@@ -3783,7 +3200,7 @@ export const PROGRAM_HANDLERS = {
|
|
|
3783
3200
|
cmdLocalSmallOvernight,
|
|
3784
3201
|
cmdLand,
|
|
3785
3202
|
cmdResumeApproved,
|
|
3786
|
-
cmdRun,
|
|
3203
|
+
cmdRun: runRun,
|
|
3787
3204
|
cmdProxy,
|
|
3788
3205
|
cmdFactoryd,
|
|
3789
3206
|
cmdDaemonStart,
|