@on-par/factory-cli 0.9.7-next.6 → 0.9.7-next.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/index.d.ts +8 -60
- package/dist/cli/index.d.ts.map +1 -1
- package/dist/cli/index.js +38 -424
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/run-lane.d.ts +52 -0
- package/dist/cli/run-lane.d.ts.map +1 -0
- package/dist/cli/run-lane.js +239 -0
- package/dist/cli/run-lane.js.map +1 -0
- package/dist/cli/run.d.ts +52 -0
- package/dist/cli/run.d.ts.map +1 -0
- package/dist/cli/run.js +200 -0
- package/dist/cli/run.js.map +1 -0
- package/package.json +3 -3
package/dist/cli/index.js
CHANGED
|
@@ -1,21 +1,21 @@
|
|
|
1
1
|
import { fileURLToPath } from 'node:url';
|
|
2
2
|
// packages/cli/src/cli/index.ts — CLI entry point: factory <command> [options]
|
|
3
3
|
import { exec as execCb, execSync } from 'node:child_process';
|
|
4
|
-
import { randomUUID } from 'node:crypto';
|
|
5
4
|
import { appendFileSync, existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, unlinkSync, writeFileSync, } from 'node:fs';
|
|
6
5
|
import { createRequire } from 'node:module';
|
|
7
6
|
import { arch, hostname as osHostname, release, platform, userInfo } from 'node:os';
|
|
8
7
|
import { basename, dirname, isAbsolute, join, relative, resolve } from 'node:path';
|
|
9
8
|
import { promisify, styleText } from 'node:util';
|
|
10
9
|
import { INVALID_BRANCH_PREFIX_MESSAGE, resolveBranchPrefixOption } from './branch-prefix.js';
|
|
11
|
-
import { acquirePortLease, aggregateCosts, appendKpiHistoryLine, applyRepoConfig, buildPhase, checkPhase, clearProxyState, computeHealthKpis, ConstitutionLoader, createDefaultWorkSourceRegistry, createLogger, createLaneProxy, defaultFindPortListeners, describeEffectiveConfig, detectPostMergeDefects, diagnoseModels, probeDeclaredHarnesses, estimateTrailingSpend, fetchDefectSources, fetchHumanEventSources, fetchSubscriptionUsage, formatUsageReport, getConstitutionsDir, getFactoryPaths, GITHUB_ISSUE_SOURCE, hasUnresolvedPark, InvalidArtifactsDirError, InvalidWorkRequestInputError, InvalidWorkspaceError, isCommandAvailable, isProxyRunning, kpiBranchPrefixes, kpisToHistoryRecord, laneBaseUrl, laneEnv, laneHostLabel, loadFactoryConfigForRepo, loadModelsConfig, loadRepoConfig, loadRoutesConfig, LOCAL_BRIEF_SOURCE, joinClassifierOutcomes, mergeClassifierOutcomes, parseClassifierOutcomes, summarizeClassifierOutcomes, formatClassifierReport, mergedPrRefs, ModelRegistry, ModelRouter, parkReasonFor, parseKpiHistory, parseQueue, planPhase,
|
|
12
|
-
import { acquirePidFile, createCheckResumeStore, findSlicePlanComment,
|
|
10
|
+
import { acquirePortLease, aggregateCosts, appendKpiHistoryLine, applyRepoConfig, buildPhase, checkPhase, clearProxyState, computeHealthKpis, ConstitutionLoader, createDefaultWorkSourceRegistry, createLogger, createLaneProxy, defaultFindPortListeners, describeEffectiveConfig, detectPostMergeDefects, diagnoseModels, probeDeclaredHarnesses, estimateTrailingSpend, fetchDefectSources, fetchHumanEventSources, fetchSubscriptionUsage, formatUsageReport, getConstitutionsDir, getFactoryPaths, GITHUB_ISSUE_SOURCE, hasUnresolvedPark, InvalidArtifactsDirError, InvalidWorkRequestInputError, InvalidWorkspaceError, isCommandAvailable, isProxyRunning, kpiBranchPrefixes, kpisToHistoryRecord, laneBaseUrl, laneEnv, laneHostLabel, loadFactoryConfigForRepo, loadModelsConfig, loadRepoConfig, loadRoutesConfig, LOCAL_BRIEF_SOURCE, joinClassifierOutcomes, mergeClassifierOutcomes, parseClassifierOutcomes, summarizeClassifierOutcomes, formatClassifierReport, mergedPrRefs, ModelRegistry, ModelRouter, parkReasonFor, parseKpiHistory, parseQueue, planPhase, readEvents, readPortLeases, readQueue, readUsage, reapOrphanProcesses, reconstructKpiEvents, recordLeasePgid, releasePortLease, renderKpiReport, renderKpiTrend, resolveDefectWindowDays, parseSizeGateMode, parseMaxSlices, SIZE_GATE_MODES, resolveEffectiveModelPins, resolveEnvironmentProxy, resolveArtifactsDir, resolveLocalOnlyPolicy, resolveMergePolicy, resolveSandboxPolicy, resolveSkipCI, resolveTimeouts, resolveUsageCap, resolveWatchdogPolicy, runAutoIngest, shipPhase, formatCostTotal, validateQueue, watchUsage, writeLocalRunReport, writeProxyState, } from '@on-par/factory-core';
|
|
11
|
+
import { acquirePidFile, createCheckResumeStore, findSlicePlanComment, upsertSlicePlanComment, withSliceState, DEFAULT_GARDEN_MAX_CLUSTERS, DEFAULT_GARDEN_SINCE, capGardenClusters, clusterGarden, dedupGardenClusters, filterGardenOnlyNew, filterGardenWindow, markGardenTrackingUnknown, parseGardenDuration, readHarvestEvents, renderGardenJson, renderGardenReport, laneWorktreePath, migrateRepoConfigToYaml, REPO_CONFIG_YAML_HEADER, claudeConfigDirOverride, claudeKeychainService, cleanupWorktree, createDaemonLogSink, factoryBranchIssue, factoryBranchPrefixes, createFactorydServer, createRunRuntime, createShipExecutor, createGithubQueue, createLocalSmallDryRun, createOctokitFilingClient, createOctokitGreenPrClient, createOctokitQueueClient, daemonRuntimePaths, defaultRegistryPath, ensureDir, FACTORY_RUNTIME_CONFIG_KEYS, findUnmergedGreenPrs, formatGcReport, gitFetch, listMicroVms, listOrphanContainers, logCost, logEvent, planQueueMigration, readCosts, readGithubQueueSnapshot, reapLaneWorktree, reapOrphanContainers, reapOrphanMicroVm, releaseRuntimeFiles, releaseStaleClaims, resolveBranchPrefix, resolveEffectiveConfig, worktreeSandboxFor, resolveExperimental, resolveLocalOnly, resolvePrClassifierPolicy, resolveReviewFloorRules, RunLockHeldError, runOvernightQueue, parseRemoteSlug, setupWorktree, shellEscape, listBuiltInCheckers, sweepWorktrees, watchChecks, withFileLock, withGitLock, withRunLock, writePortFile, buildTriagePrompt, normalizeTriageQueue, parseTriageCandidates, recordLocalAuthFailure, } from '@on-par/factory-core/internal';
|
|
13
12
|
import { runTui } from '@on-par/factory-tui';
|
|
14
13
|
import { admissionStateReaderFor, runQueueReconcile } from './admission.js';
|
|
15
14
|
import { cmdClassifierBacktest, ClassifierBacktestInputError, } from './classifier-backtest.js';
|
|
16
|
-
import { findOpenPRForIssue, landIssue
|
|
15
|
+
import { findOpenPRForIssue, landIssue } from './land.js';
|
|
17
16
|
import { AwaitingReviewError, CiFailedError, CiUnverifiedError, getPullRequestLandState, LandConflictError, LandFailureError, MergeGatedError, } from './land-pr.js';
|
|
18
17
|
import { setLandRuntime } from './land-runtime.js';
|
|
18
|
+
import { cmdRun } from './run.js';
|
|
19
19
|
import { shipIssue } from './ship.js';
|
|
20
20
|
import { cmdStatus } from './status.js';
|
|
21
21
|
import { cmdSupervise } from './supervise.js';
|
|
@@ -34,7 +34,7 @@ import { buildUsageJson } from './usage-json.js';
|
|
|
34
34
|
import { parseResetIssues, runReset } from './reset.js';
|
|
35
35
|
import { cmdDaemonLogs, cmdDaemonStart, cmdDaemonStatus, cmdDaemonStop, createAutoUpdateDeps, DaemonCtlError, detectInstall, loadDaemonFileConfig, recoverStaleDrain, resolveFactorydConfig, runAutoUpdateLoop, } from './daemon.js';
|
|
36
36
|
import { checkForUpdate, cmdSelfUpdateCheck } from './self-update.js';
|
|
37
|
-
import { cmdDoctor
|
|
37
|
+
import { cmdDoctor } from './doctor.js';
|
|
38
38
|
import { DOCS_URL, formatOverview, availableAgentClis, missingAgentCliMessage, missingTokenMessage, notInitializedMessage, } from './first-run.js';
|
|
39
39
|
import { cmdHostedSmoke } from './hosted.js';
|
|
40
40
|
import { cmdHostedQueue } from './hosted-queue.js';
|
|
@@ -48,10 +48,8 @@ import { selectOctokit } from './github-stub.js';
|
|
|
48
48
|
import { createFactoryOctokit } from './octokit.js';
|
|
49
49
|
import { DeepCheckError, createDeepModelRunner, runIssueCheck } from './ready-check.js';
|
|
50
50
|
import { childRunSucceeded, formatChildRunSummary, runChildrenInOrder } from './run-children.js';
|
|
51
|
-
import {
|
|
52
|
-
import { readRunFlagOverrides, writeRunFlagOverrides } from './run-flags.js';
|
|
51
|
+
import { readRunFlagOverrides } from './run-flags.js';
|
|
53
52
|
import { runStalenessGuard } from './staleness.js';
|
|
54
|
-
import { readStopFileStatus, stopSentinelRunSkipMessage } from './stop-sentinel.js';
|
|
55
53
|
const DEFAULT_CONFIG_LOADERS = {
|
|
56
54
|
loadModelsConfig,
|
|
57
55
|
loadRoutesConfig,
|
|
@@ -2517,194 +2515,9 @@ function usageFlagOverrides(opts, threshold) {
|
|
|
2517
2515
|
pollSeconds: opts.usagePoll === undefined ? undefined : Number(opts.usagePoll),
|
|
2518
2516
|
};
|
|
2519
2517
|
}
|
|
2520
|
-
async function cmdRun(opts = {}) {
|
|
2521
|
-
const repoRoot = await getRepoRoot();
|
|
2522
|
-
const paths = getFactoryPaths(repoRoot);
|
|
2523
|
-
const branchPrefix = branchPrefixOrExit(opts.branchPrefix);
|
|
2524
|
-
return withRepoRunLock(paths, 'factory run', async () => {
|
|
2525
|
-
const stopStatus = readStopFileStatus(paths);
|
|
2526
|
-
if (stopStatus.present) {
|
|
2527
|
-
const msg = stopSentinelRunSkipMessage(stopStatus);
|
|
2528
|
-
console.log(styleText('yellow', `!! ${msg}`));
|
|
2529
|
-
logEvent(paths.events, 'stopped', 'all', msg);
|
|
2530
|
-
return;
|
|
2531
|
-
}
|
|
2532
|
-
writeRunFlagOverrides(paths.runFlags, { autoMerge: opts.autoMerge, prClassifier: opts.prClassifier });
|
|
2533
|
-
const ghRepo = await getGitHubRepo();
|
|
2534
|
-
const factoryConfig = configLoaders.loadFactoryConfigForRepo(paths.config);
|
|
2535
|
-
const keychainErr = keychainPreflightError(probeClaudeKeychain());
|
|
2536
|
-
if (keychainErr) {
|
|
2537
|
-
logEvent(paths.events, 'environment_warning', 'all', keychainErr);
|
|
2538
|
-
throw new Error(keychainErr);
|
|
2539
|
-
}
|
|
2540
|
-
await runAutoWorktreeGc(repoRoot, paths, ghRepo, 'all');
|
|
2541
|
-
const { lanes, diagnostics } = await planRunLanes({
|
|
2542
|
-
localQueue: opts.localQueue === true,
|
|
2543
|
-
readLocalQueue: () => (existsSync(paths.queue) ? readFileSync(paths.queue, 'utf-8') : null),
|
|
2544
|
-
queue: () => {
|
|
2545
|
-
const [owner, repo] = ghRepo.split('/');
|
|
2546
|
-
const octokit = getOctokit();
|
|
2547
|
-
return createGithubQueue({
|
|
2548
|
-
client: createOctokitQueueClient(octokit),
|
|
2549
|
-
owner,
|
|
2550
|
-
repo,
|
|
2551
|
-
preflight: withAdmissionGuard({
|
|
2552
|
-
reader: admissionStateReaderFor(paths.state),
|
|
2553
|
-
preflight: (issue) => preflightQueuedIssue(issue, createQueuePreflightOps(octokit, owner, repo, {
|
|
2554
|
-
autoMergeStack: resolveSizeGateMode(factoryConfig) === 'stack' &&
|
|
2555
|
-
resolveMergePolicy(factoryConfig, process.env, { auto: opts.autoMerge }).auto,
|
|
2556
|
-
})),
|
|
2557
|
-
onConflict: (issue, verdict) => {
|
|
2558
|
-
const line = `#${issue} refused — ${verdict.reason} | repair: ${verdict.repair}`;
|
|
2559
|
-
console.error(styleText('yellow', `factory: ${line}`));
|
|
2560
|
-
logEvent(paths.events, 'queue_admission_conflict', String(issue), line);
|
|
2561
|
-
},
|
|
2562
|
-
}),
|
|
2563
|
-
});
|
|
2564
|
-
},
|
|
2565
|
-
});
|
|
2566
|
-
warnQueueDiagnostics(diagnostics);
|
|
2567
|
-
let knobs;
|
|
2568
|
-
let laneBreakerThreshold;
|
|
2569
|
-
try {
|
|
2570
|
-
knobs = resolveUsageKnobs(process.env, configLoaders.loadRepoConfig(repoRoot), usageFlagOverrides(opts, 'stopAt'));
|
|
2571
|
-
laneBreakerThreshold = resolveLaneBreakerThreshold(configLoaders.loadRepoConfig(repoRoot));
|
|
2572
|
-
}
|
|
2573
|
-
catch (err) {
|
|
2574
|
-
throw new CliExitError(`factory: ${err.message}`, 2);
|
|
2575
|
-
}
|
|
2576
|
-
if (knobs.watchSource === 'flag') {
|
|
2577
|
-
const detail = `usage watchdog ${knobs.watch ? 'enabled' : 'disabled'} by ${usageWatchSourceLabel(knobs.watchSource, knobs.watch)}`;
|
|
2578
|
-
console.log(`[factory] run: ${detail}`);
|
|
2579
|
-
logEvent(paths.events, 'watchdog', 'all', detail);
|
|
2580
|
-
}
|
|
2581
|
-
if (knobs.stopAtSource === 'flag') {
|
|
2582
|
-
const detail = 'usage watchdog threshold set by flag: --usage-threshold';
|
|
2583
|
-
console.log(`[factory] run: ${detail}`);
|
|
2584
|
-
logEvent(paths.events, 'watchdog', 'all', detail);
|
|
2585
|
-
}
|
|
2586
|
-
if (knobs.pollSource === 'flag') {
|
|
2587
|
-
const detail = 'usage watchdog poll set by flag: --usage-poll';
|
|
2588
|
-
console.log(`[factory] run: ${detail}`);
|
|
2589
|
-
logEvent(paths.events, 'watchdog', 'all', detail);
|
|
2590
|
-
}
|
|
2591
|
-
let watchdogHarness;
|
|
2592
|
-
if (knobs.watch) {
|
|
2593
|
-
try {
|
|
2594
|
-
const repoConfig = configLoaders.loadRepoConfig(repoRoot);
|
|
2595
|
-
const registry = new coreOps.ModelRegistry(applyRepoConfig(configLoaders.loadModelsConfig(), repoConfig));
|
|
2596
|
-
const pins = configLoaders.resolveEffectiveModelPins(registry, repoConfig, process.env);
|
|
2597
|
-
const build = resolveEffectiveBuildRoute(registry, repoConfig, pins);
|
|
2598
|
-
watchdogHarness = resolveRunUsageHarness({
|
|
2599
|
-
buildRoute: build.route,
|
|
2600
|
-
buildRouteSource: build.source,
|
|
2601
|
-
planHarnesses: [pins.plan, pins.planFallback]
|
|
2602
|
-
.filter((m) => !!m)
|
|
2603
|
-
.map((m) => registry.getHarnessId(m)),
|
|
2604
|
-
codexDisabled: resolveCodexDisabled(repoConfig),
|
|
2605
|
-
localOnly: resolveLocalOnly(),
|
|
2606
|
-
});
|
|
2607
|
-
}
|
|
2608
|
-
catch {
|
|
2609
|
-
watchdogHarness = undefined; // unresolvable config keeps today's Claude gating
|
|
2610
|
-
}
|
|
2611
|
-
}
|
|
2612
|
-
const controller = new AbortController();
|
|
2613
|
-
const watchdog = knobs.watch
|
|
2614
|
-
? coreOps
|
|
2615
|
-
.watchUsage({
|
|
2616
|
-
cap: knobs.cap,
|
|
2617
|
-
stopAt: knobs.stopAt,
|
|
2618
|
-
pollMs: knobs.pollMs,
|
|
2619
|
-
stopFile: paths.stop,
|
|
2620
|
-
eventsFile: paths.events,
|
|
2621
|
-
signal: controller.signal,
|
|
2622
|
-
estimator: knobs.estimator,
|
|
2623
|
-
...(watchdogHarness ? { harness: watchdogHarness } : {}),
|
|
2624
|
-
})
|
|
2625
|
-
.catch((err) => {
|
|
2626
|
-
// a watchdog crash must never take down the run
|
|
2627
|
-
logEvent(paths.events, 'warn', '-', `usage watchdog crashed: ${err.message}`);
|
|
2628
|
-
})
|
|
2629
|
-
: Promise.resolve();
|
|
2630
|
-
const { proxy } = await startLaneProxy(paths, resolveEnvironmentProxy(factoryConfig));
|
|
2631
|
-
const workspaceBackend = factoryConfig.workspace?.backend ?? 'host';
|
|
2632
|
-
const runId = randomUUID();
|
|
2633
|
-
const containerEngine = workspaceBackend === 'disposable-docker' ? createDockerEngine({}) : undefined;
|
|
2634
|
-
try {
|
|
2635
|
-
// Run lanes in parallel
|
|
2636
|
-
const pids = [];
|
|
2637
|
-
for (const planned of lanes) {
|
|
2638
|
-
logEvent(paths.events, 'lane-start', '-', `lane '${planned.lane}' started${planned.issues.length ? ` (${planned.issues.length} issues)` : ''}`, { lane: planned.lane });
|
|
2639
|
-
if (containerEngine) {
|
|
2640
|
-
const provision = await provisionLaneContainer(containerEngine, workspaceBackend, runId, slugify(planned.lane), ghRepo);
|
|
2641
|
-
if (provision.attempted && !provision.created) {
|
|
2642
|
-
logEvent(paths.events, 'warn', '-', `disposable-docker container creation failed for lane '${planned.lane}': ${provision.error}`, { lane: planned.lane });
|
|
2643
|
-
}
|
|
2644
|
-
else if (provision.attempted && provision.created && provision.workspaceCloned === false) {
|
|
2645
|
-
logEvent(paths.events, 'warn', '-', `disposable-docker workspace clone failed for lane '${planned.lane}': ${provision.workspaceError}`, { lane: planned.lane });
|
|
2646
|
-
}
|
|
2647
|
-
}
|
|
2648
|
-
pids.push(runLane(planned.lane, planned.issues, repoRoot, ghRepo, paths, {
|
|
2649
|
-
...planned.deps,
|
|
2650
|
-
laneBreakerThreshold,
|
|
2651
|
-
baseHoldGate: opts.localQueue === true
|
|
2652
|
-
? undefined
|
|
2653
|
-
: createBaseHoldGate({
|
|
2654
|
-
file: paths.baseHold,
|
|
2655
|
-
repo: ghRepo,
|
|
2656
|
-
onCleared: (c) => logEvent(paths.events, 'base-hold-cleared', planned.lane, formatBaseHoldCleared(c), {
|
|
2657
|
-
lane: planned.lane,
|
|
2658
|
-
}),
|
|
2659
|
-
git: (cmd) => shellOps.exec(cmd, { cwd: repoRoot }).then((r) => r.stdout),
|
|
2660
|
-
listCheckRuns: (ref) => {
|
|
2661
|
-
const [owner, repoName] = ghRepo.split('/');
|
|
2662
|
-
return createOctokitGreenPrClient(getOctokit()).listCheckRuns({ owner, repo: repoName, ref });
|
|
2663
|
-
},
|
|
2664
|
-
}),
|
|
2665
|
-
commentIssue: async (issue, body) => {
|
|
2666
|
-
const [owner, repoName] = ghRepo.split('/');
|
|
2667
|
-
await getOctokit().rest.issues.createComment({ owner, repo: repoName, issue_number: issue, body });
|
|
2668
|
-
},
|
|
2669
|
-
ship: (issue, o, c) => shipIssue(issue, { ...o, branchPrefix, prClassifier: opts.prClassifier }, c),
|
|
2670
|
-
sweepFinished: (issue) => sweepBetweenLaneIssues(issue, planned.lane, repoRoot, ghRepo, paths).then(() => { }),
|
|
2671
|
-
reapWorktree: (issue) => reapParkedLaneWorktree(issue, repoRoot, ghRepo, paths, opts.branchPrefix === undefined ? undefined : branchPrefix),
|
|
2672
|
-
recordStackMerge: (issue, prNumber) => recordSliceMerged({ octokit: getOctokit(), repo: ghRepo, issue, prNumber }).then(() => { }),
|
|
2673
|
-
waitMerge: (issue, branch, root, gh, p, d = {}) => waitForMerge(issue, branch, root, gh, p, { ...d, mergeOverrides: { auto: opts.autoMerge } }),
|
|
2674
|
-
}));
|
|
2675
|
-
}
|
|
2676
|
-
await Promise.allSettled(pids);
|
|
2677
|
-
controller.abort();
|
|
2678
|
-
await watchdog;
|
|
2679
|
-
logEvent(paths.events, 'run-done', 'all', 'all lanes finished');
|
|
2680
|
-
}
|
|
2681
|
-
finally {
|
|
2682
|
-
if (proxy) {
|
|
2683
|
-
await proxy.stop();
|
|
2684
|
-
clearProxyState(paths.proxyState);
|
|
2685
|
-
}
|
|
2686
|
-
}
|
|
2687
|
-
if (lanes.length > 0) {
|
|
2688
|
-
try {
|
|
2689
|
-
const events = existsSync(paths.events) ? readEvents(paths.events) : [];
|
|
2690
|
-
const costs = existsSync(paths.costs) ? internalOps.readCosts(paths.costs) : [];
|
|
2691
|
-
const recon = await reconstructHumanEventsForKpis(events, branchPrefix);
|
|
2692
|
-
const kpis = computeHealthKpis(recon.events, costs);
|
|
2693
|
-
await appendKpiSnapshot(paths, repoRoot, kpis, { partial: recon.partial });
|
|
2694
|
-
if (recon.partial) {
|
|
2695
|
-
logEvent(paths.events, 'warn', 'all', `kpi snapshot partial: GitHub human-event reconstruction failed: ${recon.error}`);
|
|
2696
|
-
}
|
|
2697
|
-
logEvent(paths.events, 'kpi-snapshot', 'all', `KPI snapshot appended to ${paths.kpiHistory}`);
|
|
2698
|
-
}
|
|
2699
|
-
catch (err) {
|
|
2700
|
-
logEvent(paths.events, 'warn', 'all', `kpi snapshot failed: ${err.message}`);
|
|
2701
|
-
}
|
|
2702
|
-
}
|
|
2703
|
-
});
|
|
2704
|
-
}
|
|
2705
2518
|
/** Wraps cmdRun so an empty queue is a no-op (not a throw) while auto-ingest is enabled and watching. */
|
|
2706
2519
|
export function createSuperviseRunQueue(paths, ingestCfg, deps = {}) {
|
|
2707
|
-
const { cmdRunFn =
|
|
2520
|
+
const { cmdRunFn = runRun, readQueueFile = () => (existsSync(paths.queue) ? readFileSync(paths.queue, 'utf-8') : ''), pendingCount = async () => parseQueue(readQueueFile()).entries.length, emitEvent = logEvent, } = deps;
|
|
2708
2521
|
return async () => {
|
|
2709
2522
|
if ((await pendingCount()) === 0 && ingestCfg.enabled) {
|
|
2710
2523
|
emitEvent(paths.events, 'idle', 'auto', 'queue empty — waiting for ready issues');
|
|
@@ -2937,234 +2750,6 @@ export async function planRunLanes(input) {
|
|
|
2937
2750
|
diagnostics: [],
|
|
2938
2751
|
};
|
|
2939
2752
|
}
|
|
2940
|
-
/** How often an in-flight issue's claim lease is refreshed (#1500). Well inside the
|
|
2941
|
-
* 15-minute default lease (github-queue.ts's DEFAULT_CLAIM_LEASE_MS) so a normal poll
|
|
2942
|
-
* cadence never lets the lease lapse mid-ship. */
|
|
2943
|
-
const HEARTBEAT_INTERVAL_MS = 5 * 60 * 1000;
|
|
2944
|
-
/** Runs `work` while periodically calling `heartbeat(issue)` so a long-running ship or
|
|
2945
|
-
* merge wait keeps this claimant's lease fresh. A heartbeat failure is swallowed — losing
|
|
2946
|
-
* one refresh must never abort the work it is protecting. */
|
|
2947
|
-
async function withHeartbeat(issue, heartbeat, work) {
|
|
2948
|
-
if (!heartbeat)
|
|
2949
|
-
return work();
|
|
2950
|
-
const timer = setInterval(() => {
|
|
2951
|
-
heartbeat(issue).catch(() => { });
|
|
2952
|
-
}, HEARTBEAT_INTERVAL_MS);
|
|
2953
|
-
try {
|
|
2954
|
-
return await work();
|
|
2955
|
-
}
|
|
2956
|
-
finally {
|
|
2957
|
-
clearInterval(timer);
|
|
2958
|
-
}
|
|
2959
|
-
}
|
|
2960
|
-
export function renderLanePausedComment(payload) {
|
|
2961
|
-
const checks = payload.failingChecks.length
|
|
2962
|
-
? payload.failingChecks.map((c) => `\`${c}\``).join(', ')
|
|
2963
|
-
: 'none recorded';
|
|
2964
|
-
return [
|
|
2965
|
-
`Lane '${payload.lane}' is paused: this issue and #${payload.firstIssue} parked with the same failure signature, so the factory stopped claiming issues on this lane.`,
|
|
2966
|
-
`Shared failing checks: ${checks}`,
|
|
2967
|
-
`To resume: fix the shared failure (often on the base branch), then re-run \`factory run\` to resume lane '${payload.lane}'.`,
|
|
2968
|
-
`Failure signature: \`${payload.signature}\``,
|
|
2969
|
-
].join('\n\n');
|
|
2970
|
-
}
|
|
2971
|
-
export async function runLane(lane, issues, repoRoot, ghRepo, paths, deps = {}) {
|
|
2972
|
-
const { ship = shipIssue, waitMerge = waitForMerge, recordStackMerge = async () => { }, pathExists = existsSync, emitEvent = logEvent, claimNext = async () => null, releaseIssue = async () => { }, heartbeat, reapWorktree = async () => { }, sweepFinished = async () => { }, countRemaining, baseHoldGate, laneBreakerThreshold = 0, commentIssue = async () => { }, } = deps;
|
|
2973
|
-
let streak = null;
|
|
2974
|
-
let merged = 0;
|
|
2975
|
-
let awaitingReview = 0;
|
|
2976
|
-
let skipped = 0;
|
|
2977
|
-
let decomposed = 0;
|
|
2978
|
-
let parked = 0;
|
|
2979
|
-
const buildClaim = (issue) => ({ issue, decision: { kind: 'build' } });
|
|
2980
|
-
const pending = issues.map(buildClaim);
|
|
2981
|
-
const seen = new Set(issues);
|
|
2982
|
-
// Truthful park-time remaining count (#1222): local pending plus the lane's still-queued
|
|
2983
|
-
// remote issues. Fail-closed — a count lookup may never change the message beyond the
|
|
2984
|
-
// local truth, and never lane flow.
|
|
2985
|
-
const remainingAfter = async (index) => {
|
|
2986
|
-
const local = pending.length - index - 1;
|
|
2987
|
-
if (!countRemaining)
|
|
2988
|
-
return local;
|
|
2989
|
-
try {
|
|
2990
|
-
return local + (await countRemaining());
|
|
2991
|
-
}
|
|
2992
|
-
catch {
|
|
2993
|
-
return local;
|
|
2994
|
-
}
|
|
2995
|
-
};
|
|
2996
|
-
// Every terminal outcome releases the claim, then gives finished worktrees a GC pass before
|
|
2997
|
-
// the lane moves on (#1757). The pass is fail-closed and must never change lane flow.
|
|
2998
|
-
const settle = async (issue, outcome) => {
|
|
2999
|
-
await releaseIssue(issue, outcome);
|
|
3000
|
-
await sweepFinished(issue).catch(() => { });
|
|
3001
|
-
};
|
|
3002
|
-
for (let i = 0;; i++) {
|
|
3003
|
-
if (i >= pending.length) {
|
|
3004
|
-
// A persisted base hold on the unchanged origin/main SHA means the base is still broken
|
|
3005
|
-
// (#2349): claiming would just cycle an issue again, so stop this lane without claiming.
|
|
3006
|
-
const gate = baseHoldGate ? await baseHoldGate().catch(() => ({ kind: 'claim' })) : null;
|
|
3007
|
-
if (gate?.kind === 'hold') {
|
|
3008
|
-
emitEvent(paths.events, 'warn', lane, `lane '${lane}' not claiming: base hold on ${gate.hold.baseSha.slice(0, 8)} (${gate.hold.failingChecks.join(', ')}) — origin/main unchanged and its CI is not green`, { lane });
|
|
3009
|
-
return;
|
|
3010
|
-
}
|
|
3011
|
-
const claimed = await claimNext();
|
|
3012
|
-
if (claimed === null)
|
|
3013
|
-
break;
|
|
3014
|
-
seen.add(claimed.issue);
|
|
3015
|
-
pending.push(claimed);
|
|
3016
|
-
}
|
|
3017
|
-
const { issue, decision } = pending[i];
|
|
3018
|
-
if (pathExists(paths.stop)) {
|
|
3019
|
-
emitEvent(paths.events, 'stopped', issue, 'STOP file present', { lane });
|
|
3020
|
-
await releaseIssue(issue, 'queued');
|
|
3021
|
-
return;
|
|
3022
|
-
}
|
|
3023
|
-
if (decision.kind === 'park') {
|
|
3024
|
-
emitEvent(paths.events, 'escalate', issue, decision.reason, { lane });
|
|
3025
|
-
emitEvent(paths.events, 'parked', issue, `issue #${issue} parked (${decision.reason}); lane '${lane}' continuing, ${await remainingAfter(i)} issue(s) remaining`, { lane });
|
|
3026
|
-
// A preflight-parked issue may still have a stale worktree from a prior run; the reap
|
|
3027
|
-
// is fail-closed and must never change lane flow.
|
|
3028
|
-
await reapWorktree(issue).catch(() => { });
|
|
3029
|
-
await settle(issue, 'parked');
|
|
3030
|
-
parked++;
|
|
3031
|
-
streak = null;
|
|
3032
|
-
continue;
|
|
3033
|
-
}
|
|
3034
|
-
const layer = decision.kind === 'adopt' ? decision.stackLayer : undefined;
|
|
3035
|
-
try {
|
|
3036
|
-
const sliceOutcome = {};
|
|
3037
|
-
const branch = decision.kind === 'adopt'
|
|
3038
|
-
? decision.branch
|
|
3039
|
-
: await withHeartbeat(issue, heartbeat, () => ship(issue, {}, {
|
|
3040
|
-
repoRoot,
|
|
3041
|
-
ghRepo,
|
|
3042
|
-
paths,
|
|
3043
|
-
lane,
|
|
3044
|
-
onOutcome: (o) => {
|
|
3045
|
-
sliceOutcome.value = o;
|
|
3046
|
-
},
|
|
3047
|
-
}));
|
|
3048
|
-
const shipped = sliceOutcome.value;
|
|
3049
|
-
if (shipped?.state === 'ready' && shipped.slice && shipped.slice.index < shipped.slice.count) {
|
|
3050
|
-
// ADR-0147: a non-final slice's PR is open; hand the issue back for the next slice
|
|
3051
|
-
// instead of waiting for the merge.
|
|
3052
|
-
awaitingReview++;
|
|
3053
|
-
streak = null;
|
|
3054
|
-
await settle(issue, 'queued');
|
|
3055
|
-
continue;
|
|
3056
|
-
}
|
|
3057
|
-
if (layer) {
|
|
3058
|
-
// ADR-0161: land exactly this layer's branch, record it, and re-queue for the next layer.
|
|
3059
|
-
await withHeartbeat(issue, heartbeat, () => waitMerge(issue, branch, repoRoot, ghRepo, paths, { landBranch: branch }));
|
|
3060
|
-
await recordStackMerge(issue, layer.prNumber).catch((err) => emitEvent(paths.events, 'warn', issue, `slice ${layer.index}/${layer.count} merged but recording it failed: ${errorDetail(err)}`, { lane }));
|
|
3061
|
-
merged++;
|
|
3062
|
-
streak = null;
|
|
3063
|
-
await settle(issue, layer.index < layer.count ? 'queued' : 'done');
|
|
3064
|
-
continue;
|
|
3065
|
-
}
|
|
3066
|
-
await withHeartbeat(issue, heartbeat, () => waitMerge(issue, branch, repoRoot, ghRepo, paths));
|
|
3067
|
-
merged++;
|
|
3068
|
-
streak = null;
|
|
3069
|
-
await settle(issue, 'done');
|
|
3070
|
-
}
|
|
3071
|
-
catch (err) {
|
|
3072
|
-
if (err instanceof AwaitingReviewError) {
|
|
3073
|
-
// The land path already emitted the awaiting-review event and cleaned the
|
|
3074
|
-
// worktree — this is a clean outcome, not a park; move to the next issue.
|
|
3075
|
-
awaitingReview++;
|
|
3076
|
-
streak = null;
|
|
3077
|
-
await settle(issue, layer && layer.index < layer.count ? 'queued' : 'done');
|
|
3078
|
-
continue;
|
|
3079
|
-
}
|
|
3080
|
-
if (err instanceof IssueSkippedError) {
|
|
3081
|
-
// shipIssue already emitted skipped-already-closed; nothing was attempted.
|
|
3082
|
-
skipped++;
|
|
3083
|
-
await settle(issue, 'done');
|
|
3084
|
-
continue;
|
|
3085
|
-
}
|
|
3086
|
-
if (err instanceof IssueDecomposedError) {
|
|
3087
|
-
// shipIssue already rewrote the queue — splice the children in after the
|
|
3088
|
-
// current issue so this same lane run ships them, deduping ones already queued.
|
|
3089
|
-
const fresh = err.childIssues.filter((n) => !seen.has(n));
|
|
3090
|
-
for (const n of fresh)
|
|
3091
|
-
seen.add(n);
|
|
3092
|
-
pending.splice(i + 1, 0, ...fresh.map(buildClaim));
|
|
3093
|
-
decomposed++;
|
|
3094
|
-
emitEvent(paths.events, 'decompose_filed', issue, `lane '${lane}' continuing with ${fresh.length} child issue(s) in place of #${issue}`, { lane });
|
|
3095
|
-
await settle(issue, 'done');
|
|
3096
|
-
continue;
|
|
3097
|
-
}
|
|
3098
|
-
if (err instanceof LaneEnvironmentPauseError) {
|
|
3099
|
-
// The base is broken, not the issue (#1928): release to queued (never parked),
|
|
3100
|
-
// pause the lane whatever laneBreakerThreshold is, and stop claiming.
|
|
3101
|
-
streak = null;
|
|
3102
|
-
// A preserved build is kept for `factory resume --from check` (#2564).
|
|
3103
|
-
if (!err.buildPreserved)
|
|
3104
|
-
await reapWorktree(issue).catch(() => { });
|
|
3105
|
-
await settle(issue, 'queued');
|
|
3106
|
-
// Persist the hold so a restarted supervisor remembers the broken base (#2348);
|
|
3107
|
-
// best-effort, never allowed to change lane flow.
|
|
3108
|
-
try {
|
|
3109
|
-
writeBaseHold(paths.baseHold, {
|
|
3110
|
-
repo: ghRepo,
|
|
3111
|
-
baseSha: err.baseSha,
|
|
3112
|
-
failingChecks: err.failingChecks,
|
|
3113
|
-
since: new Date().toISOString(),
|
|
3114
|
-
});
|
|
3115
|
-
}
|
|
3116
|
-
catch { }
|
|
3117
|
-
const payload = {
|
|
3118
|
-
lane,
|
|
3119
|
-
signature: err.failureSignature ?? '',
|
|
3120
|
-
failingChecks: err.failingChecks,
|
|
3121
|
-
firstIssue: issue,
|
|
3122
|
-
secondIssue: issue,
|
|
3123
|
-
cause: 'environment',
|
|
3124
|
-
baseSha: err.baseSha,
|
|
3125
|
-
};
|
|
3126
|
-
emitEvent(paths.events, 'lane-paused', lane, `lane '${lane}' paused: #${issue} failed only on checkers that also fail on base ${err.baseSha.slice(0, 8)} (${err.failingChecks.join(', ')}); #${issue} released to queued`, { lane, lanePaused: payload });
|
|
3127
|
-
return;
|
|
3128
|
-
}
|
|
3129
|
-
const reason = parkReasonFor(err);
|
|
3130
|
-
// Terminal reason events (escalate/timeout/fail/conflict) are emitted exactly
|
|
3131
|
-
// once by the layer that detects the failure — shipIssue for pipeline failures,
|
|
3132
|
-
// the land path for merge failures. runLane owns only lane-lifecycle events
|
|
3133
|
-
// (stopped/parked/lane-done), so injected ship functions never change ownership.
|
|
3134
|
-
const signature = err instanceof LaneParkError ? err.failureSignature : undefined;
|
|
3135
|
-
const checks = err instanceof LaneParkError ? (err.failingChecks ?? []) : [];
|
|
3136
|
-
if (!signature)
|
|
3137
|
-
streak = null;
|
|
3138
|
-
else if (streak && streak.signature === signature)
|
|
3139
|
-
streak.count++;
|
|
3140
|
-
else
|
|
3141
|
-
streak = { signature, firstIssue: issue, failingChecks: checks, count: 1 };
|
|
3142
|
-
const tripped = laneBreakerThreshold > 0 && streak !== null && streak.count >= laneBreakerThreshold;
|
|
3143
|
-
emitEvent(paths.events, 'parked', issue, `issue #${issue} parked (${reason}); ${tripped ? `lane '${lane}' paused (lane breaker)` : `lane '${lane}' continuing`}, ${await remainingAfter(i)} issue(s) remaining`, { lane });
|
|
3144
|
-
// Remove the parked lane's own worktree so a parked issue stops leaving a sibling
|
|
3145
|
-
// checkout behind (#1007); fail-closed and never allowed to change lane flow.
|
|
3146
|
-
await reapWorktree(issue).catch(() => { });
|
|
3147
|
-
await settle(issue, 'parked');
|
|
3148
|
-
parked++;
|
|
3149
|
-
if (tripped && streak) {
|
|
3150
|
-
const { signature: tripSignature, firstIssue, failingChecks: firstChecks } = streak;
|
|
3151
|
-
const payload = {
|
|
3152
|
-
lane,
|
|
3153
|
-
signature: tripSignature,
|
|
3154
|
-
failingChecks: checks.filter((c) => firstChecks.includes(c)),
|
|
3155
|
-
firstIssue,
|
|
3156
|
-
secondIssue: issue,
|
|
3157
|
-
};
|
|
3158
|
-
emitEvent(paths.events, 'lane-paused', lane, `lane '${lane}' paused: #${payload.firstIssue} and #${payload.secondIssue} parked with the same failure signature (${payload.failingChecks.join(', ') || 'no failing checks'})`, { lane, lanePaused: payload });
|
|
3159
|
-
// Tell the developer on GitHub why the lane stopped (#1919); fail-closed, never changes lane flow.
|
|
3160
|
-
await commentIssue(payload.secondIssue, renderLanePausedComment(payload)).catch(() => { });
|
|
3161
|
-
return;
|
|
3162
|
-
}
|
|
3163
|
-
continue;
|
|
3164
|
-
}
|
|
3165
|
-
}
|
|
3166
|
-
emitEvent(paths.events, 'lane-done', lane, `lane complete (${merged} merged, ${awaitingReview} awaiting review, ${skipped} skipped, ${decomposed} decomposed, ${parked} parked)`, { lane });
|
|
3167
|
-
}
|
|
3168
2753
|
/** The target issue was already resolved before the run started — a clean skip,
|
|
3169
2754
|
* not a park. Callers advance to the next queue entry. (#681) */
|
|
3170
2755
|
export class IssueSkippedError extends Error {
|
|
@@ -3511,7 +3096,7 @@ function runSupervise(opts) {
|
|
|
3511
3096
|
superviseLoop,
|
|
3512
3097
|
createSuperviseRunQueue,
|
|
3513
3098
|
createIngestHook,
|
|
3514
|
-
cmdRun: (o) =>
|
|
3099
|
+
cmdRun: (o) => runRun(o),
|
|
3515
3100
|
CliExitError,
|
|
3516
3101
|
});
|
|
3517
3102
|
}
|
|
@@ -3531,6 +3116,35 @@ function runStatus(opts = {}) {
|
|
|
3531
3116
|
CliExitError,
|
|
3532
3117
|
});
|
|
3533
3118
|
}
|
|
3119
|
+
/** `factory run` (#2800): the handler lives in ./run.ts; this binds its runtime. */
|
|
3120
|
+
function runRun(opts = {}) {
|
|
3121
|
+
return cmdRun(opts, {
|
|
3122
|
+
configLoaders,
|
|
3123
|
+
coreOps,
|
|
3124
|
+
shellOps,
|
|
3125
|
+
readCosts: internalOps.readCosts,
|
|
3126
|
+
getRepoRoot: () => getRepoRoot(),
|
|
3127
|
+
getGitHubRepo: () => getGitHubRepo(),
|
|
3128
|
+
getOctokit: () => getOctokit(),
|
|
3129
|
+
branchPrefixOrExit,
|
|
3130
|
+
withRepoRunLock,
|
|
3131
|
+
runAutoWorktreeGc,
|
|
3132
|
+
planRunLanes,
|
|
3133
|
+
preflightQueuedIssue,
|
|
3134
|
+
createQueuePreflightOps,
|
|
3135
|
+
warnQueueDiagnostics,
|
|
3136
|
+
resolveUsageKnobs,
|
|
3137
|
+
usageFlagOverrides,
|
|
3138
|
+
usageWatchSourceLabel,
|
|
3139
|
+
probeClaudeKeychain,
|
|
3140
|
+
startLaneProxy,
|
|
3141
|
+
sweepBetweenLaneIssues,
|
|
3142
|
+
reapParkedLaneWorktree,
|
|
3143
|
+
reconstructHumanEventsForKpis,
|
|
3144
|
+
appendKpiSnapshot,
|
|
3145
|
+
CliExitError,
|
|
3146
|
+
});
|
|
3147
|
+
}
|
|
3534
3148
|
/** `factory doctor` (#2799): the handler lives in ./doctor.ts; this binds its runtime. */
|
|
3535
3149
|
function runDoctor(opts = {}) {
|
|
3536
3150
|
return cmdDoctor(opts, {
|
|
@@ -3586,7 +3200,7 @@ export const PROGRAM_HANDLERS = {
|
|
|
3586
3200
|
cmdLocalSmallOvernight,
|
|
3587
3201
|
cmdLand,
|
|
3588
3202
|
cmdResumeApproved,
|
|
3589
|
-
cmdRun,
|
|
3203
|
+
cmdRun: runRun,
|
|
3590
3204
|
cmdProxy,
|
|
3591
3205
|
cmdFactoryd,
|
|
3592
3206
|
cmdDaemonStart,
|