headlesscode 1.0.2

package/src/watcher/cli.ts

@@ -0,0 +1,369 @@
+/**
+ * `headlesscode watch` subcommand — Phase 5 GitHub issue watcher.
+ *
+ *   npx tsx src/cli.ts watch --owner <o> --repo <local-clone-path> --label <target-label> \
+ *       [--gh-repo <name>] [--poll-interval-ms <n>] [--run-once] [--max-per-sweep <n>] \
+ *       [--state-file <path>] [--mode <slug>] [--qa] [--deploy] [--memory-dir <path>] \
+ *       [--dry-run] [--retry-failed]
+ *
+ * Thin dispatch to src/watcher/watch.ts — no logic duplicated here. The
+ * GitHub repo name defaults to the basename of `--repo` (the local clone),
+ * which is the common case; override with `--gh-repo` when the directory
+ * name differs.
+ *
+ * `--dry-run` sweeps + reports what would be spawned (list issues + split
+ * plan per issue) WITHOUT spawning or writing state. It still needs a GitHub
+ * token for listIssues — a missing token is a clear error.
+ *
+ * `--run-once` performs one sweep and exits 0 (or 1 if any spawn failed) —
+ * the cron/CI-friendly form. Continuous mode (default) logs each sweep and
+ * runs until SIGINT/SIGTERM (clean shutdown via AbortSignal; state saved).
+ *
+ * Pass-through (recorded per batch / forwarded to the spawner): --mode and
+ * --memory-dir reach the spawner env (ORCHESTRATOR_MODE /
+ * HEADLESSCODE_MEMORY_DIR); --qa and --deploy are stored on the batch's
+ * state entry for the follow-up `orchestrate` completion run (see
+ * docs/phase5-issue-watcher.md — the watcher itself only spawns; completion
+ * watching / review / QA / deploy stays the orchestrator's job).
+ */
+
+import { execFileSync } from "node:child_process"
+import * as path from "node:path"
+
+import { Logger } from "../engine/logger.js"
+import { watchIssues, type SweepResult } from "./watch.js"
+
+const WATCH_USAGE = `headlesscode watch — Phase 5 GitHub issue watcher
+
+Usage:
+  headlesscode watch --owner <o> --repo <path> --label <target-label> [options]
+
+Options:
+  --owner <o>            GitHub owner (required)
+  --repo <path>          Local clone of the target repo (required; worktrees
+                         are spawned under <path>/.worktrees/). The GitHub
+                         repo name defaults to the directory basename.
+  --gh-repo <name>       GitHub repo name override (default: basename of --repo)
+  --label <name>         The label that triggers processing, e.g. needs-agent
+  --poll-interval-ms <n> Sweep interval in continuous mode (default: 60000)
+  --run-once             One sweep then exit 0 (or 1 if any spawn failed);
+                         the cron/CI form of the watcher
+  --max-per-sweep <n>    Max NEW issues spawned per sweep (default: 5); the
+                         rest stay in state as 'pending' and are picked up
+                         next sweep (bounds label storms)
+  --max-concurrent-sessions <n>  Phase 6 GLOBAL cap on concurrent sessions
+                         across processes (default: $HEADLESSCODE_MAX_
+                         CONCURRENT_SESSIONS or 3). Interplay: maxPerSweep
+                         bounds one sweep's burst; this bounds the total
+                         running/spawned fleet — issues beyond it stay
+                         'pending' until slots free up
+  --state-file <path>    Durable idempotency state (default:
+                         <repo>/.worktrees/.watcher-state.json)
+  --mode <slug>          Harness mode for workers (default: code; forwarded
+                         to the spawner as ORCHESTRATOR_MODE)
+  --qa                   Pass-through: record that a QA round should follow
+                         each batch (see docs/phase5-issue-watcher.md)
+  --deploy               Pass-through: record that a deploy gate should follow
+                         each batch (see docs/phase5-issue-watcher.md)
+  --memory-dir <path>    Memory dir for workers (forwarded to the spawner as
+                         HEADLESSCODE_MEMORY_DIR)
+  --dry-run              Sweep + print the spawn plan per new issue, spawn
+                         nothing, write no state (still needs a GH token)
+  --retry-failed         Retry previously-failed spawns on the next sweep
+                         (default: failed entries are NOT auto-retried)
+  --help                 Show this help and exit
+
+Environment:
+  GH_TOKEN               GitHub token (or GITHUB_TOKEN) — required; never logged
+  GITHUB_API_BASE_URL    API base URL override (tests/mocks)
+  HEADLESSCODE_SPAWN_SCRIPT  Override the spawner script path (e2e stubbing)
+  HEADLESSCODE_MAX_CONCURRENT_SESSIONS  Global concurrency cap (default 3)
+`
+
+export interface WatchCliOptions {
+	owner: string
+	repo: string
+	ghRepo?: string
+	label: string
+	pollIntervalMs: number
+	runOnce: boolean
+	maxPerSweep: number
+	/** Phase 6: global concurrent-session cap (default env or 3). */
+	maxConcurrentSessions: number
+	stateFile?: string
+	mode: string
+	qa: boolean
+	deploy: boolean
+	memoryDir?: string
+	dryRun: boolean
+	retryFailed: boolean
+	help: boolean
+}
+
+export function parseWatchArgs(argv: string[]): { options: WatchCliOptions; error?: string } {
+	const options: WatchCliOptions = {
+		owner: "",
+		repo: "",
+		label: "",
+		pollIntervalMs: 60_000,
+		runOnce: false,
+		maxPerSweep: 5,
+		maxConcurrentSessions: 3,
+		mode: "code",
+		qa: false,
+		deploy: false,
+		dryRun: false,
+		retryFailed: false,
+		help: false,
+	}
+
+	for (let i = 0; i < argv.length; i++) {
+		const arg = argv[i]
+		const eq = arg.indexOf("=")
+		const flag = eq === -1 ? arg : arg.slice(0, eq)
+		const inlineValue = eq === -1 ? undefined : arg.slice(eq + 1)
+		const next = (): string | undefined => {
+			if (inlineValue !== undefined) {
+				return inlineValue
+			}
+			const v = argv[i + 1]
+			if (v === undefined || v.startsWith("--")) {
+				return undefined
+			}
+			i++
+			return v
+		}
+
+		switch (flag) {
+			case "--owner": {
+				const v = next()
+				if (v === undefined) {
+					return { options, error: "Missing value for --owner" }
+				}
+				options.owner = v
+				break
+			}
+			case "--repo": {
+				const v = next()
+				if (v === undefined) {
+					return { options, error: "Missing value for --repo" }
+				}
+				options.repo = v
+				break
+			}
+			case "--gh-repo": {
+				const v = next()
+				if (v === undefined) {
+					return { options, error: "Missing value for --gh-repo" }
+				}
+				options.ghRepo = v
+				break
+			}
+			case "--label": {
+				const v = next()
+				if (v === undefined) {
+					return { options, error: "Missing value for --label" }
+				}
+				options.label = v
+				break
+			}
+			case "--poll-interval-ms": {
+				const v = next()
+				const n = v === undefined ? Number.NaN : Number(v)
+				if (!Number.isInteger(n) || n <= 0) {
+					return { options, error: "--poll-interval-ms requires a positive integer" }
+				}
+				options.pollIntervalMs = n
+				break
+			}
+			case "--max-per-sweep": {
+				const v = next()
+				const n = v === undefined ? Number.NaN : Number(v)
+				if (!Number.isInteger(n) || n <= 0) {
+					return { options, error: "--max-per-sweep requires a positive integer" }
+				}
+				options.maxPerSweep = n
+				break
+			}
+			case "--max-concurrent-sessions": {
+				const v = next()
+				const n = v === undefined ? Number.NaN : Number(v)
+				if (!Number.isInteger(n) || n <= 0) {
+					return { options, error: "--max-concurrent-sessions requires a positive integer" }
+				}
+				options.maxConcurrentSessions = n
+				break
+			}
+			case "--state-file": {
+				const v = next()
+				if (v === undefined) {
+					return { options, error: "Missing value for --state-file" }
+				}
+				options.stateFile = v
+				break
+			}
+			case "--mode": {
+				const v = next()
+				if (v === undefined) {
+					return { options, error: "Missing value for --mode" }
+				}
+				options.mode = v
+				break
+			}
+			case "--memory-dir": {
+				const v = next()
+				if (v === undefined) {
+					return { options, error: "Missing value for --memory-dir" }
+				}
+				options.memoryDir = v
+				break
+			}
+			case "--run-once":
+				options.runOnce = true
+				break
+			case "--qa":
+				options.qa = true
+				break
+			case "--deploy":
+				options.deploy = true
+				break
+			case "--dry-run":
+				options.dryRun = true
+				break
+			case "--retry-failed":
+				options.retryFailed = true
+				break
+			case "--help":
+			case "-h":
+				options.help = true
+				break
+			default:
+				return { options, error: `Unknown watch argument: ${arg}` }
+		}
+	}
+
+	return { options }
+}
+
+/** Print the dry-run spawn plan for the sweeps that were simulated. */
+function printDryRunPlan(sweeps: SweepResult[]): void {
+	process.stdout.write("── Watch plan (dry run) ────────────────────────────────────\n")
+	for (const sweep of sweeps) {
+		for (const plan of sweep.dryRunPlan ?? []) {
+			const { issue, specs } = plan
+			process.stdout.write(`  issue #${issue.number}: ${issue.title}\n`)
+			for (const spec of specs) {
+				process.stdout.write(
+					`    -> ${spec.name} issues [${spec.issues.join(", ")}] plans/parallel-tasks/${spec.taskFile}\n`,
+				)
+			}
+		}
+		if ((sweep.dryRunPlan ?? []).length === 0) {
+			process.stdout.write("  (no new issues to spawn)\n")
+		}
+	}
+}
+
+export async function watchMain(argv: string[]): Promise<number> {
+	const { options, error } = parseWatchArgs(argv)
+	if (error) {
+		process.stderr.write(`headlesscode watch: ${error}\n\n${WATCH_USAGE}`)
+		return 2
+	}
+	if (options.help) {
+		process.stdout.write(WATCH_USAGE)
+		return 0
+	}
+	if (!options.owner || !options.repo || !options.label) {
+		process.stderr.write(
+			`headlesscode watch: --owner <o>, --repo <path> and --label <name> are required\n\n${WATCH_USAGE}`,
+		)
+		return 2
+	}
+
+	const token = process.env.GH_TOKEN ?? process.env.GITHUB_TOKEN
+	if (!token) {
+		process.stderr.write(
+			"headlesscode watch: no GitHub token. Set GH_TOKEN (or GITHUB_TOKEN) — the watcher\n" +
+				"  needs it for listIssues, even with --dry-run. The token is never logged.\n",
+		)
+		return 2
+	}
+
+	const repoRoot = path.resolve(options.repo)
+	// Dry-run never touches the repo, so git validation only applies to real runs.
+	if (!options.dryRun) {
+		try {
+			execFileSync("git", ["-C", repoRoot, "rev-parse", "--git-dir"], { stdio: "ignore", timeout: 5000 })
+		} catch {
+			process.stderr.write(`headlesscode watch: not a git repo: ${repoRoot}\n`)
+			return 2
+		}
+	}
+
+	const ghRepo = options.ghRepo ?? path.basename(repoRoot)
+	const logger = new Logger({ level: "info" })
+
+	// Continuous mode: clean shutdown on SIGINT/SIGTERM (state saved by
+	// watchIssues before returning).
+	const controller = new AbortController()
+	const onSignal = (): void => controller.abort()
+	process.on("SIGINT", onSignal)
+	process.on("SIGTERM", onSignal)
+
+	let result
+	try {
+		result = await watchIssues({
+			owner: options.owner,
+			repo: ghRepo,
+			targetLabel: options.label,
+			token,
+			ghBaseUrl: process.env.GITHUB_API_BASE_URL,
+			stateFile: options.stateFile,
+			repoRoot,
+			pollIntervalMs: options.pollIntervalMs,
+			runOnce: options.runOnce,
+			maxPerSweep: options.maxPerSweep,
+			maxConcurrentSessions: options.maxConcurrentSessions,
+			retryFailed: options.retryFailed,
+			dryRun: options.dryRun,
+			mode: options.mode,
+			memoryDir: options.memoryDir,
+			qa: options.qa,
+			deploy: options.deploy,
+			logger,
+			signal: controller.signal,
+		})
+	} finally {
+		process.off("SIGINT", onSignal)
+		process.off("SIGTERM", onSignal)
+	}
+
+	for (const sweep of result.sweeps) {
+		if (sweep.error) {
+			process.stderr.write(`[watch] ${sweep.error}\n`)
+		}
+	}
+	if (options.dryRun) {
+		printDryRunPlan(result.sweeps)
+	} else {
+		for (const sweep of result.sweeps) {
+			process.stdout.write(
+				`[watch] sweep: ${sweep.issuesFound} found, ${sweep.newIssues} new, ${sweep.skipped} skipped, ` +
+					`${sweep.spawned} spawned, ${sweep.deferred} deferred, ${sweep.failures} failed\n`,
+			)
+		}
+	}
+
+	if (options.dryRun) {
+		return 0
+	}
+	if (result.exitCode !== 0) {
+		process.stderr.write(
+			`headlesscode watch: ${result.spawned} batch(es) spawned, but one or more sweeps reported errors\n`,
+		)
+		return 1
+	}
+	process.stdout.write(`[watch] done: ${result.spawned} batch(es) spawned\n`)
+	return 0
+}

package/src/watcher/github.ts

@@ -0,0 +1,304 @@
+/**
+ * Phase 5 — minimal authenticated GitHub REST client for the issue watcher.
+ *
+ * Native `fetch` only (Node >= 18), no new dependencies. GET-only for the
+ * watcher's needs (list issues by label, fetch a single issue) plus an
+ * optional `addLabel` POST used by the label-marking idempotency strategy
+ * (see docs/phase5-issue-watcher.md — the default watcher uses the state
+ * file, but the label-marking option is documented and this function exists
+ * for it).
+ *
+ * Auth: `Authorization: Bearer <token>` from `GH_TOKEN` or `GITHUB_TOKEN`
+ * (the caller passes the token in; this module never reads the environment
+ * itself so tests can inject a fake token). The token is NEVER logged and
+ * never written to the watcher state file.
+ *
+ * Base URL override: `GITHUB_API_BASE_URL` env (or the `baseUrl` option)
+ * points the client at a mock server for tests/e2e. Default api.github.com.
+ *
+ * Errors: non-2xx -> typed `GhApiError` with HTTP status + message; network
+ * failures are wrapped in a plain Error.
+ */
+
+export const DEFAULT_GH_BASE_URL = "https://api.github.com"
+/** GitHub REST API version header (see https://docs.github.com/en/rest). */
+export const GH_API_VERSION = "2022-11-28"
+/** Page size for list endpoints (GitHub max is 100). */
+export const ISSUES_PAGE_SIZE = 100
+/** Hard cap on total issues fetched per listIssues call (bounds runtime). */
+export const DEFAULT_MAX_ISSUES = 1000
+
+/** Typed error for non-2xx GitHub API responses. */
+export class GhApiError extends Error {
+	readonly status: number
+	readonly url: string
+
+	constructor(status: number, message: string, url: string) {
+		super(message)
+		this.name = "GhApiError"
+		this.status = status
+		this.url = url
+	}
+}
+
+/** A normalized GitHub issue (PRs are filtered out by listIssues/getIssue). */
+export interface GitHubIssue {
+	number: number
+	title: string
+	body?: string
+	/** Label names, e.g. ["needs-agent"]. */
+	labels: string[]
+	updated_at?: string
+	html_url?: string
+}
+
+export interface GhRequestOptions {
+	token: string
+	/** API base URL (default: $GITHUB_API_BASE_URL or https://api.github.com). */
+	baseUrl?: string
+	method?: string
+	body?: unknown
+	/** Injectable fetch for tests (default: global fetch). */
+	fetchImpl?: typeof fetch
+}
+
+/**
+ * Build a GitHub API URL from a base URL + API path + query params.
+ * Exported so tests can assert URL construction without a network.
+ */
+export function buildGhUrl(
+	baseUrl: string,
+	apiPath: string,
+	params?: Record<string, string | number | undefined>,
+): string {
+	const cleanBase = baseUrl.replace(/\/+$/, "")
+	const pathPart = apiPath.startsWith("/") ? apiPath : `/${apiPath}`
+	const url = new URL(`${cleanBase}${pathPart}`)
+	if (params) {
+		for (const [key, value] of Object.entries(params)) {
+			if (value !== undefined && value !== "") {
+				url.searchParams.set(key, String(value))
+			}
+		}
+	}
+	return url.toString()
+}
+
+/**
+ * One authenticated GitHub REST call. GET by default (plus POST for
+ * addLabel). Non-2xx -> GhApiError; network errors are wrapped.
+ */
+export async function ghApi(apiPath: string, options: GhRequestOptions): Promise<unknown> {
+	const {
+		token,
+		baseUrl = process.env.GITHUB_API_BASE_URL || DEFAULT_GH_BASE_URL,
+		method = "GET",
+		body,
+		fetchImpl = fetch,
+	} = options
+	if (!token) {
+		throw new Error("ghApi: no GitHub token provided (set GH_TOKEN or GITHUB_TOKEN)")
+	}
+	// apiPath may be a relative API path ("/repos/o/r/issues") OR a fully
+	// built URL (listIssues constructs the paginated URL itself) — pass
+	// absolute URLs through unchanged.
+	const url = /^https?:\/\//i.test(apiPath) ? apiPath : buildGhUrl(baseUrl, apiPath)
+
+	const headers: Record<string, string> = {
+		Accept: "application/vnd.github+json",
+		"X-GitHub-Api-Version": GH_API_VERSION,
+		Authorization: `Bearer ${token}`,
+		"User-Agent": "headlesscode-watcher",
+	}
+
+	let response: Response
+	try {
+		response = await fetchImpl(url, {
+			method,
+			headers,
+			...(body === undefined ? {} : { body: JSON.stringify(body) }),
+		})
+	} catch (err) {
+		throw new Error(
+			`GitHub API network error (${method} ${url}): ${err instanceof Error ? err.message : String(err)}`,
+			{ cause: err },
+		)
+	}
+
+	if (!response.ok) {
+		let message = `GitHub API HTTP ${response.status} for ${method} ${url}`
+		try {
+			const payload = (await response.json()) as { message?: unknown }
+			if (typeof payload?.message === "string" && payload.message !== "") {
+				message = `${payload.message} (HTTP ${response.status})`
+			}
+		} catch {
+			// body is not JSON — keep the generic message
+		}
+		throw new GhApiError(response.status, message, url)
+	}
+
+	// 204/empty bodies (rare for GETs; addLabel returns 200 with a body).
+	const text = await response.text()
+	if (text.trim() === "") {
+		return undefined
+	}
+	return JSON.parse(text)
+}
+
+export interface ListIssuesOptions {
+	owner: string
+	repo: string
+	/** Filter by label name (e.g. "needs-agent"). */
+	label?: string
+	/** Issue state filter (default "open"). */
+	state?: string
+	token: string
+	baseUrl?: string
+	fetchImpl?: typeof fetch
+	/** Hard cap on total issues (default 1000) to bound pagination runtime. */
+	maxTotal?: number
+}
+
+/** Normalize one raw GitHub issue payload; undefined for PRs / garbage. */
+export function toIssue(raw: unknown): GitHubIssue | undefined {
+	if (raw === null || typeof raw !== "object") {
+		return undefined
+	}
+	const r = raw as {
+		number?: unknown
+		title?: unknown
+		body?: unknown
+		labels?: unknown
+		updated_at?: unknown
+		html_url?: unknown
+		pull_request?: unknown
+	}
+	// The /issues endpoint returns pull requests too — filter them out.
+	if (r.pull_request !== undefined) {
+		return undefined
+	}
+	const number = Number(r.number)
+	if (!Number.isInteger(number) || number <= 0) {
+		return undefined
+	}
+	const labels = Array.isArray(r.labels)
+		? (r.labels as unknown[])
+				.filter(
+					(l): l is { name: string } =>
+						l !== null && typeof l === "object" && typeof (l as { name?: unknown }).name === "string",
+				)
+				.map((l) => l.name)
+		: []
+	return {
+		number,
+		title: typeof r.title === "string" ? r.title : String(r.title ?? ""),
+		...(typeof r.body === "string" ? { body: r.body } : {}),
+		labels,
+		...(typeof r.updated_at === "string" ? { updated_at: r.updated_at } : {}),
+		...(typeof r.html_url === "string" ? { html_url: r.html_url } : {}),
+	}
+}
+
+/**
+ * List issues filtered by label (plus state), PRs filtered out, paginated via
+ * a simple per_page/page loop (per_page=100 until a short page), capped at
+ * `maxTotal` to bound runtime.
+ */
+export async function listIssues(options: ListIssuesOptions): Promise<GitHubIssue[]> {
+	const { owner, repo, label, state = "open", token, baseUrl, fetchImpl, maxTotal = DEFAULT_MAX_ISSUES } = options
+
+	const issues: GitHubIssue[] = []
+	let page = 1
+
+	while (issues.length < maxTotal) {
+		const url = buildGhUrl(
+			baseUrl ?? process.env.GITHUB_API_BASE_URL ?? DEFAULT_GH_BASE_URL,
+			`/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/issues`,
+			{
+				per_page: ISSUES_PAGE_SIZE,
+				page,
+				state,
+				labels: label,
+			},
+		)
+		const raw = await ghApi(url, { token, baseUrl, fetchImpl })
+		const batch = Array.isArray(raw) ? raw : []
+		for (const entry of batch) {
+			const issue = toIssue(entry)
+			if (issue) {
+				issues.push(issue)
+				if (issues.length >= maxTotal) {
+					break
+				}
+			}
+		}
+		// A short page (or an empty one) means we've reached the end.
+		if (batch.length < ISSUES_PAGE_SIZE) {
+			break
+		}
+		page++
+	}
+
+	return issues
+}
+
+/**
+ * Fetch one issue by number. Used when an issue was only referenced by
+ * number (e.g. a webhook payload or a plain-number trigger).
+ */
+export async function getIssue(
+	owner: string,
+	repo: string,
+	number: number,
+	options: GhRequestOptions,
+): Promise<GitHubIssue> {
+	const raw = await ghApi(`/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/issues/${number}`, options)
+	const issue = toIssue(raw)
+	if (!issue) {
+		throw new Error(`GitHub API: issue #${number} in ${owner}/${repo} is not a usable issue (or is a PR)`)
+	}
+	return issue
+}
+
+/**
+ * Add a label to an issue (POST /repos/{o}/{r}/issues/{n}/labels). Used by
+ * the documented label-marking idempotency strategy — the default watcher
+ * relies on the state file and does not call this.
+ */
+export async function addLabel(
+	owner: string,
+	repo: string,
+	number: number,
+	label: string,
+	options: GhRequestOptions,
+): Promise<void> {
+	await ghApi(`/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/issues/${number}/labels`, {
+		...options,
+		method: "POST",
+		body: { labels: [label] },
+	})
+}
+
+export interface GhClientOptions {
+	token: string
+	baseUrl?: string
+	fetchImpl?: typeof fetch
+}
+
+/** The client shape the watcher consumes (DI-friendly for tests). */
+export interface GhClient {
+	listIssues(options: { owner: string; repo: string; label?: string; state?: string }): Promise<GitHubIssue[]>
+	getIssue(owner: string, repo: string, number: number): Promise<GitHubIssue>
+	addLabel(owner: string, repo: string, number: number, label: string): Promise<void>
+}
+
+/** Build a GhClient bound to a token/base URL (or injectable fetch). */
+export function createGhClient(options: GhClientOptions): GhClient {
+	const { token, baseUrl, fetchImpl } = options
+	return {
+		listIssues: (o) => listIssues({ ...o, token, baseUrl, fetchImpl }),
+		getIssue: (owner, repo, number) => getIssue(owner, repo, number, { token, baseUrl, fetchImpl }),
+		addLabel: (owner, repo, number, label) => addLabel(owner, repo, number, label, { token, baseUrl, fetchImpl }),
+	}
+}