@markjaquith/agency 2.56.0 → 2.57.0

package/src/services/SyncService.test.ts

@@ -59,6 +59,7 @@ describe("SyncService", () => {
 		await Bun.write(
 			gh,
 			`#!/bin/sh
+if [ -n "$GH_CAPTURE" ]; then printf '%s\n' "$@" "GIT_DIR=$GIT_DIR" > "$GH_CAPTURE"; fi
 case "$*" in
 *mergeable*) ;;
 *) echo "mergeable field was not requested" >&2; exit 2 ;;
@@ -82,6 +83,7 @@ JSON
 	afterEach(async () => {
 		if (originalPath === undefined) delete process.env.PATH
 		else process.env.PATH = originalPath
+		delete process.env.GH_CAPTURE
 		await cleanupTempDir(root)
 	})
 
@@ -125,8 +127,14 @@ pr: null
 		if (!Bun.which("jj")) return
 		const repository = join(root, "repos/agency")
 		await rm(repository, { recursive: true, force: true })
-		await git(["clone", join(root, "source"), repository])
-		await jj(["git", "init", "--colocate", repository])
+		await jj([
+			"git",
+			"clone",
+			"--no-colocate",
+			join(root, "source"),
+			repository,
+		])
+		expect(await Bun.file(join(repository, ".git")).exists()).toBe(false)
 		await Bun.write(
 			join(root, "agency.json"),
 			JSON.stringify({ version: 2, vcs: "jj" }),
@@ -147,6 +155,8 @@ pr: null
 				),
 			),
 		)
+		const capture = join(root, "gh-capture")
+		process.env.GH_CAPTURE = capture
 
 		const planned = await runTestEffect(
 			SyncService.pipe(
@@ -160,6 +170,10 @@ pr: null
 				status: "planned",
 			}),
 		)
+		const invocation = await Bun.file(capture).text()
+		expect(invocation).toContain("--repo\n")
+		expect(invocation).toContain("GIT_DIR=")
+		expect(invocation).toContain(".jj")
 
 		const applied = await runTestEffect(
 			SyncService.pipe(
@@ -174,6 +188,7 @@ pr: null
 			branch: "task/jj-sync",
 			dirty: false,
 		})
+		delete process.env.GH_CAPTURE
 	})
 
 	test("observes drift without mutation and applies only safe transitions", async () => {
@@ -796,4 +811,119 @@ process.stdout.write(${JSON.stringify(JSON.stringify(record))})
 			),
 		).toMatchObject({ valid: true, issues: [] })
 	})
+
+	test("reads each repository workspace inventory once", async () => {
+		for (const id of ["first", "second"]) {
+			await runTestEffect(
+				TaskService.pipe(
+					Effect.flatMap((service) =>
+						service.create(
+							{
+								id,
+								ticketUrl: null,
+								repo: "agency",
+								branch: `feat/${id}`,
+								base: "main",
+							},
+							root,
+						),
+					),
+				),
+			)
+			await runTestEffect(
+				WorktreeService.pipe(
+					Effect.flatMap((service) => service.materialize(id, undefined, root)),
+				),
+			)
+			await runTestEffect(
+				TaskService.pipe(
+					Effect.flatMap((service) =>
+						service.setStatus(id, "done", root, {
+							summary: `Completed ${id}`,
+						}),
+					),
+				),
+			)
+		}
+
+		const callsPath = join(root, "workspace-list-calls")
+		const realGit = Bun.which("git")!
+		const gitWrapper = join(root, "bin", "git")
+		await Bun.write(
+			gitWrapper,
+			`#!/bin/sh
+case "$*" in
+*"worktree list --porcelain -z"*) printf 'call\\n' >> ${JSON.stringify(callsPath)} ;;
+esac
+exec ${JSON.stringify(realGit)} "$@"
+`,
+		)
+		await chmod(gitWrapper, 0o755)
+
+		await runTestEffect(
+			SyncService.pipe(
+				Effect.flatMap((service) => service.reconcile({ cwd: root })),
+			),
+		)
+		expect((await Bun.file(callsPath).text()).trim().split("\n")).toHaveLength(
+			1,
+		)
+	})
+
+	test("queries pull request providers concurrently", async () => {
+		for (const id of ["first", "second", "third"]) {
+			await runTestEffect(
+				TaskService.pipe(
+					Effect.flatMap((service) =>
+						service.create(
+							{
+								id,
+								ticketUrl: null,
+								repo: "agency",
+								branch: `feat/${id}`,
+								base: "main",
+							},
+							root,
+						),
+					),
+				),
+			)
+		}
+
+		const barrier = join(root, "query-barrier")
+		await mkdir(barrier)
+		await Bun.write(
+			join(root, "bin", "gh"),
+			`#!/bin/sh
+branch=""
+while [ "$#" -gt 0 ]; do
+  if [ "$1" = "--head" ]; then branch="$2"; break; fi
+  shift
+done
+id="\${branch##*/}"
+touch ${JSON.stringify(barrier)}/"\${id}"
+attempt=0
+while [ "$attempt" -lt 200 ]; do
+  set -- ${JSON.stringify(barrier)}/*
+  if [ -e "$1" ] && [ "$#" -ge 3 ]; then printf '[]\\n'; exit 0; fi
+  attempt=$((attempt + 1))
+  sleep 0.01
+done
+echo "provider queries were serialized" >&2
+exit 9
+`,
+		)
+		await chmod(join(root, "bin", "gh"), 0o755)
+
+		const result = await runTestEffect(
+			SyncService.pipe(
+				Effect.flatMap((service) => service.reconcile({ cwd: root })),
+			),
+		)
+		expect(
+			result.warnings.filter(
+				(warning) => warning.kind === "pr-discovery-unavailable",
+			),
+		).toEqual([])
+	})
 })

package/src/services/SyncService.ts

@@ -48,6 +48,7 @@ interface RegisteredWorktree {
 	readonly path: string
 	readonly head: string | null
 	readonly branch: string | null
+	readonly dirty?: boolean
 }
 
 interface SyncChange {
@@ -106,6 +107,13 @@ interface SyncResult {
 	readonly repositories: RepositorySetupResult
 }
 
+export interface SyncProgress {
+	readonly stage: "repositories" | "pull-requests" | "executions"
+	readonly current: number
+	readonly total: number
+	readonly target?: string
+}
+
 const parseWorktrees = (output: string): RegisteredWorktree[] => {
 	const worktrees: RegisteredWorktree[] = []
 	let current: RegisteredWorktree | undefined
@@ -152,6 +160,7 @@ export class SyncService extends Effect.Service<SyncService>()("SyncService", {
 				readonly cwd?: string
 				readonly apply?: boolean
 				readonly now?: Date
+				readonly onProgress?: (progress: SyncProgress) => void
 			} = {},
 		) =>
 			Effect.gen(function* () {
@@ -177,6 +186,11 @@ export class SyncService extends Effect.Service<SyncService>()("SyncService", {
 					cwd: root,
 					apply: options.apply === true,
 				})
+				options.onProgress?.({
+					stage: "repositories",
+					current: repositorySetup.repositories.length,
+					total: repositorySetup.repositories.length,
+				})
 
 				const apply = options.apply === true
 				const now = options.now ?? new Date()
@@ -192,6 +206,10 @@ export class SyncService extends Effect.Service<SyncService>()("SyncService", {
 					})
 				}
 				const executions: ExecutionSyncState[] = []
+				const registeredByRepository = new Map<
+					string,
+					RegisteredWorktree[] | null
+				>()
 				const runExternal = (
 					args: readonly string[],
 					commandOptions?: {
@@ -214,8 +232,9 @@ export class SyncService extends Effect.Service<SyncService>()("SyncService", {
 								}),
 							),
 						)
+				const taskRecords = yield* tasks.list(root)
 				const records: ExecutionRecord[] = []
-				for (const task of yield* tasks.list(root)) {
+				for (const task of taskRecords) {
 					if ("phases" in task.data) {
 						for (const phase of yield* phases.list(task.id, root)) {
 							records.push({
@@ -237,6 +256,164 @@ export class SyncService extends Effect.Service<SyncService>()("SyncService", {
 						})
 					}
 				}
+				const listRegistered = (repositoryPath: string) =>
+					Effect.gen(function* () {
+						if (registeredByRepository.has(repositoryPath))
+							return registeredByRepository.get(repositoryPath)!
+						const listed = yield* Effect.either(
+							backend.listWorkspaces(repositoryPath),
+						)
+						if (Either.isLeft(listed)) {
+							registeredByRepository.set(repositoryPath, null)
+							return null
+						}
+						const registered: RegisteredWorktree[] = []
+						for (const item of listed.right) {
+							registered.push({
+								head: backend.kind === "jj" ? (item.head ?? null) : item.commit,
+								branch: item.branch,
+								path: (yield* fs.exists(item.path))
+									? yield* fs.realPath(item.path)
+									: resolve(item.path),
+								...(item.dirty === undefined ? {} : { dirty: item.dirty }),
+							})
+						}
+						registeredByRepository.set(repositoryPath, registered)
+						return registered
+					})
+				const queryRecords = records.filter((record) => !record.data.completion)
+				let queriedPullRequests = 0
+				const remoteName = config.delivery?.remote ?? "origin"
+				const repositoryPaths = [
+					...new Set(
+						queryRecords.map((record) => join(root, "repos", record.data.repo)),
+					),
+				]
+				const remoteUrls = new Map(
+					yield* Effect.forEach(
+						repositoryPaths,
+						(repositoryPath) =>
+							backend
+								.remoteUrl(repositoryPath, remoteName)
+								.pipe(
+									Effect.map(
+										(remoteUrl) => [repositoryPath, remoteUrl] as const,
+									),
+								),
+						{ concurrency: 8 },
+					),
+				)
+				const prQueries = new Map(
+					yield* Effect.forEach(
+						queryRecords,
+						(record) =>
+							Effect.gen(function* () {
+								const data = record.data
+								const repositoryPath = join(root, "repos", data.repo)
+								const remoteUrl = remoteUrls.get(repositoryPath) ?? null
+								const remoteRepository = (remoteUrl ?? "")
+									.trim()
+									.replace(/^[a-z][a-z0-9+.-]*:\/\/(?:[^@/]+@)?[^/]+\//i, "")
+									.replace(/^[^:]+:/, "")
+									.replace(/\.git\/?$/, "")
+									.replace(/\/$/, "")
+								const existing = data.pr
+									? normalizePullRequestRecord(data.pr)
+									: null
+								let result
+								if (config.delivery && remoteUrl) {
+									const resolved = resolveDeliveryCommand(
+										config.delivery,
+										"query",
+										{
+											repository: remoteRepository,
+											branch: data.branch,
+											base: data.base,
+											draft: existing ? String(existing.draft) : "",
+											url: existing?.url ?? "",
+											identifier: existing?.identifier ?? "",
+										},
+									)
+									result = yield* runExternal(resolved.argv, {
+										cwd: repositoryPath,
+										env: resolved.environment,
+									})
+								} else if (!config.delivery && existing) {
+									const environment = yield* Effect.either(
+										backend.gitEnvironment(repositoryPath),
+									)
+									result = yield* runExternal(
+										[
+											"gh",
+											"pr",
+											"view",
+											existing.url,
+											"--json",
+											"number,state,title,isDraft,headRefName,baseRefName,url,mergedAt,mergeCommit,mergeable",
+										],
+										{
+											cwd: repositoryPath,
+											env: Either.isRight(environment) ? environment.right : {},
+										},
+									)
+								} else if (!config.delivery) {
+									const environment = yield* Effect.either(
+										backend.gitEnvironment(repositoryPath),
+									)
+									result = yield* runExternal(
+										[
+											"gh",
+											"pr",
+											"list",
+											"--repo",
+											remoteRepository,
+											"--head",
+											data.branch,
+											"--state",
+											"all",
+											"--json",
+											"number,state,title,isDraft,headRefName,baseRefName,url,mergedAt,mergeCommit,mergeable",
+										],
+										{
+											cwd: repositoryPath,
+											env: Either.isRight(environment) ? environment.right : {},
+										},
+									)
+								}
+								return [
+									record.key,
+									{ remoteUrl, remoteRepository, result },
+								] as const
+							}).pipe(
+								Effect.tap(() =>
+									Effect.sync(() => {
+										queriedPullRequests += 1
+										options.onProgress?.({
+											stage: "pull-requests",
+											current: queriedPullRequests,
+											total: queryRecords.length,
+											target: record.key,
+										})
+									}),
+								),
+							),
+						{ concurrency: 8 },
+					),
+				)
+				const reviewRecords = taskRecords.filter(
+					(task) => "review" in task.data,
+				)
+				const executionTotal = records.length + reviewRecords.length
+				let reconciledExecutions = 0
+				const reportExecution = (target: string) => {
+					reconciledExecutions += 1
+					options.onProgress?.({
+						stage: "executions",
+						current: reconciledExecutions,
+						total: executionTotal,
+						target,
+					})
+				}
 
 				for (const record of records.sort((a, b) =>
 					a.key.localeCompare(b.key),
@@ -271,10 +448,8 @@ export class SyncService extends Effect.Service<SyncService>()("SyncService", {
 							workspaceConflict = true
 							continue
 						}
-						const listed = yield* Effect.either(
-							backend.listWorkspaces(repositoryPath),
-						)
-						if (Either.isLeft(listed)) {
+						const registered = yield* listRegistered(repositoryPath)
+						if (registered === null) {
 							unresolved.push({
 								kind: "worktree-inspection-failed",
 								target: record.key,
@@ -284,26 +459,21 @@ export class SyncService extends Effect.Service<SyncService>()("SyncService", {
 							continue
 						}
 						const exists = yield* fs.isDirectory(checkoutPath)
-						const registered: RegisteredWorktree[] = []
-						for (const item of listed.right) {
-							registered.push({
-								head: item.commit,
-								branch:
-									backend.kind === "jj" && "branch" in checkout
-										? checkout.branch
-										: item.branch,
-								path: (yield* fs.exists(item.path))
-									? yield* fs.realPath(item.path)
-									: resolve(item.path),
-							})
-						}
 						const expectedPath = exists
 							? yield* fs.realPath(checkoutPath)
 							: (yield* fs.isDirectory(codePath))
 								? join(yield* fs.realPath(codePath), checkout.repo)
 								: resolve(checkoutPath)
 						let atPath = registered.find((item) => item.path === expectedPath)
-						if (backend.kind === "jj" && atPath && exists) {
+						if (backend.kind === "jj" && atPath && "branch" in checkout) {
+							atPath = { ...atPath, branch: checkout.branch }
+						}
+						if (
+							backend.kind === "jj" &&
+							atPath &&
+							exists &&
+							atPath.head === null
+						) {
 							atPath = {
 								...atPath,
 								head: yield* backend.workspaceHead(checkoutPath),
@@ -378,14 +548,18 @@ export class SyncService extends Effect.Service<SyncService>()("SyncService", {
 						let resolvedCommit: string | null = null
 						if ("ref" in checkout) {
 							if (!isCommitId(checkout.ref)) {
-								const remoteRef = yield* runExternal([
-									"git",
-									"-C",
+								const remote = yield* backend.remoteUrl(
 									repositoryPath,
-									"ls-remote",
 									"origin",
-									originRef(checkout.ref),
-								])
+								)
+								const remoteRef = remote
+									? yield* runExternal([
+											"git",
+											"ls-remote",
+											remote,
+											originRef(checkout.ref),
+										])
+									: { exitCode: -1, stdout: "", stderr: "origin unavailable" }
 								resolvedCommit =
 									remoteRef.stdout.match(/^([0-9a-f]{40,64})\s/m)?.[1] ?? null
 								if (!resolvedCommit) {
@@ -415,7 +589,8 @@ export class SyncService extends Effect.Service<SyncService>()("SyncService", {
 
 						const dirty =
 							exists && atPath
-								? yield* backend.workspaceDirty(checkoutPath)
+								? (atPath.dirty ??
+									(yield* backend.workspaceDirty(checkoutPath)))
 								: null
 						if (exists && atPath && dirty === null) {
 							warnings.push({
@@ -487,6 +662,7 @@ export class SyncService extends Effect.Service<SyncService>()("SyncService", {
 								{ silent: true },
 							)
 							for (const checkout of workspace.checkouts) {
+								const repositoryPath = join(root, "repos", checkout.repo)
 								const index = checkoutStates.findIndex(
 									(item) => item.repo === checkout.repo,
 								)
@@ -504,6 +680,20 @@ export class SyncService extends Effect.Service<SyncService>()("SyncService", {
 												: null,
 										dirty: false,
 									}
+									const cached = registeredByRepository.get(repositoryPath)
+									if (cached) {
+										cached.push({
+											path: yield* fs.realPath(checkout.path),
+											head: checkout.resolvedCommit,
+											branch:
+												checkout.kind === "writable"
+													? backend.kind === "jj"
+														? checkout.requestedRef
+														: `refs/heads/${checkout.requestedRef}`
+													: null,
+											dirty: false,
+										})
+									}
 								}
 							}
 						}
@@ -569,6 +759,7 @@ export class SyncService extends Effect.Service<SyncService>()("SyncService", {
 							checkouts: checkoutStates,
 							pr: { url: null, state: "none" },
 						})
+						reportExecution(record.key)
 						continue
 					}
 
@@ -579,15 +770,8 @@ export class SyncService extends Effect.Service<SyncService>()("SyncService", {
 						state: "none",
 					}
 					let prConflict = false
-					const repositoryPath = join(root, "repos", data.repo)
-					const remoteName = config.delivery?.remote ?? "origin"
-					const remoteUrl = yield* backend.remoteUrl(repositoryPath, remoteName)
-					const remoteRepository = (remoteUrl ?? "")
-						.trim()
-						.replace(/^[a-z][a-z0-9+.-]*:\/\/(?:[^@/]+@)?[^/]+\//i, "")
-						.replace(/^[^:]+:/, "")
-						.replace(/\.git\/?$/, "")
-						.replace(/\/$/, "")
+					const query = prQueries.get(record.key)!
+					const { remoteUrl, remoteRepository } = query
 
 					if (
 						existing &&
@@ -609,18 +793,7 @@ export class SyncService extends Effect.Service<SyncService>()("SyncService", {
 							message: `Could not inspect delivery remote '${remoteName}'`,
 						})
 					} else if (config.delivery) {
-						const resolved = resolveDeliveryCommand(config.delivery, "query", {
-							repository: remoteRepository,
-							branch: data.branch,
-							base: data.base,
-							draft: existing ? String(existing.draft) : "",
-							url: existing?.url ?? "",
-							identifier: existing?.identifier ?? "",
-						})
-						const queried = yield* runExternal(resolved.argv, {
-							cwd: repositoryPath,
-							env: resolved.environment,
-						})
+						const queried = query.result!
 						if (queried.exitCode === 0) {
 							const parsed = yield* Effect.try({
 								try: () => parseOptionalPullRequestRecord(queried.stdout),
@@ -668,14 +841,7 @@ export class SyncService extends Effect.Service<SyncService>()("SyncService", {
 							})
 						}
 					} else if (existing) {
-						const viewed = yield* runExternal([
-							"gh",
-							"pr",
-							"view",
-							existing.url,
-							"--json",
-							"number,state,title,isDraft,headRefName,baseRefName,url,mergedAt,mergeCommit,mergeable",
-						])
+						const viewed = query.result!
 						if (viewed.exitCode === 0) {
 							const detail = parseJson<Record<string, unknown>>(
 								viewed.stdout,
@@ -704,20 +870,7 @@ export class SyncService extends Effect.Service<SyncService>()("SyncService", {
 							})
 						}
 					} else {
-						const listed = yield* runExternal(
-							[
-								"gh",
-								"pr",
-								"list",
-								"--head",
-								data.branch,
-								"--state",
-								"all",
-								"--json",
-								"number,state,title,isDraft,headRefName,baseRefName,url,mergedAt,mergeCommit,mergeable",
-							],
-							{ cwd: repositoryPath },
-						)
+						const listed = query.result!
 						if (listed.exitCode === 0) {
 							const matches = parseJson<Record<string, unknown>[]>(
 								listed.stdout,
@@ -816,11 +969,10 @@ export class SyncService extends Effect.Service<SyncService>()("SyncService", {
 						checkouts: checkoutStates,
 						pr,
 					})
+					reportExecution(record.key)
 				}
 
-				for (const task of (yield* tasks.list(root)).filter(
-					(task) => "review" in task.data,
-				)) {
+				for (const task of reviewRecords) {
 					if (!("review" in task.data)) continue
 					let data = task.data
 					let revision = task.revision
@@ -867,12 +1019,14 @@ export class SyncService extends Effect.Service<SyncService>()("SyncService", {
 						})
 					}
 					const repositoryPath = join(root, "repos", data.review.repo)
+					const reviewRemote = yield* backend.remoteUrl(
+						repositoryPath,
+						"origin",
+					)
 					const source = yield* runExternal([
 						"git",
-						"-C",
-						repositoryPath,
 						"ls-remote",
-						"origin",
+						reviewRemote ?? "origin",
 						data.review.source.kind === "pull-request"
 							? data.review.source.fetchRef
 							: originRef(data.review.source.ref),
@@ -915,6 +1069,7 @@ export class SyncService extends Effect.Service<SyncService>()("SyncService", {
 							sourceAvailable: sourceCommit !== null,
 						},
 					})
+					reportExecution(`task:${task.id}`)
 				}
 
 				return {

package/src/services/TaskService.ts

@@ -3,6 +3,7 @@ import { Data, Effect, Either } from "effect"
 import { join } from "node:path"
 import { FileSystemService } from "./FileSystemService"
 import { WorkbaseService } from "./WorkbaseService"
+import { VersionControlService } from "./VersionControlService"
 import { EpicService, type EpicRecord } from "./EpicService"
 import {
 	EntityId,
@@ -88,11 +89,16 @@ const reviewPinStep = (
 	root: string,
 	taskId: string,
 	review: ReviewRecord,
+	environment: Record<string, string>,
 ): TransactionStep => {
 	const repositoryPath = join(root, "repos", review.repo)
 	const ref = reviewPinRef(taskId)
 	const run = async (args: readonly string[]) => {
-		const child = Bun.spawn([...args], { stdout: "pipe", stderr: "pipe" })
+		const child = Bun.spawn([...args], {
+			stdout: "pipe",
+			stderr: "pipe",
+			env: { ...process.env, ...environment },
+		})
 		const [exitCode, stderr] = await Promise.all([
 			child.exited,
 			new Response(child.stderr).text(),
@@ -132,6 +138,7 @@ export class TaskService extends Effect.Service<TaskService>()("TaskService", {
 				const fs = yield* FileSystemService
 				const workbase = yield* WorkbaseService
 				const epics = yield* EpicService
+				const versionControl = yield* VersionControlService
 				const root = yield* workbase.discover(startPath)
 				const id = yield* decodeId(input.id)
 				const directory = join(root, "tasks", id)
@@ -257,13 +264,22 @@ export class TaskService extends Effect.Service<TaskService>()("TaskService", {
 					const updated = formatMarkdownDocument(epicData, parsed.body)
 					writes.push({ path: parentEpic.path, content: updated })
 				}
+				let reviewEnvironment: Record<string, string> = {}
+				if (input.review) {
+					const backend = yield* versionControl.forWorkbase(root)
+					reviewEnvironment = yield* backend.gitEnvironment(
+						join(root, "repos", input.review.repo),
+					)
+				}
 				yield* runLifecycleTransaction({
 					root,
 					preconditions: parentEpic
 						? [{ path: parentEpic.path, revision: parentEpic.revision }]
 						: [],
 					steps: [
-						...(input.review ? [reviewPinStep(root, id, input.review)] : []),
+						...(input.review
+							? [reviewPinStep(root, id, input.review, reviewEnvironment)]
+							: []),
 						documentWriteStep(root, writes),
 					],
 				})