@markjaquith/agency 2.70.0 → 2.71.0

package/README.md

@@ -21,6 +21,21 @@ bun install -g @markjaquith/agency
 
 For development, run `bun link` from this repository.
 
+## Local Usage Logging
+
+Agency records privacy-safe CLI usage events locally so command journeys,
+failures, and flag adoption can be analyzed. Events are stored in SQLite at
+`$XDG_STATE_HOME/agency/usage.sqlite3` (or
+`~/.local/state/agency/usage.sqlite3`) and retained for 90 days by default.
+Each event contains the normalized command path, flag names, timing, outcome,
+Agency version, and ordered `AGENCY_SESSION_ID` correlation. Raw arguments,
+flag values, free-form input, and the current directory are never recorded.
+
+Export events as JSON Lines with `agency usage export`. Set
+`AGENCY_NO_USAGE_LOG=1` to opt out, `AGENCY_USAGE_RETENTION_DAYS` to change
+retention, or `AGENCY_USAGE_DB` to select a different database path. Logging is
+best effort and never changes command output or exit behavior.
+
 ## Core Model
 
 - A **workbase** is the root containing durable documents and local repository

package/cli-main.ts

@@ -67,6 +67,7 @@ import {
 	successEnvelope,
 	writeEnvelope,
 } from "./src/protocol"
+import { exportUsageEvents, recordUsageEvent } from "./src/usage-log"
 
 // Create CLI layer with all services
 const CliLayer = Layer.mergeAll(
@@ -176,6 +177,12 @@ const VERSION = packageJson.version
 
 // Define commands
 const commands: Record<string, Command> = {
+	usage: {
+		run: async () => {
+			for (const event of await exportUsageEvents())
+				console.log(JSON.stringify(event))
+		},
+	},
 	act: {
 		run: async (args: string[], options: Record<string, any>) => {
 			if (options.help) return console.log(actHelp)
@@ -785,6 +792,7 @@ Commands:
   init [path]            Initialize an Agency workbase
   workbase <subcommand>  Manage registered workbases
   integration <command> Inspect or sync managed integration files
+  usage export          Export local usage events as JSON Lines
   epic <subcommand>      Manage epics
   phase <subcommand>     Manage task phases
   claim <task> [phase]   Claim an execution unit
@@ -830,9 +838,30 @@ const machineMode = process.argv
 	.slice(2)
 	.some((argument) => argument === "--json" || argument === "--jsonl")
 
+const invocationStartedAt = performance.now()
+const rawArguments = process.argv.slice(2)
+let usageCommandPath = "invalid"
+let usageFlagNames = rawArguments
+	.filter((argument) => argument.startsWith("--"))
+	.map((argument) => argument.slice(2).split("=", 1)[0]!)
+
 try {
-	const args = process.argv.slice(2)
-	const { commandName, args: commandArgs, passthrough, values } = parseCli(args)
+	const {
+		commandName,
+		args: commandArgs,
+		passthrough,
+		values,
+	} = parseCli(rawArguments)
+	usageCommandPath =
+		[commandName, commandArgs[0]]
+			.filter(
+				(part): part is string =>
+					typeof part === "string" && part.length > 0 && !part.startsWith("-"),
+			)
+			.join("/") || "root"
+	usageFlagNames = Object.entries(values)
+		.filter(([, value]) => value !== undefined && value !== false)
+		.map(([name]) => name)
 
 	// Handle global flags
 	if (values.version) {
@@ -841,6 +870,16 @@ try {
 		} else {
 			console.log(`v${VERSION}`)
 		}
+		await recordUsageEvent(
+			{
+				commandPath: "version",
+				flagNames: usageFlagNames,
+				durationMs: performance.now() - invocationStartedAt,
+				outcome: "success",
+				exitStatus: 0,
+			},
+			VERSION,
+		)
 		process.exit(0)
 	}
 
@@ -848,7 +887,18 @@ try {
 	// Show help if no command
 	if (!commandName) {
 		showMainHelp()
-		process.exit(values.help ? 0 : 1)
+		const exitStatus = values.help ? 0 : 1
+		await recordUsageEvent(
+			{
+				commandPath: "help",
+				flagNames: usageFlagNames,
+				durationMs: performance.now() - invocationStartedAt,
+				outcome: exitStatus === 0 ? "success" : "failure",
+				exitStatus,
+			},
+			VERSION,
+		)
+		process.exit(exitStatus)
 	}
 
 	const command = commands[commandName]!
@@ -880,7 +930,28 @@ try {
 			passthrough,
 		})
 	}
+	const exitStatus = Number(process.exitCode ?? 0)
+	await recordUsageEvent(
+		{
+			commandPath: usageCommandPath,
+			flagNames: usageFlagNames,
+			durationMs: performance.now() - invocationStartedAt,
+			outcome: exitStatus === 0 ? "success" : "failure",
+			exitStatus,
+		},
+		VERSION,
+	)
 } catch (error) {
+	await recordUsageEvent(
+		{
+			commandPath: usageCommandPath,
+			flagNames: usageFlagNames,
+			durationMs: performance.now() - invocationStartedAt,
+			outcome: "failure",
+			exitStatus: 1,
+		},
+		VERSION,
+	)
 	if (machineMode) {
 		writeEnvelope(errorEnvelope(error))
 		process.exit(1)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@markjaquith/agency",
-  "version": "2.70.0",
+  "version": "2.71.0",
   "description": "Manage agentic work across repositories with durable workbases",
   "keywords": [
     "agents",

package/src/cli-parser.ts

@@ -136,6 +136,17 @@ const nonPrCompletionOptions = {
 } satisfies OptionConfig
 
 const commands = {
+	usage: {
+		usage: "agency usage export",
+		options: commonOptions,
+		subcommands: {
+			export: {
+				usage: "agency usage export",
+				minArgs: 0,
+				maxArgs: 0,
+			},
+		},
+	},
 	init: {
 		usage: "agency init [path] [--json]",
 		options: outputOptions,

package/src/cli.test.ts

@@ -36,6 +36,8 @@ async function runCli(
 		env: {
 			...process.env,
 			XDG_CONFIG_HOME: isolatedConfigHome,
+			XDG_STATE_HOME: isolatedConfigHome,
+			AGENCY_NO_USAGE_LOG: "1",
 			...env,
 		},
 		stdout: "pipe",
@@ -170,6 +172,44 @@ describe("CLI", () => {
 		expect(taggedError.stderr).not.toContain("An error has occurred")
 	})
 
+	test("records successful and failed invocations without argument values", async () => {
+		const state = await createTempDir()
+		tempDirs.push(state)
+		const env = {
+			XDG_STATE_HOME: state,
+			AGENCY_SESSION_ID: "cli-session",
+			AGENCY_NO_USAGE_LOG: "0",
+		}
+		expect((await runCli(["--version"], projectRoot, env)).exitCode).toBe(0)
+		expect(
+			(await runCli(["unknown", "--cwd", "/private/value"], projectRoot, env))
+				.exitCode,
+		).toBe(1)
+
+		const exported = await runCli(["usage", "export"], projectRoot, env)
+		expect(exported).toMatchObject({ exitCode: 0, stderr: "" })
+		const events = exported.stdout
+			.trim()
+			.split("\n")
+			.map((line) => JSON.parse(line))
+		expect(events).toEqual([
+			expect.objectContaining({
+				sessionId: "cli-session",
+				sessionSequence: 1,
+				commandPath: "version",
+				flagNames: ["version"],
+				outcome: "success",
+			}),
+			expect.objectContaining({
+				sessionSequence: 2,
+				commandPath: "invalid",
+				flagNames: ["cwd"],
+				outcome: "failure",
+			}),
+		])
+		expect(exported.stdout).not.toContain("/private/value")
+	})
+
 	test("coordinates claims through revision-guarded machine commands", async () => {
 		const root = await createTempDir()
 		tempDirs.push(root)

package/src/usage-log.test.ts

@@ -0,0 +1,94 @@
+import { afterEach, describe, expect, test } from "bun:test"
+import { rm } from "node:fs/promises"
+import { join } from "node:path"
+import { cleanupTempDir, createTempDir } from "./test-utils"
+import { exportUsageEvents, recordUsageEvent } from "./usage-log"
+
+describe("usage logging", () => {
+	const tempDirs: string[] = []
+
+	afterEach(() => Promise.all(tempDirs.splice(0).map(cleanupTempDir)))
+
+	test("stores versioned privacy-safe events in session order", async () => {
+		const state = await createTempDir()
+		tempDirs.push(state)
+		const env = {
+			XDG_STATE_HOME: state,
+			AGENCY_SESSION_ID: "session-1",
+		} as NodeJS.ProcessEnv
+		for (const commandPath of ["worktree/prepare", "context"]) {
+			await recordUsageEvent(
+				{
+					commandPath,
+					flagNames: ["json", "task", "json"],
+					durationMs: 12.4,
+					outcome: "success",
+					exitStatus: 0,
+				},
+				"1.2.3",
+				env,
+			)
+		}
+
+		expect(await Bun.file(join(state, "agency/usage.sqlite3")).exists()).toBe(
+			true,
+		)
+		expect(await exportUsageEvents(env)).toEqual([
+			expect.objectContaining({
+				version: 1,
+				sessionId: "session-1",
+				sessionSequence: 1,
+				agencyVersion: "1.2.3",
+				commandPath: "worktree/prepare",
+				flagNames: ["json", "task"],
+				durationMs: 12,
+				outcome: "success",
+				exitStatus: 0,
+			}),
+			expect.objectContaining({
+				sessionSequence: 2,
+				commandPath: "context",
+			}),
+		])
+	})
+
+	test("supports opt-out and ignores unavailable storage", async () => {
+		const state = await createTempDir()
+		tempDirs.push(state)
+		const disabled = {
+			XDG_STATE_HOME: state,
+			AGENCY_NO_USAGE_LOG: "true",
+		} as NodeJS.ProcessEnv
+		await recordUsageEvent(
+			{
+				commandPath: "status",
+				flagNames: [],
+				durationMs: 1,
+				outcome: "failure",
+				exitStatus: 1,
+			},
+			"1.2.3",
+			disabled,
+		)
+		expect(await Bun.file(join(state, "agency/usage.sqlite3")).exists()).toBe(
+			false,
+		)
+
+		const blocked = join(state, "blocked")
+		await Bun.write(blocked, "not a directory")
+		await expect(
+			recordUsageEvent(
+				{
+					commandPath: "status",
+					flagNames: [],
+					durationMs: 1,
+					outcome: "failure",
+					exitStatus: 1,
+				},
+				"1.2.3",
+				{ AGENCY_USAGE_DB: join(blocked, "usage.sqlite3") },
+			),
+		).resolves.toBeUndefined()
+		await rm(blocked)
+	})
+})

package/src/usage-log.ts

@@ -0,0 +1,143 @@
+import { Database } from "bun:sqlite"
+import { mkdir } from "node:fs/promises"
+import { dirname, join } from "node:path"
+
+const USAGE_EVENT_VERSION = 1 as const
+const DEFAULT_RETENTION_DAYS = 90
+
+export interface UsageEvent {
+	readonly commandPath: string
+	readonly flagNames: readonly string[]
+	readonly durationMs: number
+	readonly exitStatus: number
+	readonly outcome: "success" | "failure"
+}
+
+const stateDirectory = (env: NodeJS.ProcessEnv) =>
+	env.XDG_STATE_HOME ?? join(env.HOME ?? ".", ".local", "state")
+
+const usageDatabasePath = (env: NodeJS.ProcessEnv = process.env) =>
+	env.AGENCY_USAGE_DB ?? join(stateDirectory(env), "agency", "usage.sqlite3")
+
+const enabled = (env: NodeJS.ProcessEnv) =>
+	!["1", "true", "yes"].includes((env.AGENCY_NO_USAGE_LOG ?? "").toLowerCase())
+
+const retentionDays = (env: NodeJS.ProcessEnv) => {
+	const value = Number.parseInt(env.AGENCY_USAGE_RETENTION_DAYS ?? "", 10)
+	return Number.isSafeInteger(value) && value >= 0
+		? value
+		: DEFAULT_RETENTION_DAYS
+}
+
+const openDatabase = async (env: NodeJS.ProcessEnv) => {
+	const path = usageDatabasePath(env)
+	await mkdir(dirname(path), { recursive: true, mode: 0o700 })
+	const database = new Database(path, { create: true, strict: true })
+	database.run("PRAGMA journal_mode = WAL")
+	database.run("PRAGMA busy_timeout = 1000")
+	database.run(`
+		CREATE TABLE IF NOT EXISTS usage_events (
+			id INTEGER PRIMARY KEY AUTOINCREMENT,
+			event_version INTEGER NOT NULL,
+			session_id TEXT NOT NULL,
+			session_sequence INTEGER NOT NULL,
+			occurred_at TEXT NOT NULL,
+			agency_version TEXT NOT NULL,
+			command_path TEXT NOT NULL,
+			flag_names TEXT NOT NULL,
+			duration_ms INTEGER NOT NULL,
+			outcome TEXT NOT NULL,
+			exit_status INTEGER NOT NULL
+		)
+	`)
+	database.run(
+		"CREATE INDEX IF NOT EXISTS usage_events_session ON usage_events(session_id, session_sequence)",
+	)
+	database.run(
+		"CREATE INDEX IF NOT EXISTS usage_events_command ON usage_events(command_path, occurred_at)",
+	)
+	return database
+}
+
+export async function recordUsageEvent(
+	event: UsageEvent,
+	agencyVersion: string,
+	env: NodeJS.ProcessEnv = process.env,
+): Promise<void> {
+	if (!enabled(env)) return
+	let database: Database | undefined
+	try {
+		database = await openDatabase(env)
+		const sessionId = env.AGENCY_SESSION_ID || `process-${process.pid}`
+		database
+			.query(`
+					INSERT INTO usage_events (
+						event_version, session_id, session_sequence, occurred_at,
+						agency_version, command_path, flag_names, duration_ms,
+						outcome, exit_status
+					) VALUES (
+						?, ?,
+						(SELECT COALESCE(MAX(session_sequence), 0) + 1 FROM usage_events WHERE session_id = ?),
+						?, ?, ?, ?, ?, ?, ?
+					)
+				`)
+			.run(
+				USAGE_EVENT_VERSION,
+				sessionId,
+				sessionId,
+				new Date().toISOString(),
+				agencyVersion,
+				event.commandPath,
+				JSON.stringify([...new Set(event.flagNames)].sort()),
+				Math.max(0, Math.round(event.durationMs)),
+				event.outcome,
+				event.exitStatus,
+			)
+		if (Math.random() < 0.01) {
+			const days = retentionDays(env)
+			database
+				.query(
+					"DELETE FROM usage_events WHERE occurred_at < datetime('now', ?)",
+				)
+				.run(`-${days} days`)
+		}
+	} catch {
+		// Usage logging must never affect command behavior.
+	} finally {
+		database?.close()
+	}
+}
+
+export async function exportUsageEvents(
+	env: NodeJS.ProcessEnv = process.env,
+): Promise<readonly Record<string, unknown>[]> {
+	if (!enabled(env)) return []
+	let database: Database | undefined
+	try {
+		database = await openDatabase(env)
+		const rows = database
+			.query(`
+				SELECT event_version, session_id, session_sequence, occurred_at,
+					agency_version, command_path, flag_names, duration_ms,
+					outcome, exit_status
+				FROM usage_events ORDER BY occurred_at, id
+			`)
+			.all() as Record<string, string | number>[]
+		return rows.map((row) => ({
+			version: row.event_version,
+			sessionId: row.session_id,
+			sessionSequence: row.session_sequence,
+			occurredAt: row.occurred_at,
+			agencyVersion: row.agency_version,
+			commandPath: row.command_path,
+			flagNames: JSON.parse(String(row.flag_names)),
+			durationMs: row.duration_ms,
+			outcome: row.outcome,
+			exitStatus: row.exit_status,
+		}))
+	} catch {
+		return []
+	} finally {
+		database?.close()
+	}
+}