@markjaquith/agency 3.2.3 → 3.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -4
- package/cli-main.ts +20 -11
- package/package.json +1 -1
- package/src/cli-parser.test.ts +21 -0
- package/src/cli-parser.ts +35 -1
- package/src/cli.test.ts +48 -6
- package/src/services/IntegrationService.test.ts +3 -0
- package/src/usage-log.test.ts +89 -5
- package/src/usage-log.ts +139 -31
- package/src/workbase/agent-command.test.ts +1 -0
- package/src/workbase/agent-command.ts +1 -0
- package/src/workbase/opencode-plugin-file.ts +1 -0
package/README.md
CHANGED
|
@@ -26,14 +26,20 @@ Agency records privacy-safe CLI usage events locally so command journeys,
|
|
|
26
26
|
failures, and flag adoption can be analyzed. Events are stored in SQLite at
|
|
27
27
|
`$XDG_STATE_HOME/agency/usage.sqlite3` (or
|
|
28
28
|
`~/.local/state/agency/usage.sqlite3`) and retained for 90 days by default.
|
|
29
|
-
Each event contains the
|
|
30
|
-
Agency version,
|
|
29
|
+
Each event contains the parser-derived command and subcommand path, flag names,
|
|
30
|
+
timing, a bounded outcome code, Agency version, invocation source, explicit test
|
|
31
|
+
attribution, and ordered journey correlation. Journey IDs are one-way hashes of
|
|
32
|
+
`AGENCY_SESSION_ID`; raw session IDs, positional arguments, entity IDs, paths,
|
|
31
33
|
flag values, free-form input, and the current directory are never recorded.
|
|
32
34
|
|
|
33
35
|
Export events as JSON Lines with `agency usage export`. Set
|
|
34
36
|
`AGENCY_NO_USAGE_LOG=1` to opt out, `AGENCY_USAGE_RETENTION_DAYS` to change
|
|
35
|
-
retention, or `AGENCY_USAGE_DB` to select a different database path.
|
|
36
|
-
|
|
37
|
+
retention, or `AGENCY_USAGE_DB` to select a different database path. Expired
|
|
38
|
+
events are pruned on every read and write. Set `AGENCY_INVOCATION_SOURCE` to one
|
|
39
|
+
of `human`, `agent`, or `automation`, and set `AGENCY_USAGE_TEST=1` for explicit
|
|
40
|
+
test attribution. Logging is best effort and never changes command output or
|
|
41
|
+
exit behavior. Databases created by versions before this privacy boundary are
|
|
42
|
+
cleared because their command paths may contain positional values.
|
|
37
43
|
|
|
38
44
|
## Core Model
|
|
39
45
|
|
package/cli-main.ts
CHANGED
|
@@ -57,7 +57,11 @@ import {
|
|
|
57
57
|
successEnvelope,
|
|
58
58
|
writeEnvelope,
|
|
59
59
|
} from "./src/protocol"
|
|
60
|
-
import {
|
|
60
|
+
import {
|
|
61
|
+
exportUsageEvents,
|
|
62
|
+
recordUsageEvent,
|
|
63
|
+
usageOutcomeCode,
|
|
64
|
+
} from "./src/usage-log"
|
|
61
65
|
|
|
62
66
|
// Create CLI layer with all services
|
|
63
67
|
const CliLayer = Layer.mergeAll(
|
|
@@ -752,9 +756,7 @@ const machineMode = process.argv
|
|
|
752
756
|
const invocationStartedAt = performance.now()
|
|
753
757
|
const rawArguments = process.argv.slice(2)
|
|
754
758
|
let usageCommandPath = "invalid"
|
|
755
|
-
let usageFlagNames =
|
|
756
|
-
.filter((argument) => argument.startsWith("--"))
|
|
757
|
-
.map((argument) => argument.slice(2).split("=", 1)[0]!)
|
|
759
|
+
let usageFlagNames: string[] = []
|
|
758
760
|
|
|
759
761
|
const pushUsageDetails = (error?: unknown) => {
|
|
760
762
|
if (usageCommandPath !== "push") return {}
|
|
@@ -795,17 +797,12 @@ const pushUsageDetails = (error?: unknown) => {
|
|
|
795
797
|
try {
|
|
796
798
|
const {
|
|
797
799
|
commandName,
|
|
800
|
+
commandPath,
|
|
798
801
|
args: commandArgs,
|
|
799
802
|
passthrough,
|
|
800
803
|
values,
|
|
801
804
|
} = parseCli(rawArguments)
|
|
802
|
-
usageCommandPath =
|
|
803
|
-
[commandName, commandArgs[0]]
|
|
804
|
-
.filter(
|
|
805
|
-
(part): part is string =>
|
|
806
|
-
typeof part === "string" && part.length > 0 && !part.startsWith("-"),
|
|
807
|
-
)
|
|
808
|
-
.join("/") || "root"
|
|
805
|
+
usageCommandPath = commandPath
|
|
809
806
|
usageFlagNames = Object.entries(values)
|
|
810
807
|
.filter(([, value]) => value !== undefined && value !== false)
|
|
811
808
|
.map(([name]) => name)
|
|
@@ -823,6 +820,7 @@ try {
|
|
|
823
820
|
flagNames: usageFlagNames,
|
|
824
821
|
durationMs: performance.now() - invocationStartedAt,
|
|
825
822
|
outcome: "success",
|
|
823
|
+
outcomeCode: "SUCCESS",
|
|
826
824
|
exitStatus: 0,
|
|
827
825
|
},
|
|
828
826
|
VERSION,
|
|
@@ -841,6 +839,7 @@ try {
|
|
|
841
839
|
flagNames: usageFlagNames,
|
|
842
840
|
durationMs: performance.now() - invocationStartedAt,
|
|
843
841
|
outcome: exitStatus === 0 ? "success" : "failure",
|
|
842
|
+
outcomeCode: exitStatus === 0 ? "SUCCESS" : "NONZERO_EXIT",
|
|
844
843
|
exitStatus,
|
|
845
844
|
},
|
|
846
845
|
VERSION,
|
|
@@ -884,18 +883,28 @@ try {
|
|
|
884
883
|
flagNames: usageFlagNames,
|
|
885
884
|
durationMs: performance.now() - invocationStartedAt,
|
|
886
885
|
outcome: exitStatus === 0 ? "success" : "failure",
|
|
886
|
+
outcomeCode: exitStatus === 0 ? "SUCCESS" : "NONZERO_EXIT",
|
|
887
887
|
exitStatus,
|
|
888
888
|
...pushUsageDetails(),
|
|
889
889
|
},
|
|
890
890
|
VERSION,
|
|
891
891
|
)
|
|
892
892
|
} catch (error) {
|
|
893
|
+
if (
|
|
894
|
+
typeof error === "object" &&
|
|
895
|
+
error !== null &&
|
|
896
|
+
"commandPath" in error &&
|
|
897
|
+
typeof error.commandPath === "string"
|
|
898
|
+
) {
|
|
899
|
+
usageCommandPath = error.commandPath
|
|
900
|
+
}
|
|
893
901
|
await recordUsageEvent(
|
|
894
902
|
{
|
|
895
903
|
commandPath: usageCommandPath,
|
|
896
904
|
flagNames: usageFlagNames,
|
|
897
905
|
durationMs: performance.now() - invocationStartedAt,
|
|
898
906
|
outcome: "failure",
|
|
907
|
+
outcomeCode: usageOutcomeCode(error),
|
|
899
908
|
exitStatus: 1,
|
|
900
909
|
...pushUsageDetails(error),
|
|
901
910
|
},
|
package/package.json
CHANGED
package/src/cli-parser.test.ts
CHANGED
|
@@ -7,6 +7,25 @@ const expectUsageError = (args: string[], usage: string) => {
|
|
|
7
7
|
}
|
|
8
8
|
|
|
9
9
|
describe("strict CLI parsing", () => {
|
|
10
|
+
test("returns canonical paths without positional values", () => {
|
|
11
|
+
expect(parseCli(["task", "show", "private-task-id"]).commandPath).toBe(
|
|
12
|
+
"task/show",
|
|
13
|
+
)
|
|
14
|
+
expect(parseCli(["validate", "/private/customer/path"]).commandPath).toBe(
|
|
15
|
+
"validate",
|
|
16
|
+
)
|
|
17
|
+
expect(parseCli(["work", "prepare", "private-task-id"]).commandPath).toBe(
|
|
18
|
+
"work/prepare",
|
|
19
|
+
)
|
|
20
|
+
expect(parseCli(["pr", "view", "private-task-id"]).commandPath).toBe("pr")
|
|
21
|
+
try {
|
|
22
|
+
parseCli(["task", "create", "private-task-id"])
|
|
23
|
+
expect.unreachable()
|
|
24
|
+
} catch (error) {
|
|
25
|
+
expect(error).toMatchObject({ commandPath: "task/create" })
|
|
26
|
+
}
|
|
27
|
+
})
|
|
28
|
+
|
|
10
29
|
test("parses act selectors, dry-run, and JSON options", () => {
|
|
11
30
|
expect(
|
|
12
31
|
parseCli([
|
|
@@ -363,6 +382,7 @@ describe("strict CLI parsing", () => {
|
|
|
363
382
|
const args = ["create", "--title", "two words", "--", "--literal"]
|
|
364
383
|
expect(parseCli(["--cwd", "/workbase", "pr", ...args])).toEqual({
|
|
365
384
|
commandName: "pr",
|
|
385
|
+
commandPath: "pr",
|
|
366
386
|
args,
|
|
367
387
|
passthrough: true,
|
|
368
388
|
values: { cwd: "/workbase" },
|
|
@@ -382,6 +402,7 @@ describe("strict CLI parsing", () => {
|
|
|
382
402
|
]),
|
|
383
403
|
).toEqual({
|
|
384
404
|
commandName: "pr",
|
|
405
|
+
commandPath: "pr/create",
|
|
385
406
|
args: ["create", "ship", "release"],
|
|
386
407
|
values: { draft: true, force: true, json: true },
|
|
387
408
|
})
|
package/src/cli-parser.ts
CHANGED
|
@@ -1032,6 +1032,7 @@ const preCommandValueOptions = new Set(["--workbase", "--cwd"])
|
|
|
1032
1032
|
|
|
1033
1033
|
export interface ParsedCli {
|
|
1034
1034
|
readonly commandName?: keyof typeof commands
|
|
1035
|
+
readonly commandPath: string
|
|
1035
1036
|
readonly args: string[]
|
|
1036
1037
|
readonly passthrough?: boolean
|
|
1037
1038
|
readonly values: Record<
|
|
@@ -1040,8 +1041,14 @@ export interface ParsedCli {
|
|
|
1040
1041
|
>
|
|
1041
1042
|
}
|
|
1042
1043
|
|
|
1044
|
+
const canonicalCommandPath = (
|
|
1045
|
+
commandName: string,
|
|
1046
|
+
subcommand: string | undefined,
|
|
1047
|
+
) => (subcommand ? `${commandName}/${subcommand}` : commandName)
|
|
1048
|
+
|
|
1043
1049
|
class CliUsageError extends Error {
|
|
1044
1050
|
readonly _tag = "CliUsageError"
|
|
1051
|
+
readonly commandPath: string
|
|
1045
1052
|
|
|
1046
1053
|
constructor(
|
|
1047
1054
|
readonly detail: string,
|
|
@@ -1049,6 +1056,15 @@ class CliUsageError extends Error {
|
|
|
1049
1056
|
) {
|
|
1050
1057
|
super(`${detail}\n\nUsage: ${usage}`)
|
|
1051
1058
|
this.name = "CliUsageError"
|
|
1059
|
+
const [, commandName, subcommand] = usage.split(/\s+/)
|
|
1060
|
+
const definition: CommandDefinition | undefined =
|
|
1061
|
+
commands[commandName as keyof typeof commands]
|
|
1062
|
+
this.commandPath = !definition
|
|
1063
|
+
? "invalid"
|
|
1064
|
+
: subcommand && definition.subcommands?.[subcommand]
|
|
1065
|
+
? `${commandName}/${subcommand}`
|
|
1066
|
+
: commandName!
|
|
1067
|
+
Object.defineProperty(this, "commandPath", { enumerable: false })
|
|
1052
1068
|
}
|
|
1053
1069
|
}
|
|
1054
1070
|
|
|
@@ -1323,7 +1339,7 @@ export function parseCli(args: readonly string[]): ParsedCli {
|
|
|
1323
1339
|
"agency <command> [options]",
|
|
1324
1340
|
)
|
|
1325
1341
|
}
|
|
1326
|
-
return { args: [], values: parsed.values }
|
|
1342
|
+
return { commandPath: "root", args: [], values: parsed.values }
|
|
1327
1343
|
}
|
|
1328
1344
|
|
|
1329
1345
|
const commandName = args[commandIndex]!
|
|
@@ -1368,6 +1384,7 @@ export function parseCli(args: readonly string[]): ParsedCli {
|
|
|
1368
1384
|
}
|
|
1369
1385
|
return {
|
|
1370
1386
|
commandName,
|
|
1387
|
+
commandPath: "pr",
|
|
1371
1388
|
args: prArgs,
|
|
1372
1389
|
passthrough: true,
|
|
1373
1390
|
values: parsed.values,
|
|
@@ -1391,6 +1408,7 @@ export function parseCli(args: readonly string[]): ParsedCli {
|
|
|
1391
1408
|
assertNoDuplicateOptions(parsed.tokens, new Set(), definition.usage)
|
|
1392
1409
|
return {
|
|
1393
1410
|
commandName: commandName as keyof typeof commands,
|
|
1411
|
+
commandPath: commandName,
|
|
1394
1412
|
args: parsed.positionals,
|
|
1395
1413
|
values: parsed.values,
|
|
1396
1414
|
}
|
|
@@ -1441,6 +1459,10 @@ export function parseCli(args: readonly string[]): ParsedCli {
|
|
|
1441
1459
|
if (parsed.values.version) {
|
|
1442
1460
|
return {
|
|
1443
1461
|
commandName: commandName as keyof typeof commands,
|
|
1462
|
+
commandPath: canonicalCommandPath(
|
|
1463
|
+
commandName,
|
|
1464
|
+
selectedSubcommand ? subcommand : undefined,
|
|
1465
|
+
),
|
|
1444
1466
|
args: parsed.positionals,
|
|
1445
1467
|
values: parsed.values,
|
|
1446
1468
|
}
|
|
@@ -1448,6 +1470,10 @@ export function parseCli(args: readonly string[]): ParsedCli {
|
|
|
1448
1470
|
if (parsed.values.help) {
|
|
1449
1471
|
return {
|
|
1450
1472
|
commandName: commandName as keyof typeof commands,
|
|
1473
|
+
commandPath: canonicalCommandPath(
|
|
1474
|
+
commandName,
|
|
1475
|
+
selectedSubcommand ? subcommand : undefined,
|
|
1476
|
+
),
|
|
1451
1477
|
args: parsed.positionals,
|
|
1452
1478
|
values: parsed.values,
|
|
1453
1479
|
}
|
|
@@ -1622,6 +1648,14 @@ export function parseCli(args: readonly string[]): ParsedCli {
|
|
|
1622
1648
|
|
|
1623
1649
|
return {
|
|
1624
1650
|
commandName: commandName as keyof typeof commands,
|
|
1651
|
+
commandPath: canonicalCommandPath(
|
|
1652
|
+
commandName,
|
|
1653
|
+
selectedSubcommand
|
|
1654
|
+
? subcommand
|
|
1655
|
+
: commandName === "work" && commandPositionals[0] === "prepare"
|
|
1656
|
+
? "prepare"
|
|
1657
|
+
: undefined,
|
|
1658
|
+
),
|
|
1625
1659
|
args: selectedSubcommand
|
|
1626
1660
|
? [subcommand!, ...commandPositionals]
|
|
1627
1661
|
: commandPositionals,
|
package/src/cli.test.ts
CHANGED
|
@@ -179,10 +179,31 @@ describe("CLI", () => {
|
|
|
179
179
|
XDG_STATE_HOME: state,
|
|
180
180
|
AGENCY_SESSION_ID: "cli-session",
|
|
181
181
|
AGENCY_NO_USAGE_LOG: "0",
|
|
182
|
+
AGENCY_INVOCATION_SOURCE: "automation",
|
|
183
|
+
AGENCY_USAGE_TEST: "1",
|
|
182
184
|
}
|
|
183
185
|
expect((await runCli(["--version"], projectRoot, env)).exitCode).toBe(0)
|
|
184
186
|
expect(
|
|
185
|
-
(
|
|
187
|
+
(
|
|
188
|
+
await runCli(
|
|
189
|
+
[
|
|
190
|
+
"task",
|
|
191
|
+
"create",
|
|
192
|
+
"private-customer-id",
|
|
193
|
+
"--repo",
|
|
194
|
+
"private-repository",
|
|
195
|
+
"--description",
|
|
196
|
+
"private free-form input",
|
|
197
|
+
"--cwd",
|
|
198
|
+
"/private/customer/path",
|
|
199
|
+
],
|
|
200
|
+
projectRoot,
|
|
201
|
+
env,
|
|
202
|
+
)
|
|
203
|
+
).exitCode,
|
|
204
|
+
).toBe(1)
|
|
205
|
+
expect(
|
|
206
|
+
(await runCli(["unknown", "--private-flag=value"], projectRoot, env))
|
|
186
207
|
.exitCode,
|
|
187
208
|
).toBe(1)
|
|
188
209
|
|
|
@@ -194,20 +215,41 @@ describe("CLI", () => {
|
|
|
194
215
|
.map((line) => JSON.parse(line))
|
|
195
216
|
expect(events).toEqual([
|
|
196
217
|
expect.objectContaining({
|
|
197
|
-
|
|
198
|
-
|
|
218
|
+
journeyId: expect.stringMatching(/^sha256:[a-f0-9]{64}$/),
|
|
219
|
+
journeySequence: 1,
|
|
220
|
+
invocationSource: "automation",
|
|
221
|
+
isTest: true,
|
|
199
222
|
commandPath: "version",
|
|
200
223
|
flagNames: ["version"],
|
|
201
224
|
outcome: "success",
|
|
225
|
+
outcomeCode: "SUCCESS",
|
|
202
226
|
}),
|
|
203
227
|
expect.objectContaining({
|
|
204
|
-
|
|
228
|
+
journeySequence: 2,
|
|
229
|
+
commandPath: "task/create",
|
|
230
|
+
flagNames: ["cwd", "description", "repo"],
|
|
231
|
+
outcome: "failure",
|
|
232
|
+
outcomeCode: "WORKBASE_NOT_FOUND",
|
|
233
|
+
}),
|
|
234
|
+
expect.objectContaining({
|
|
235
|
+
journeySequence: 3,
|
|
205
236
|
commandPath: "invalid",
|
|
206
|
-
flagNames: [
|
|
237
|
+
flagNames: [],
|
|
207
238
|
outcome: "failure",
|
|
239
|
+
outcomeCode: "CLI_USAGE",
|
|
208
240
|
}),
|
|
209
241
|
])
|
|
210
|
-
|
|
242
|
+
for (const value of [
|
|
243
|
+
"cli-session",
|
|
244
|
+
"private-customer-id",
|
|
245
|
+
"private-repository",
|
|
246
|
+
"private free-form input",
|
|
247
|
+
"/private/customer/path",
|
|
248
|
+
"private-flag",
|
|
249
|
+
"value",
|
|
250
|
+
]) {
|
|
251
|
+
expect(exported.stdout).not.toContain(value)
|
|
252
|
+
}
|
|
211
253
|
})
|
|
212
254
|
|
|
213
255
|
test("records status-based non-PR completion", async () => {
|
|
@@ -260,6 +260,9 @@ describe("IntegrationService", () => {
|
|
|
260
260
|
"!result.authority?.writable?.checkoutPath",
|
|
261
261
|
)
|
|
262
262
|
expect(managedWorkbaseOpencodePlugin).toContain('status !== "working"')
|
|
263
|
+
expect(managedWorkbaseOpencodePlugin).toContain(
|
|
264
|
+
'output.env.AGENCY_INVOCATION_SOURCE = "agent"',
|
|
265
|
+
)
|
|
263
266
|
expect(managedWorkbaseOpencodePlugin).toContain(
|
|
264
267
|
"output.env.AGENCY_SESSION_ID = sessionID",
|
|
265
268
|
)
|
package/src/usage-log.test.ts
CHANGED
|
@@ -1,20 +1,36 @@
|
|
|
1
1
|
import { afterEach, describe, expect, test } from "bun:test"
|
|
2
|
+
import { Database } from "bun:sqlite"
|
|
2
3
|
import { rm } from "node:fs/promises"
|
|
3
4
|
import { join } from "node:path"
|
|
4
5
|
import { cleanupTempDir, createTempDir } from "./test-utils"
|
|
5
|
-
import {
|
|
6
|
+
import {
|
|
7
|
+
exportUsageEvents,
|
|
8
|
+
recordUsageEvent,
|
|
9
|
+
usageOutcomeCode,
|
|
10
|
+
} from "./usage-log"
|
|
6
11
|
|
|
7
12
|
describe("usage logging", () => {
|
|
8
13
|
const tempDirs: string[] = []
|
|
9
14
|
|
|
10
15
|
afterEach(() => Promise.all(tempDirs.splice(0).map(cleanupTempDir)))
|
|
11
16
|
|
|
17
|
+
test("maps only reviewed failure categories", () => {
|
|
18
|
+
expect(usageOutcomeCode({ _tag: "WorkbaseNotFoundError" })).toBe(
|
|
19
|
+
"WORKBASE_NOT_FOUND",
|
|
20
|
+
)
|
|
21
|
+
expect(usageOutcomeCode({ _tag: "private-customer-id" })).toBe(
|
|
22
|
+
"COMMAND_FAILED",
|
|
23
|
+
)
|
|
24
|
+
})
|
|
25
|
+
|
|
12
26
|
test("stores versioned privacy-safe events in session order", async () => {
|
|
13
27
|
const state = await createTempDir()
|
|
14
28
|
tempDirs.push(state)
|
|
15
29
|
const env = {
|
|
16
30
|
XDG_STATE_HOME: state,
|
|
17
31
|
AGENCY_SESSION_ID: "session-1",
|
|
32
|
+
AGENCY_INVOCATION_SOURCE: "automation",
|
|
33
|
+
AGENCY_USAGE_TEST: "1",
|
|
18
34
|
} as NodeJS.ProcessEnv
|
|
19
35
|
for (const commandPath of ["worktree/prepare", "context"]) {
|
|
20
36
|
await recordUsageEvent(
|
|
@@ -23,6 +39,7 @@ describe("usage logging", () => {
|
|
|
23
39
|
flagNames: ["json", "task", "json"],
|
|
24
40
|
durationMs: 12.4,
|
|
25
41
|
outcome: "success",
|
|
42
|
+
outcomeCode: "SUCCESS",
|
|
26
43
|
exitStatus: 0,
|
|
27
44
|
...(commandPath === "context"
|
|
28
45
|
? {
|
|
@@ -40,26 +57,31 @@ describe("usage logging", () => {
|
|
|
40
57
|
expect(await Bun.file(join(state, "agency/usage.sqlite3")).exists()).toBe(
|
|
41
58
|
true,
|
|
42
59
|
)
|
|
43
|
-
|
|
60
|
+
const events = await exportUsageEvents(env)
|
|
61
|
+
expect(events).toEqual([
|
|
44
62
|
expect.objectContaining({
|
|
45
63
|
version: 2,
|
|
46
|
-
|
|
47
|
-
|
|
64
|
+
journeyId: expect.stringMatching(/^sha256:[a-f0-9]{64}$/),
|
|
65
|
+
journeySequence: 1,
|
|
66
|
+
invocationSource: "automation",
|
|
67
|
+
isTest: true,
|
|
48
68
|
agencyVersion: "1.2.3",
|
|
49
69
|
commandPath: "worktree/prepare",
|
|
50
70
|
flagNames: ["json", "task"],
|
|
51
71
|
durationMs: 12,
|
|
52
72
|
outcome: "success",
|
|
73
|
+
outcomeCode: "SUCCESS",
|
|
53
74
|
exitStatus: 0,
|
|
54
75
|
}),
|
|
55
76
|
expect.objectContaining({
|
|
56
|
-
|
|
77
|
+
journeySequence: 2,
|
|
57
78
|
commandPath: "context",
|
|
58
79
|
vcs: "git",
|
|
59
80
|
terminalStage: "publish",
|
|
60
81
|
category: "success",
|
|
61
82
|
}),
|
|
62
83
|
])
|
|
84
|
+
expect(JSON.stringify(events)).not.toContain("session-1")
|
|
63
85
|
})
|
|
64
86
|
|
|
65
87
|
test("supports opt-out and ignores unavailable storage", async () => {
|
|
@@ -75,6 +97,7 @@ describe("usage logging", () => {
|
|
|
75
97
|
flagNames: [],
|
|
76
98
|
durationMs: 1,
|
|
77
99
|
outcome: "failure",
|
|
100
|
+
outcomeCode: "COMMAND_FAILED",
|
|
78
101
|
exitStatus: 1,
|
|
79
102
|
},
|
|
80
103
|
"1.2.3",
|
|
@@ -93,6 +116,7 @@ describe("usage logging", () => {
|
|
|
93
116
|
flagNames: [],
|
|
94
117
|
durationMs: 1,
|
|
95
118
|
outcome: "failure",
|
|
119
|
+
outcomeCode: "COMMAND_FAILED",
|
|
96
120
|
exitStatus: 1,
|
|
97
121
|
},
|
|
98
122
|
"1.2.3",
|
|
@@ -101,4 +125,64 @@ describe("usage logging", () => {
|
|
|
101
125
|
).resolves.toBeUndefined()
|
|
102
126
|
await rm(blocked)
|
|
103
127
|
})
|
|
128
|
+
|
|
129
|
+
test("prunes expired events on every database access", async () => {
|
|
130
|
+
const state = await createTempDir()
|
|
131
|
+
tempDirs.push(state)
|
|
132
|
+
const env = {
|
|
133
|
+
XDG_STATE_HOME: state,
|
|
134
|
+
AGENCY_USAGE_RETENTION_DAYS: "30",
|
|
135
|
+
} as NodeJS.ProcessEnv
|
|
136
|
+
await recordUsageEvent(
|
|
137
|
+
{
|
|
138
|
+
commandPath: "status",
|
|
139
|
+
flagNames: [],
|
|
140
|
+
durationMs: 1,
|
|
141
|
+
outcome: "success",
|
|
142
|
+
outcomeCode: "SUCCESS",
|
|
143
|
+
exitStatus: 0,
|
|
144
|
+
},
|
|
145
|
+
"1.2.3",
|
|
146
|
+
env,
|
|
147
|
+
)
|
|
148
|
+
const database = new Database(join(state, "agency/usage.sqlite3"))
|
|
149
|
+
database.run(
|
|
150
|
+
"UPDATE usage_events SET occurred_at = datetime('now', '-31 days')",
|
|
151
|
+
)
|
|
152
|
+
database.close()
|
|
153
|
+
|
|
154
|
+
expect(await exportUsageEvents(env)).toEqual([])
|
|
155
|
+
})
|
|
156
|
+
|
|
157
|
+
test("removes legacy events that may contain positional values", async () => {
|
|
158
|
+
const state = await createTempDir()
|
|
159
|
+
tempDirs.push(state)
|
|
160
|
+
const path = join(state, "usage.sqlite3")
|
|
161
|
+
const database = new Database(path, { create: true })
|
|
162
|
+
database.run(
|
|
163
|
+
"CREATE TABLE usage_events (id INTEGER PRIMARY KEY, command_path TEXT NOT NULL)",
|
|
164
|
+
)
|
|
165
|
+
database.run("INSERT INTO usage_events (command_path) VALUES (?)", [
|
|
166
|
+
"task/private-customer-id",
|
|
167
|
+
])
|
|
168
|
+
database.close()
|
|
169
|
+
const env = { AGENCY_USAGE_DB: path } as NodeJS.ProcessEnv
|
|
170
|
+
|
|
171
|
+
await recordUsageEvent(
|
|
172
|
+
{
|
|
173
|
+
commandPath: "task/show",
|
|
174
|
+
flagNames: [],
|
|
175
|
+
durationMs: 1,
|
|
176
|
+
outcome: "success",
|
|
177
|
+
outcomeCode: "SUCCESS",
|
|
178
|
+
exitStatus: 0,
|
|
179
|
+
},
|
|
180
|
+
"1.2.3",
|
|
181
|
+
env,
|
|
182
|
+
)
|
|
183
|
+
|
|
184
|
+
expect(await exportUsageEvents(env)).toEqual([
|
|
185
|
+
expect.objectContaining({ commandPath: "task/show", version: 2 }),
|
|
186
|
+
])
|
|
187
|
+
})
|
|
104
188
|
})
|
package/src/usage-log.ts
CHANGED
|
@@ -1,21 +1,92 @@
|
|
|
1
1
|
import { Database } from "bun:sqlite"
|
|
2
|
+
import { createHash } from "node:crypto"
|
|
2
3
|
import { mkdir } from "node:fs/promises"
|
|
3
4
|
import { dirname, join } from "node:path"
|
|
4
5
|
|
|
5
6
|
const USAGE_EVENT_VERSION = 2 as const
|
|
6
7
|
const DEFAULT_RETENTION_DAYS = 90
|
|
7
8
|
|
|
9
|
+
const INVOCATION_SOURCES = new Set(["human", "agent", "automation"])
|
|
10
|
+
const USAGE_EVENT_COLUMNS = new Set([
|
|
11
|
+
"id",
|
|
12
|
+
"event_version",
|
|
13
|
+
"journey_id",
|
|
14
|
+
"journey_sequence",
|
|
15
|
+
"invocation_source",
|
|
16
|
+
"is_test",
|
|
17
|
+
"occurred_at",
|
|
18
|
+
"agency_version",
|
|
19
|
+
"command_path",
|
|
20
|
+
"flag_names",
|
|
21
|
+
"duration_ms",
|
|
22
|
+
"outcome",
|
|
23
|
+
"outcome_code",
|
|
24
|
+
"exit_status",
|
|
25
|
+
"vcs",
|
|
26
|
+
"terminal_stage",
|
|
27
|
+
"category",
|
|
28
|
+
])
|
|
29
|
+
|
|
30
|
+
export type UsageOutcomeCode =
|
|
31
|
+
| "SUCCESS"
|
|
32
|
+
| "NONZERO_EXIT"
|
|
33
|
+
| "CLI_USAGE"
|
|
34
|
+
| "WORKBASE_NOT_FOUND"
|
|
35
|
+
| "WORKBASE_INVALID"
|
|
36
|
+
| "VALIDATION_FAILED"
|
|
37
|
+
| "CONFLICT"
|
|
38
|
+
| "FILESYSTEM_ERROR"
|
|
39
|
+
| "PROCESS_ERROR"
|
|
40
|
+
| "COMMAND_FAILED"
|
|
41
|
+
|
|
8
42
|
export interface UsageEvent {
|
|
9
43
|
readonly commandPath: string
|
|
10
44
|
readonly flagNames: readonly string[]
|
|
11
45
|
readonly durationMs: number
|
|
12
46
|
readonly exitStatus: number
|
|
13
47
|
readonly outcome: "success" | "failure"
|
|
48
|
+
readonly outcomeCode: UsageOutcomeCode
|
|
14
49
|
readonly vcs?: "git"
|
|
15
50
|
readonly terminalStage?: string
|
|
16
51
|
readonly category?: string
|
|
17
52
|
}
|
|
18
53
|
|
|
54
|
+
export const usageOutcomeCode = (error: unknown): UsageOutcomeCode => {
|
|
55
|
+
const tag =
|
|
56
|
+
typeof error === "object" &&
|
|
57
|
+
error !== null &&
|
|
58
|
+
"_tag" in error &&
|
|
59
|
+
typeof error._tag === "string"
|
|
60
|
+
? error._tag
|
|
61
|
+
: error instanceof Error
|
|
62
|
+
? error.name
|
|
63
|
+
: undefined
|
|
64
|
+
switch (tag) {
|
|
65
|
+
case "CliUsageError":
|
|
66
|
+
return "CLI_USAGE"
|
|
67
|
+
case "WorkbaseNotFoundError":
|
|
68
|
+
return "WORKBASE_NOT_FOUND"
|
|
69
|
+
case "WorkbaseConfigError":
|
|
70
|
+
case "WorkbaseRegistryError":
|
|
71
|
+
case "FrontmatterParseError":
|
|
72
|
+
return "WORKBASE_INVALID"
|
|
73
|
+
case "ValidationFailedError":
|
|
74
|
+
return "VALIDATION_FAILED"
|
|
75
|
+
case "ClaimConflictError":
|
|
76
|
+
case "ClaimOwnershipError":
|
|
77
|
+
case "RevisionConflictError":
|
|
78
|
+
case "ExecutionGuardError":
|
|
79
|
+
return "CONFLICT"
|
|
80
|
+
case "FileNotFoundError":
|
|
81
|
+
case "FileSystemError":
|
|
82
|
+
return "FILESYSTEM_ERROR"
|
|
83
|
+
case "ProcessError":
|
|
84
|
+
return "PROCESS_ERROR"
|
|
85
|
+
default:
|
|
86
|
+
return "COMMAND_FAILED"
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
|
|
19
90
|
const stateDirectory = (env: NodeJS.ProcessEnv) =>
|
|
20
91
|
env.XDG_STATE_HOME ?? join(env.HOME ?? ".", ".local", "state")
|
|
21
92
|
|
|
@@ -32,36 +103,68 @@ const retentionDays = (env: NodeJS.ProcessEnv) => {
|
|
|
32
103
|
: DEFAULT_RETENTION_DAYS
|
|
33
104
|
}
|
|
34
105
|
|
|
106
|
+
const invocationSource = (env: NodeJS.ProcessEnv) => {
|
|
107
|
+
const source = env.AGENCY_INVOCATION_SOURCE?.toLowerCase()
|
|
108
|
+
if (source && INVOCATION_SOURCES.has(source)) return source
|
|
109
|
+
return env.AGENCY_SESSION_ID ? "agent" : "human"
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
const isTestInvocation = (env: NodeJS.ProcessEnv) =>
|
|
113
|
+
["1", "true", "yes"].includes((env.AGENCY_USAGE_TEST ?? "").toLowerCase())
|
|
114
|
+
|
|
115
|
+
const journeyId = (env: NodeJS.ProcessEnv) => {
|
|
116
|
+
if (!env.AGENCY_SESSION_ID) return null
|
|
117
|
+
return `sha256:${createHash("sha256").update(env.AGENCY_SESSION_ID).digest("hex")}`
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
const pruneExpiredEvents = (database: Database, env: NodeJS.ProcessEnv) => {
|
|
121
|
+
database
|
|
122
|
+
.query(
|
|
123
|
+
"DELETE FROM usage_events WHERE datetime(occurred_at) < datetime('now', ?)",
|
|
124
|
+
)
|
|
125
|
+
.run(`-${retentionDays(env)} days`)
|
|
126
|
+
}
|
|
127
|
+
|
|
35
128
|
const openDatabase = async (env: NodeJS.ProcessEnv) => {
|
|
36
129
|
const path = usageDatabasePath(env)
|
|
37
130
|
await mkdir(dirname(path), { recursive: true, mode: 0o700 })
|
|
38
131
|
const database = new Database(path, { create: true, strict: true })
|
|
39
132
|
database.run("PRAGMA journal_mode = WAL")
|
|
40
133
|
database.run("PRAGMA busy_timeout = 1000")
|
|
134
|
+
const existingColumns = database
|
|
135
|
+
.query("PRAGMA table_info(usage_events)")
|
|
136
|
+
.all() as { name: string }[]
|
|
137
|
+
if (
|
|
138
|
+
existingColumns.length > 0 &&
|
|
139
|
+
(existingColumns.length !== USAGE_EVENT_COLUMNS.size ||
|
|
140
|
+
existingColumns.some(({ name }) => !USAGE_EVENT_COLUMNS.has(name)))
|
|
141
|
+
) {
|
|
142
|
+
// Version 1 could contain positional values. Do not preserve unsafe telemetry.
|
|
143
|
+
database.run("DROP TABLE usage_events")
|
|
144
|
+
}
|
|
41
145
|
database.run(`
|
|
42
146
|
CREATE TABLE IF NOT EXISTS usage_events (
|
|
43
147
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
44
148
|
event_version INTEGER NOT NULL,
|
|
45
|
-
|
|
46
|
-
|
|
149
|
+
journey_id TEXT,
|
|
150
|
+
journey_sequence INTEGER,
|
|
151
|
+
invocation_source TEXT NOT NULL,
|
|
152
|
+
is_test INTEGER NOT NULL,
|
|
47
153
|
occurred_at TEXT NOT NULL,
|
|
48
154
|
agency_version TEXT NOT NULL,
|
|
49
155
|
command_path TEXT NOT NULL,
|
|
50
156
|
flag_names TEXT NOT NULL,
|
|
51
157
|
duration_ms INTEGER NOT NULL,
|
|
52
158
|
outcome TEXT NOT NULL,
|
|
53
|
-
|
|
159
|
+
outcome_code TEXT NOT NULL,
|
|
160
|
+
exit_status INTEGER NOT NULL,
|
|
161
|
+
vcs TEXT,
|
|
162
|
+
terminal_stage TEXT,
|
|
163
|
+
category TEXT
|
|
54
164
|
)
|
|
55
165
|
`)
|
|
56
|
-
for (const column of ["vcs", "terminal_stage", "category"]) {
|
|
57
|
-
try {
|
|
58
|
-
database.run(`ALTER TABLE usage_events ADD COLUMN ${column} TEXT`)
|
|
59
|
-
} catch {
|
|
60
|
-
// Existing databases already have migrated columns.
|
|
61
|
-
}
|
|
62
|
-
}
|
|
63
166
|
database.run(
|
|
64
|
-
"CREATE INDEX IF NOT EXISTS
|
|
167
|
+
"CREATE INDEX IF NOT EXISTS usage_events_journey ON usage_events(journey_id, journey_sequence)",
|
|
65
168
|
)
|
|
66
169
|
database.run(
|
|
67
170
|
"CREATE INDEX IF NOT EXISTS usage_events_command ON usage_events(command_path, occurred_at)",
|
|
@@ -78,42 +181,42 @@ export async function recordUsageEvent(
|
|
|
78
181
|
let database: Database | undefined
|
|
79
182
|
try {
|
|
80
183
|
database = await openDatabase(env)
|
|
81
|
-
|
|
184
|
+
pruneExpiredEvents(database, env)
|
|
185
|
+
const eventJourneyId = journeyId(env)
|
|
82
186
|
database
|
|
83
187
|
.query(`
|
|
84
188
|
INSERT INTO usage_events (
|
|
85
|
-
event_version,
|
|
189
|
+
event_version, journey_id, journey_sequence,
|
|
190
|
+
invocation_source, is_test, occurred_at,
|
|
86
191
|
agency_version, command_path, flag_names, duration_ms,
|
|
87
|
-
outcome, exit_status, vcs, terminal_stage, category
|
|
192
|
+
outcome, outcome_code, exit_status, vcs, terminal_stage, category
|
|
88
193
|
) VALUES (
|
|
89
194
|
?, ?,
|
|
90
|
-
|
|
91
|
-
|
|
195
|
+
CASE WHEN ? IS NULL THEN NULL ELSE
|
|
196
|
+
(SELECT COALESCE(MAX(journey_sequence), 0) + 1 FROM usage_events WHERE journey_id = ?)
|
|
197
|
+
END,
|
|
198
|
+
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
|
|
92
199
|
)
|
|
93
200
|
`)
|
|
94
201
|
.run(
|
|
95
202
|
USAGE_EVENT_VERSION,
|
|
96
|
-
|
|
97
|
-
|
|
203
|
+
eventJourneyId,
|
|
204
|
+
eventJourneyId,
|
|
205
|
+
eventJourneyId,
|
|
206
|
+
invocationSource(env),
|
|
207
|
+
isTestInvocation(env) ? 1 : 0,
|
|
98
208
|
new Date().toISOString(),
|
|
99
209
|
agencyVersion,
|
|
100
210
|
event.commandPath,
|
|
101
211
|
JSON.stringify([...new Set(event.flagNames)].sort()),
|
|
102
212
|
Math.max(0, Math.round(event.durationMs)),
|
|
103
213
|
event.outcome,
|
|
214
|
+
event.outcomeCode,
|
|
104
215
|
event.exitStatus,
|
|
105
216
|
event.vcs ?? null,
|
|
106
217
|
event.terminalStage ?? null,
|
|
107
218
|
event.category ?? null,
|
|
108
219
|
)
|
|
109
|
-
if (Math.random() < 0.01) {
|
|
110
|
-
const days = retentionDays(env)
|
|
111
|
-
database
|
|
112
|
-
.query(
|
|
113
|
-
"DELETE FROM usage_events WHERE occurred_at < datetime('now', ?)",
|
|
114
|
-
)
|
|
115
|
-
.run(`-${days} days`)
|
|
116
|
-
}
|
|
117
220
|
} catch {
|
|
118
221
|
// Usage logging must never affect command behavior.
|
|
119
222
|
} finally {
|
|
@@ -128,24 +231,29 @@ export async function exportUsageEvents(
|
|
|
128
231
|
let database: Database | undefined
|
|
129
232
|
try {
|
|
130
233
|
database = await openDatabase(env)
|
|
234
|
+
pruneExpiredEvents(database, env)
|
|
131
235
|
const rows = database
|
|
132
236
|
.query(`
|
|
133
|
-
SELECT event_version,
|
|
237
|
+
SELECT event_version, journey_id, journey_sequence,
|
|
238
|
+
invocation_source, is_test, occurred_at,
|
|
134
239
|
agency_version, command_path, flag_names, duration_ms,
|
|
135
|
-
outcome, exit_status, vcs, terminal_stage, category
|
|
240
|
+
outcome, outcome_code, exit_status, vcs, terminal_stage, category
|
|
136
241
|
FROM usage_events ORDER BY occurred_at, id
|
|
137
242
|
`)
|
|
138
|
-
.all() as Record<string, string | number>[]
|
|
243
|
+
.all() as Record<string, string | number | null>[]
|
|
139
244
|
return rows.map((row) => ({
|
|
140
245
|
version: row.event_version,
|
|
141
|
-
|
|
142
|
-
|
|
246
|
+
journeyId: row.journey_id,
|
|
247
|
+
journeySequence: row.journey_sequence,
|
|
248
|
+
invocationSource: row.invocation_source,
|
|
249
|
+
isTest: row.is_test === 1,
|
|
143
250
|
occurredAt: row.occurred_at,
|
|
144
251
|
agencyVersion: row.agency_version,
|
|
145
252
|
commandPath: row.command_path,
|
|
146
253
|
flagNames: JSON.parse(String(row.flag_names)),
|
|
147
254
|
durationMs: row.duration_ms,
|
|
148
255
|
outcome: row.outcome,
|
|
256
|
+
outcomeCode: row.outcome_code,
|
|
149
257
|
exitStatus: row.exit_status,
|
|
150
258
|
...(row.vcs == null ? {} : { vcs: row.vcs }),
|
|
151
259
|
...(row.terminal_stage == null
|
|
@@ -114,6 +114,7 @@ describe("agent commands", () => {
|
|
|
114
114
|
|
|
115
115
|
expect(environment).toMatchObject({
|
|
116
116
|
AGENCY_AGENT: "custom",
|
|
117
|
+
AGENCY_INVOCATION_SOURCE: "agent",
|
|
117
118
|
AGENCY_SESSION_ID: "session-1",
|
|
118
119
|
AGENCY_WORKBASE: "/workbase",
|
|
119
120
|
AGENCY_TARGET: "execution-unit:phase/task/build",
|
|
@@ -118,6 +118,7 @@ export const agentEnvironment = (
|
|
|
118
118
|
variables: AgentCommandVariables,
|
|
119
119
|
): Record<string, string> => ({
|
|
120
120
|
AGENCY_AGENT: agent,
|
|
121
|
+
AGENCY_INVOCATION_SOURCE: "agent",
|
|
121
122
|
AGENCY_SESSION_ID: variables.sessionId,
|
|
122
123
|
AGENCY_WORKBASE: variables.workbase,
|
|
123
124
|
AGENCY_TARGET: variables.target,
|
|
@@ -195,6 +195,7 @@ const plugin: Plugin = async ({ directory }) => {
|
|
|
195
195
|
if (!sessionID) return
|
|
196
196
|
const context = workerSessions.get(sessionID)
|
|
197
197
|
if (!context?.target) return
|
|
198
|
+
output.env.AGENCY_INVOCATION_SOURCE = "agent"
|
|
198
199
|
output.env.AGENCY_SESSION_ID = sessionID
|
|
199
200
|
output.env.AGENCY_TARGET = context.target
|
|
200
201
|
if (context.root) output.env.AGENCY_WORKBASE = context.root
|