@skillrecordings/cli 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/.env.encrypted +0 -0
  2. package/CHANGELOG.md +35 -0
  3. package/README.md +214 -0
  4. package/bin/skill.ts +3 -0
  5. package/data/tt-archive-dataset.json +1 -0
  6. package/data/validate-test-dataset.json +97 -0
  7. package/docs/CLI-AUTH.md +504 -0
  8. package/package.json +38 -0
  9. package/preload.ts +18 -0
  10. package/src/__tests__/init.test.ts +74 -0
  11. package/src/alignment-test.ts +64 -0
  12. package/src/check-apps.ts +16 -0
  13. package/src/commands/auth/decrypt.ts +123 -0
  14. package/src/commands/auth/encrypt.ts +81 -0
  15. package/src/commands/auth/index.ts +50 -0
  16. package/src/commands/auth/keygen.ts +41 -0
  17. package/src/commands/auth/status.ts +164 -0
  18. package/src/commands/axiom/forensic.ts +868 -0
  19. package/src/commands/axiom/index.ts +697 -0
  20. package/src/commands/build-dataset.ts +311 -0
  21. package/src/commands/db-status.ts +47 -0
  22. package/src/commands/deploys.ts +219 -0
  23. package/src/commands/eval-local/compare.ts +171 -0
  24. package/src/commands/eval-local/health.ts +212 -0
  25. package/src/commands/eval-local/index.ts +76 -0
  26. package/src/commands/eval-local/real-tools.ts +416 -0
  27. package/src/commands/eval-local/run.ts +1168 -0
  28. package/src/commands/eval-local/score-production.ts +256 -0
  29. package/src/commands/eval-local/seed.ts +276 -0
  30. package/src/commands/eval-pipeline/index.ts +53 -0
  31. package/src/commands/eval-pipeline/real-tools.ts +492 -0
  32. package/src/commands/eval-pipeline/run.ts +1316 -0
  33. package/src/commands/eval-pipeline/seed.ts +395 -0
  34. package/src/commands/eval-prompt.ts +496 -0
  35. package/src/commands/eval.test.ts +253 -0
  36. package/src/commands/eval.ts +108 -0
  37. package/src/commands/faq-classify.ts +460 -0
  38. package/src/commands/faq-cluster.ts +135 -0
  39. package/src/commands/faq-extract.ts +249 -0
  40. package/src/commands/faq-mine.ts +432 -0
  41. package/src/commands/faq-review.ts +426 -0
  42. package/src/commands/front/index.ts +351 -0
  43. package/src/commands/front/pull-conversations.ts +275 -0
  44. package/src/commands/front/tags.ts +825 -0
  45. package/src/commands/front-cache.ts +1277 -0
  46. package/src/commands/front-stats.ts +75 -0
  47. package/src/commands/health.test.ts +82 -0
  48. package/src/commands/health.ts +362 -0
  49. package/src/commands/init.test.ts +89 -0
  50. package/src/commands/init.ts +106 -0
  51. package/src/commands/inngest/client.ts +294 -0
  52. package/src/commands/inngest/events.ts +296 -0
  53. package/src/commands/inngest/investigate.ts +382 -0
  54. package/src/commands/inngest/runs.ts +149 -0
  55. package/src/commands/inngest/signal.ts +143 -0
  56. package/src/commands/kb-sync.ts +498 -0
  57. package/src/commands/memory/find.ts +135 -0
  58. package/src/commands/memory/get.ts +87 -0
  59. package/src/commands/memory/index.ts +97 -0
  60. package/src/commands/memory/stats.ts +163 -0
  61. package/src/commands/memory/store.ts +49 -0
  62. package/src/commands/memory/vote.ts +159 -0
  63. package/src/commands/pipeline.ts +127 -0
  64. package/src/commands/responses.ts +856 -0
  65. package/src/commands/tools.ts +293 -0
  66. package/src/commands/wizard.ts +319 -0
  67. package/src/index.ts +172 -0
  68. package/src/lib/crypto.ts +56 -0
  69. package/src/lib/env-loader.ts +206 -0
  70. package/src/lib/onepassword.ts +137 -0
  71. package/src/test-agent-local.ts +115 -0
  72. package/tsconfig.json +11 -0
  73. package/vitest.config.ts +10 -0
@@ -0,0 +1,868 @@
1
+ /**
2
+ * Forensic Query Toolkit for Agent Self-Diagnosis
3
+ *
4
+ * Canned queries that agents use to trace pipelines, measure step timings,
5
+ * detect errors, verify data flow, and check overall system health.
6
+ *
7
+ * Usage:
8
+ * skill axiom pipeline-trace <conversationId> [--since 7d]
9
+ * skill axiom step-timings [--since 7d]
10
+ * skill axiom error-rate [--since 7d]
11
+ * skill axiom data-flow-check [--since 7d]
12
+ * skill axiom tag-health [--since 7d]
13
+ * skill axiom approval-stats [--since 7d]
14
+ * skill axiom pipeline-health [--since 7d]
15
+ */
16
+
17
+ import { Axiom } from '@axiomhq/js'
18
+ import type { Command } from 'commander'
19
+
20
+ // ---------------------------------------------------------------------------
21
+ // Shared helpers (mirror the patterns in index.ts)
22
+ // ---------------------------------------------------------------------------
23
+
24
+ function getDataset(): string {
25
+ return process.env.AXIOM_DATASET || 'support-agent'
26
+ }
27
+
28
+ function getAxiomClient(): Axiom {
29
+ const token = process.env.AXIOM_TOKEN
30
+ if (!token) {
31
+ console.error('AXIOM_TOKEN environment variable is required')
32
+ process.exit(1)
33
+ }
34
+ return new Axiom({ token })
35
+ }
36
+
37
+ function parseTimeRange(since: string): { startTime: Date; endTime: Date } {
38
+ const endTime = new Date()
39
+ const match = since.match(/^(\d+)([hmd])$/)
40
+ if (match && match[1] && match[2]) {
41
+ const value = parseInt(match[1], 10)
42
+ const unit = match[2] as 'h' | 'm' | 'd'
43
+ const msPerUnit: Record<'h' | 'm' | 'd', number> = {
44
+ h: 60 * 60 * 1000,
45
+ m: 60 * 1000,
46
+ d: 24 * 60 * 60 * 1000,
47
+ }
48
+ return {
49
+ startTime: new Date(endTime.getTime() - value * msPerUnit[unit]),
50
+ endTime,
51
+ }
52
+ }
53
+ const startTime = new Date(since)
54
+ if (isNaN(startTime.getTime())) {
55
+ console.error(
56
+ `Invalid time range: ${since}. Use format like "1h", "24h", "7d" or ISO date.`
57
+ )
58
+ process.exit(1)
59
+ }
60
+ return { startTime, endTime }
61
+ }
62
+
63
+ function formatDuration(ms: number): string {
64
+ if (ms < 1000) return `${Math.round(ms)}ms`
65
+ if (ms < 60000) return `${(ms / 1000).toFixed(1)}s`
66
+ return `${(ms / 60000).toFixed(1)}m`
67
+ }
68
+
69
+ function formatTime(timestamp: string | Date): string {
70
+ const date = new Date(timestamp)
71
+ return date.toLocaleString('en-US', {
72
+ month: 'short',
73
+ day: 'numeric',
74
+ hour: '2-digit',
75
+ minute: '2-digit',
76
+ second: '2-digit',
77
+ })
78
+ }
79
+
80
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
81
+ type AnyBucket = any
82
+
83
+ /** Safely extract a numeric aggregation value from an Axiom bucket */
84
+ function aggVal(bucket: AnyBucket, index: number): number {
85
+ const aggs = bucket?.aggregations as Array<{ value: unknown }> | undefined
86
+ return Number(aggs?.[index]?.value ?? 0)
87
+ }
88
+
89
+ /** Safely extract a group field from an Axiom bucket */
90
+ function groupVal(bucket: AnyBucket, field: string): string {
91
+ const group = bucket?.group as Record<string, string> | undefined
92
+ return group?.[field] ?? ''
93
+ }
94
+
95
+ // ---------------------------------------------------------------------------
96
+ // 1. pipeline-trace — Full trace for a single conversation
97
+ // ---------------------------------------------------------------------------
98
+
99
+ async function pipelineTrace(
100
+ conversationId: string,
101
+ options: { since?: string; json?: boolean }
102
+ ): Promise<void> {
103
+ const client = getAxiomClient()
104
+ const ds = getDataset()
105
+ const { startTime, endTime } = parseTimeRange(options.since ?? '7d')
106
+ const timeOpts = {
107
+ startTime: startTime.toISOString(),
108
+ endTime: endTime.toISOString(),
109
+ }
110
+
111
+ const baseProjection =
112
+ '_time, name, step, level, message, category, confidence, durationMs, tagged'
113
+ const baseQuery = `['${ds}'] | where conversationId == '${conversationId}' | sort by _time asc`
114
+
115
+ // traceId may not exist yet (T0.3 adds it). Try with it, fall back without.
116
+ let result
117
+ try {
118
+ result = await client.query(
119
+ `${baseQuery} | project ${baseProjection}, traceId`,
120
+ timeOpts
121
+ )
122
+ } catch {
123
+ result = await client.query(
124
+ `${baseQuery} | project ${baseProjection}`,
125
+ timeOpts
126
+ )
127
+ }
128
+
129
+ try {
130
+ const matches = result.matches ?? []
131
+
132
+ if (options.json) {
133
+ console.log(
134
+ JSON.stringify(
135
+ matches.map((m) => ({ _time: m._time, ...(m.data as object) })),
136
+ null,
137
+ 2
138
+ )
139
+ )
140
+ return
141
+ }
142
+
143
+ if (matches.length === 0) {
144
+ console.log(`No events found for conversation: ${conversationId}`)
145
+ return
146
+ }
147
+
148
+ console.log(`\n🔍 Pipeline Trace: ${conversationId}`)
149
+ console.log(
150
+ ` Events: ${matches.length} | Window: ${options.since ?? '7d'}`
151
+ )
152
+ console.log('═'.repeat(90))
153
+
154
+ for (const match of matches) {
155
+ const d = match.data as Record<string, unknown>
156
+ const time = formatTime(match._time)
157
+ const name = String(d.name ?? '—')
158
+ const step = d.step ? ` [${d.step}]` : ''
159
+ const level = d.level ? ` ${String(d.level).toUpperCase()}` : ''
160
+ const dur = d.durationMs ? ` ${formatDuration(Number(d.durationMs))}` : ''
161
+ const cat = d.category ? ` cat=${d.category}` : ''
162
+ const conf = d.confidence != null ? ` conf=${d.confidence}` : ''
163
+ const tag = d.tagged != null ? ` tagged=${d.tagged}` : ''
164
+ const trace = d.traceId ? ` trace=${d.traceId}` : ''
165
+
166
+ console.log(
167
+ ` ${time} ${name}${step}${level}${dur}${cat}${conf}${tag}${trace}`
168
+ )
169
+ if (d.message) {
170
+ console.log(` ${String(d.message).slice(0, 120)}`)
171
+ }
172
+ }
173
+
174
+ console.log('─'.repeat(90))
175
+ console.log(`Total: ${matches.length} events`)
176
+ } catch (error) {
177
+ console.error(
178
+ 'Query failed:',
179
+ error instanceof Error ? error.message : error
180
+ )
181
+ process.exit(1)
182
+ }
183
+ }
184
+
185
+ // ---------------------------------------------------------------------------
186
+ // 2. step-timings — P50/P95 duration by step name
187
+ // ---------------------------------------------------------------------------
188
+
189
+ async function stepTimings(options: {
190
+ since?: string
191
+ json?: boolean
192
+ }): Promise<void> {
193
+ const client = getAxiomClient()
194
+ const ds = getDataset()
195
+ const { startTime, endTime } = parseTimeRange(options.since ?? '7d')
196
+
197
+ const apl = `['${ds}']
198
+ | where isnotnull(durationMs) and durationMs > 0
199
+ | summarize p50=percentile(durationMs, 50), p95=percentile(durationMs, 95), avg=avg(durationMs), count=count() by name
200
+ | sort by p95 desc`
201
+
202
+ try {
203
+ const result = await client.query(apl, {
204
+ startTime: startTime.toISOString(),
205
+ endTime: endTime.toISOString(),
206
+ })
207
+
208
+ const buckets = result.buckets?.totals ?? []
209
+
210
+ if (options.json) {
211
+ console.log(
212
+ JSON.stringify(
213
+ buckets.map((b) => ({
214
+ name: groupVal(b, 'name'),
215
+ p50: aggVal(b, 0),
216
+ p95: aggVal(b, 1),
217
+ avg: aggVal(b, 2),
218
+ count: aggVal(b, 3),
219
+ })),
220
+ null,
221
+ 2
222
+ )
223
+ )
224
+ return
225
+ }
226
+
227
+ if (buckets.length === 0) {
228
+ console.log('No timing data found')
229
+ return
230
+ }
231
+
232
+ console.log(`\n⏱ Step Timings (${options.since ?? '7d'})`)
233
+ console.log('═'.repeat(90))
234
+ console.log(
235
+ `${'Step'.padEnd(30)} ${'P50'.padStart(10)} ${'P95'.padStart(10)} ${'Avg'.padStart(10)} ${'Count'.padStart(8)}`
236
+ )
237
+ console.log('─'.repeat(90))
238
+
239
+ for (const bucket of buckets) {
240
+ const name = groupVal(bucket, 'name') || '—'
241
+ const p50 = formatDuration(aggVal(bucket, 0))
242
+ const p95 = formatDuration(aggVal(bucket, 1))
243
+ const avg = formatDuration(aggVal(bucket, 2))
244
+ const count = String(aggVal(bucket, 3))
245
+
246
+ console.log(
247
+ `${name.padEnd(30)} ${p50.padStart(10)} ${p95.padStart(10)} ${avg.padStart(10)} ${count.padStart(8)}`
248
+ )
249
+ }
250
+
251
+ console.log('─'.repeat(90))
252
+ } catch (error) {
253
+ console.error(
254
+ 'Query failed:',
255
+ error instanceof Error ? error.message : error
256
+ )
257
+ process.exit(1)
258
+ }
259
+ }
260
+
261
+ // ---------------------------------------------------------------------------
262
+ // 3. error-rate — Failure rate by step over time window
263
+ // ---------------------------------------------------------------------------
264
+
265
+ async function errorRate(options: {
266
+ since?: string
267
+ json?: boolean
268
+ }): Promise<void> {
269
+ const client = getAxiomClient()
270
+ const ds = getDataset()
271
+ const { startTime, endTime } = parseTimeRange(options.since ?? '7d')
272
+
273
+ // Note: Using extend + where after summarize causes Axiom to return results
274
+ // in matches (not buckets.totals), so we read from matches.
275
+ const apl = `['${ds}']
276
+ | summarize errors=countif(level == 'error' or success == false), total=count() by name
277
+ | extend rate=errors * 100.0 / total
278
+ | where errors > 0
279
+ | sort by rate desc`
280
+
281
+ try {
282
+ const result = await client.query(apl, {
283
+ startTime: startTime.toISOString(),
284
+ endTime: endTime.toISOString(),
285
+ })
286
+
287
+ const matches = result.matches ?? []
288
+
289
+ if (options.json) {
290
+ console.log(
291
+ JSON.stringify(
292
+ matches.map((m) => {
293
+ const d = m.data as Record<string, unknown>
294
+ return {
295
+ name: d.name ?? '—',
296
+ errors: Number(d.errors ?? 0),
297
+ total: Number(d.total ?? 0),
298
+ rate: Number(Number(d.rate ?? 0).toFixed(2)),
299
+ }
300
+ }),
301
+ null,
302
+ 2
303
+ )
304
+ )
305
+ return
306
+ }
307
+
308
+ if (matches.length === 0) {
309
+ console.log('No errors found — pipeline is clean 🎉')
310
+ return
311
+ }
312
+
313
+ console.log(`\n🚨 Error Rate by Step (${options.since ?? '7d'})`)
314
+ console.log('═'.repeat(80))
315
+ console.log(
316
+ `${'Step'.padEnd(30)} ${'Errors'.padStart(8)} ${'Total'.padStart(8)} ${'Rate'.padStart(8)}`
317
+ )
318
+ console.log('─'.repeat(80))
319
+
320
+ for (const match of matches) {
321
+ const d = match.data as Record<string, unknown>
322
+ const name = String(d.name ?? '—')
323
+ const errors = Number(d.errors ?? 0)
324
+ const total = Number(d.total ?? 0)
325
+ const rate = Number(d.rate ?? 0)
326
+
327
+ const indicator = rate > 10 ? '🔴' : rate > 5 ? '🟡' : '🟢'
328
+ console.log(
329
+ `${indicator} ${name.padEnd(28)} ${String(errors).padStart(8)} ${String(total).padStart(8)} ${rate.toFixed(1).padStart(7)}%`
330
+ )
331
+ }
332
+
333
+ console.log('─'.repeat(80))
334
+ } catch (error) {
335
+ console.error(
336
+ 'Query failed:',
337
+ error instanceof Error ? error.message : error
338
+ )
339
+ process.exit(1)
340
+ }
341
+ }
342
+
343
+ // ---------------------------------------------------------------------------
344
+ // 4. data-flow-check — Verify field presence at each boundary
345
+ // ---------------------------------------------------------------------------
346
+
347
+ async function dataFlowCheck(options: {
348
+ since?: string
349
+ json?: boolean
350
+ }): Promise<void> {
351
+ const client = getAxiomClient()
352
+ const ds = getDataset()
353
+ const { startTime, endTime } = parseTimeRange(options.since ?? '7d')
354
+ const timeOpts = {
355
+ startTime: startTime.toISOString(),
356
+ endTime: endTime.toISOString(),
357
+ }
358
+
359
+ // traceId may not exist yet (T0.3 is adding it). Try with it, fall back without.
360
+ const baseFields =
361
+ 'hasConversationId=countif(isnotnull(conversationId)), hasAppId=countif(isnotnull(appId)), hasMessageId=countif(isnotnull(messageId)), hasStep=countif(isnotnull(step))'
362
+ const withTraceId = `${baseFields}, hasTraceId=countif(isnotnull(traceId)), total=count()`
363
+ const withoutTraceId = `${baseFields}, total=count()`
364
+
365
+ let hasTraceIdField = true
366
+
367
+ // traceId may not exist yet (T0.3 adds it). Try with it, fall back without.
368
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
369
+ async function runDataFlowQuery(): Promise<any> {
370
+ try {
371
+ return await client.query(
372
+ `['${ds}'] | where name contains 'workflow' or name == 'log' | summarize ${withTraceId} by name`,
373
+ timeOpts
374
+ )
375
+ } catch {
376
+ hasTraceIdField = false
377
+ return await client.query(
378
+ `['${ds}'] | where name contains 'workflow' or name == 'log' | summarize ${withoutTraceId} by name`,
379
+ timeOpts
380
+ )
381
+ }
382
+ }
383
+
384
+ try {
385
+ const result = await runDataFlowQuery()
386
+ const buckets = result.buckets?.totals ?? []
387
+
388
+ // Field indices shift depending on whether traceId is present
389
+ const totalIdx = hasTraceIdField ? 5 : 4
390
+ const fieldNames = hasTraceIdField
391
+ ? ['convId', 'appId', 'msgId', 'step', 'traceId']
392
+ : ['convId', 'appId', 'msgId', 'step']
393
+ if (options.json) {
394
+ console.log(
395
+ JSON.stringify(
396
+ buckets.map((b: AnyBucket) => {
397
+ const total = aggVal(b, totalIdx)
398
+ const entry: Record<string, unknown> = {
399
+ name: groupVal(b, 'name'),
400
+ conversationId: {
401
+ present: aggVal(b, 0),
402
+ pct: total ? Math.round((aggVal(b, 0) * 100) / total) : 0,
403
+ },
404
+ appId: {
405
+ present: aggVal(b, 1),
406
+ pct: total ? Math.round((aggVal(b, 1) * 100) / total) : 0,
407
+ },
408
+ messageId: {
409
+ present: aggVal(b, 2),
410
+ pct: total ? Math.round((aggVal(b, 2) * 100) / total) : 0,
411
+ },
412
+ step: {
413
+ present: aggVal(b, 3),
414
+ pct: total ? Math.round((aggVal(b, 3) * 100) / total) : 0,
415
+ },
416
+ total,
417
+ }
418
+ if (hasTraceIdField) {
419
+ entry.traceId = {
420
+ present: aggVal(b, 4),
421
+ pct: total ? Math.round((aggVal(b, 4) * 100) / total) : 0,
422
+ }
423
+ }
424
+ return entry
425
+ }),
426
+ null,
427
+ 2
428
+ )
429
+ )
430
+ return
431
+ }
432
+
433
+ if (buckets.length === 0) {
434
+ console.log('No workflow/log events found')
435
+ return
436
+ }
437
+
438
+ const headerFields = fieldNames.map((f) => f.padStart(8)).join(' ')
439
+ const lineWidth = 28 + fieldNames.length * 9 + 9
440
+
441
+ console.log(`\n🔗 Data Flow Check (${options.since ?? '7d'})`)
442
+ if (!hasTraceIdField)
443
+ console.log(' ⚠ traceId field not yet in schema (T0.3 pending)')
444
+ console.log('═'.repeat(lineWidth))
445
+ console.log(`${'Step'.padEnd(28)} ${headerFields} ${'total'.padStart(8)}`)
446
+ console.log('─'.repeat(lineWidth))
447
+
448
+ for (const bucket of buckets) {
449
+ const name = groupVal(bucket, 'name') || '—'
450
+ const total = aggVal(bucket, totalIdx)
451
+ const fields = fieldNames.map((_, i) => {
452
+ const count = aggVal(bucket, i)
453
+ const pct = total ? Math.round((count * 100) / total) : 0
454
+ const indicator =
455
+ pct === 100 ? '✓' : pct > 80 ? '~' : pct === 0 ? '✗' : '!'
456
+ return `${indicator}${String(pct).padStart(3)}%`
457
+ })
458
+
459
+ console.log(
460
+ `${name.padEnd(28)} ${fields.map((f) => f.padStart(8)).join(' ')} ${String(total).padStart(8)}`
461
+ )
462
+ }
463
+
464
+ console.log('─'.repeat(lineWidth))
465
+ console.log('Legend: ✓=100% | ~=>80% | !=partial | ✗=0%')
466
+ } catch (error) {
467
+ console.error(
468
+ 'Query failed:',
469
+ error instanceof Error ? error.message : error
470
+ )
471
+ process.exit(1)
472
+ }
473
+ }
474
+
475
+ // ---------------------------------------------------------------------------
476
+ // 5. tag-health — Tag application success/failure breakdown
477
+ // ---------------------------------------------------------------------------
478
+
479
+ async function tagHealth(options: {
480
+ since?: string
481
+ json?: boolean
482
+ }): Promise<void> {
483
+ const client = getAxiomClient()
484
+ const ds = getDataset()
485
+ const { startTime, endTime } = parseTimeRange(options.since ?? '7d')
486
+
487
+ // Note: The spec suggested grouping by errorType, but that field doesn't exist
488
+ // in the dataset. We group by appId + name instead (which separates log events
489
+ // from workflow.step events for richer diagnostics).
490
+ const apl = `['${ds}']
491
+ | where step == 'apply-tag' or name contains 'tag'
492
+ | summarize success=countif(tagged == true), failure=countif(tagged == false), total=count() by appId, name`
493
+
494
+ try {
495
+ const result = await client.query(apl, {
496
+ startTime: startTime.toISOString(),
497
+ endTime: endTime.toISOString(),
498
+ })
499
+
500
+ const buckets = result.buckets?.totals ?? []
501
+
502
+ if (options.json) {
503
+ console.log(
504
+ JSON.stringify(
505
+ buckets.map((b) => ({
506
+ appId: groupVal(b, 'appId'),
507
+ name: groupVal(b, 'name'),
508
+ success: aggVal(b, 0),
509
+ failure: aggVal(b, 1),
510
+ total: aggVal(b, 2),
511
+ successRate: aggVal(b, 2)
512
+ ? Number(((aggVal(b, 0) * 100) / aggVal(b, 2)).toFixed(1))
513
+ : 0,
514
+ })),
515
+ null,
516
+ 2
517
+ )
518
+ )
519
+ return
520
+ }
521
+
522
+ if (buckets.length === 0) {
523
+ console.log('No tagging events found')
524
+ return
525
+ }
526
+
527
+ console.log(`\n🏷 Tag Health (${options.since ?? '7d'})`)
528
+ console.log('═'.repeat(90))
529
+ console.log(
530
+ `${'App'.padEnd(25)} ${'Event'.padEnd(25)} ${'OK'.padStart(6)} ${'Fail'.padStart(6)} ${'Total'.padStart(6)} ${'Rate'.padStart(8)}`
531
+ )
532
+ console.log('─'.repeat(90))
533
+
534
+ for (const bucket of buckets) {
535
+ const appId = groupVal(bucket, 'appId') || '—'
536
+ const name = groupVal(bucket, 'name') || '—'
537
+ const success = aggVal(bucket, 0)
538
+ const failure = aggVal(bucket, 1)
539
+ const total = aggVal(bucket, 2)
540
+ const rate = total ? ((success * 100) / total).toFixed(1) : '—'
541
+
542
+ const indicator =
543
+ Number(rate) >= 95 ? '🟢' : Number(rate) >= 80 ? '🟡' : '🔴'
544
+ console.log(
545
+ `${indicator} ${appId.padEnd(23)} ${name.padEnd(25)} ${String(success).padStart(6)} ${String(failure).padStart(6)} ${String(total).padStart(6)} ${String(rate).padStart(7)}%`
546
+ )
547
+ }
548
+
549
+ console.log('─'.repeat(90))
550
+ } catch (error) {
551
+ console.error(
552
+ 'Query failed:',
553
+ error instanceof Error ? error.message : error
554
+ )
555
+ process.exit(1)
556
+ }
557
+ }
558
+
559
+ // ---------------------------------------------------------------------------
560
+ // 6. approval-stats — Auto-approval vs manual review breakdown
561
+ // ---------------------------------------------------------------------------
562
+
563
+ async function approvalStats(options: {
564
+ since?: string
565
+ json?: boolean
566
+ }): Promise<void> {
567
+ const client = getAxiomClient()
568
+ const ds = getDataset()
569
+ const { startTime, endTime } = parseTimeRange(options.since ?? '7d')
570
+
571
+ const apl = `['${ds}']
572
+ | where name == 'log' and (message contains 'auto-approv' or message contains 'approval')
573
+ | summarize auto=countif(autoApprove == true), manual=countif(autoApprove == false), total=count() by appId`
574
+
575
+ try {
576
+ const result = await client.query(apl, {
577
+ startTime: startTime.toISOString(),
578
+ endTime: endTime.toISOString(),
579
+ })
580
+
581
+ const buckets = result.buckets?.totals ?? []
582
+
583
+ if (options.json) {
584
+ console.log(
585
+ JSON.stringify(
586
+ buckets.map((b) => {
587
+ const total = aggVal(b, 2)
588
+ return {
589
+ appId: groupVal(b, 'appId'),
590
+ auto: aggVal(b, 0),
591
+ manual: aggVal(b, 1),
592
+ total,
593
+ autoRate: total
594
+ ? Number(((aggVal(b, 0) * 100) / total).toFixed(1))
595
+ : 0,
596
+ }
597
+ }),
598
+ null,
599
+ 2
600
+ )
601
+ )
602
+ return
603
+ }
604
+
605
+ if (buckets.length === 0) {
606
+ console.log('No approval events found')
607
+ return
608
+ }
609
+
610
+ console.log(`\n✅ Approval Stats (${options.since ?? '7d'})`)
611
+ console.log('═'.repeat(70))
612
+ console.log(
613
+ `${'App'.padEnd(30)} ${'Auto'.padStart(8)} ${'Manual'.padStart(8)} ${'Total'.padStart(8)} ${'Auto %'.padStart(8)}`
614
+ )
615
+ console.log('─'.repeat(70))
616
+
617
+ for (const bucket of buckets) {
618
+ const appId = groupVal(bucket, 'appId') || '—'
619
+ const auto = aggVal(bucket, 0)
620
+ const manual = aggVal(bucket, 1)
621
+ const total = aggVal(bucket, 2)
622
+ const autoRate = total ? ((auto * 100) / total).toFixed(1) : '—'
623
+
624
+ console.log(
625
+ `${appId.padEnd(30)} ${String(auto).padStart(8)} ${String(manual).padStart(8)} ${String(total).padStart(8)} ${String(autoRate).padStart(7)}%`
626
+ )
627
+ }
628
+
629
+ console.log('─'.repeat(70))
630
+ } catch (error) {
631
+ console.error(
632
+ 'Query failed:',
633
+ error instanceof Error ? error.message : error
634
+ )
635
+ process.exit(1)
636
+ }
637
+ }
638
+
639
+ // ---------------------------------------------------------------------------
640
+ // 7. pipeline-health — Overall pipeline health dashboard
641
+ // ---------------------------------------------------------------------------
642
+
643
+ async function pipelineHealth(options: {
644
+ since?: string
645
+ json?: boolean
646
+ }): Promise<void> {
647
+ const client = getAxiomClient()
648
+ const ds = getDataset()
649
+ const { startTime, endTime } = parseTimeRange(options.since ?? '7d')
650
+
651
+ const timeOpts = {
652
+ startTime: startTime.toISOString(),
653
+ endTime: endTime.toISOString(),
654
+ }
655
+
656
+ try {
657
+ // Run all sub-queries in parallel
658
+ const [
659
+ totalResult,
660
+ errorResult,
661
+ timingResult,
662
+ tagResult,
663
+ approvalResult,
664
+ topErrorsResult,
665
+ ] = await Promise.all([
666
+ // Total messages processed
667
+ client.query(
668
+ `['${ds}'] | where name == 'agent.run' | summarize total=count()`,
669
+ timeOpts
670
+ ),
671
+ // Error count and rate
672
+ client.query(
673
+ `['${ds}'] | summarize errors=countif(level == 'error' or success == false), total=count()`,
674
+ timeOpts
675
+ ),
676
+ // Average pipeline duration (from agent.run which has durationMs)
677
+ client.query(
678
+ `['${ds}'] | where name == 'agent.run' and isnotnull(durationMs) and durationMs > 0 | summarize avg=avg(durationMs), p50=percentile(durationMs, 50), p95=percentile(durationMs, 95)`,
679
+ timeOpts
680
+ ),
681
+ // Tag success rate
682
+ client.query(
683
+ `['${ds}'] | where step == 'apply-tag' or name contains 'tag' | summarize success=countif(tagged == true), total=count()`,
684
+ timeOpts
685
+ ),
686
+ // Auto-approval rate
687
+ client.query(
688
+ `['${ds}'] | where name == 'log' and (message contains 'auto-approv' or message contains 'approval') | summarize auto=countif(autoApprove == true), total=count()`,
689
+ timeOpts
690
+ ),
691
+ // Top error categories
692
+ client.query(
693
+ `['${ds}'] | where level == 'error' or success == false | summarize count=count() by name | sort by count desc | limit 5`,
694
+ timeOpts
695
+ ),
696
+ ])
697
+
698
+ // Extract values safely
699
+ const totalProcessed = aggVal(
700
+ (totalResult.buckets?.totals?.[0] ?? {}) as Record<string, unknown>,
701
+ 0
702
+ )
703
+
704
+ const errorBucket = (errorResult.buckets?.totals?.[0] ?? {}) as Record<
705
+ string,
706
+ unknown
707
+ >
708
+ const totalErrors = aggVal(errorBucket, 0)
709
+ const totalEvents = aggVal(errorBucket, 1)
710
+ const overallErrorRate = totalEvents ? (totalErrors * 100) / totalEvents : 0
711
+
712
+ const timingBucket = (timingResult.buckets?.totals?.[0] ?? {}) as Record<
713
+ string,
714
+ unknown
715
+ >
716
+ const avgDuration = aggVal(timingBucket, 0)
717
+ const p50Duration = aggVal(timingBucket, 1)
718
+ const p95Duration = aggVal(timingBucket, 2)
719
+
720
+ const tagBucket = (tagResult.buckets?.totals?.[0] ?? {}) as Record<
721
+ string,
722
+ unknown
723
+ >
724
+ const tagSuccess = aggVal(tagBucket, 0)
725
+ const tagTotal = aggVal(tagBucket, 1)
726
+ const tagRate = tagTotal ? (tagSuccess * 100) / tagTotal : 0
727
+
728
+ const approvalBucket = (approvalResult.buckets?.totals?.[0] ??
729
+ {}) as Record<string, unknown>
730
+ const autoApproval = aggVal(approvalBucket, 0)
731
+ const approvalTotal = aggVal(approvalBucket, 1)
732
+ const autoRate = approvalTotal ? (autoApproval * 100) / approvalTotal : 0
733
+
734
+ const topErrors = (topErrorsResult.buckets?.totals ?? []).map((b) => ({
735
+ name: groupVal(b, 'name'),
736
+ count: aggVal(b, 0),
737
+ }))
738
+
739
+ const dashboard = {
740
+ window: options.since ?? '7d',
741
+ totalProcessed,
742
+ totalEvents,
743
+ errors: { count: totalErrors, rate: Number(overallErrorRate.toFixed(2)) },
744
+ duration: {
745
+ avg: Math.round(avgDuration),
746
+ p50: Math.round(p50Duration),
747
+ p95: Math.round(p95Duration),
748
+ },
749
+ tags: {
750
+ success: tagSuccess,
751
+ total: tagTotal,
752
+ rate: Number(tagRate.toFixed(1)),
753
+ },
754
+ approval: {
755
+ auto: autoApproval,
756
+ total: approvalTotal,
757
+ rate: Number(autoRate.toFixed(1)),
758
+ },
759
+ topErrors,
760
+ }
761
+
762
+ if (options.json) {
763
+ console.log(JSON.stringify(dashboard, null, 2))
764
+ return
765
+ }
766
+
767
+ // Pretty dashboard
768
+ const statusIcon =
769
+ overallErrorRate > 5 ? '🔴' : overallErrorRate > 2 ? '🟡' : '🟢'
770
+
771
+ console.log(
772
+ `\n${statusIcon} Pipeline Health Dashboard (${options.since ?? '7d'})`
773
+ )
774
+ console.log('═'.repeat(60))
775
+ console.log()
776
+ console.log(` 📬 Messages processed: ${totalProcessed}`)
777
+ console.log(` 📊 Total events: ${totalEvents}`)
778
+ console.log()
779
+ console.log(
780
+ ` 🚨 Error rate: ${overallErrorRate.toFixed(2)}% (${totalErrors} errors)`
781
+ )
782
+ console.log()
783
+ console.log(` ⏱ Pipeline duration:`)
784
+ console.log(` Avg: ${formatDuration(avgDuration)}`)
785
+ console.log(` P50: ${formatDuration(p50Duration)}`)
786
+ console.log(` P95: ${formatDuration(p95Duration)}`)
787
+ console.log()
788
+ console.log(
789
+ ` 🏷 Tag success rate: ${tagRate.toFixed(1)}% (${tagSuccess}/${tagTotal})`
790
+ )
791
+ console.log(
792
+ ` ✅ Auto-approval rate: ${autoRate.toFixed(1)}% (${autoApproval}/${approvalTotal})`
793
+ )
794
+
795
+ if (topErrors.length > 0) {
796
+ console.log()
797
+ console.log(' 🔥 Top Error Sources:')
798
+ for (const e of topErrors) {
799
+ console.log(` ${String(e.count).padStart(5)} ${e.name}`)
800
+ }
801
+ }
802
+
803
+ console.log()
804
+ console.log('─'.repeat(60))
805
+ } catch (error) {
806
+ console.error(
807
+ 'Query failed:',
808
+ error instanceof Error ? error.message : error
809
+ )
810
+ process.exit(1)
811
+ }
812
+ }
813
+
814
+ // ---------------------------------------------------------------------------
815
+ // Registration
816
+ // ---------------------------------------------------------------------------
817
+
818
+ export function registerForensicCommands(axiom: Command): void {
819
+ axiom
820
+ .command('pipeline-trace')
821
+ .description('Full trace for a single conversation (pipeline debugging)')
822
+ .argument('<conversationId>', 'Conversation ID to trace')
823
+ .option('-s, --since <time>', 'Time range (e.g., 1h, 24h, 7d)', '7d')
824
+ .option('--json', 'Output as JSON')
825
+ .action(pipelineTrace)
826
+
827
+ axiom
828
+ .command('step-timings')
829
+ .description('P50/P95 duration by step name')
830
+ .option('-s, --since <time>', 'Time range (e.g., 1h, 24h, 7d)', '7d')
831
+ .option('--json', 'Output as JSON')
832
+ .action(stepTimings)
833
+
834
+ axiom
835
+ .command('error-rate')
836
+ .description('Failure rate by step over time window')
837
+ .option('-s, --since <time>', 'Time range (e.g., 1h, 24h, 7d)', '7d')
838
+ .option('--json', 'Output as JSON')
839
+ .action(errorRate)
840
+
841
+ axiom
842
+ .command('data-flow-check')
843
+ .description('Verify field presence at each pipeline boundary')
844
+ .option('-s, --since <time>', 'Time range (e.g., 1h, 24h, 7d)', '7d')
845
+ .option('--json', 'Output as JSON')
846
+ .action(dataFlowCheck)
847
+
848
+ axiom
849
+ .command('tag-health')
850
+ .description('Tag application success/failure breakdown')
851
+ .option('-s, --since <time>', 'Time range (e.g., 1h, 24h, 7d)', '7d')
852
+ .option('--json', 'Output as JSON')
853
+ .action(tagHealth)
854
+
855
+ axiom
856
+ .command('approval-stats')
857
+ .description('Auto-approval vs manual review breakdown')
858
+ .option('-s, --since <time>', 'Time range (e.g., 1h, 24h, 7d)', '7d')
859
+ .option('--json', 'Output as JSON')
860
+ .action(approvalStats)
861
+
862
+ axiom
863
+ .command('pipeline-health')
864
+ .description('Overall pipeline health dashboard (agent-readable)')
865
+ .option('-s, --since <time>', 'Time range (e.g., 1h, 24h, 7d)', '7d')
866
+ .option('--json', 'Output as JSON')
867
+ .action(pipelineHealth)
868
+ }