@skillrecordings/cli 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.encrypted +0 -0
- package/CHANGELOG.md +35 -0
- package/README.md +214 -0
- package/bin/skill.ts +3 -0
- package/data/tt-archive-dataset.json +1 -0
- package/data/validate-test-dataset.json +97 -0
- package/docs/CLI-AUTH.md +504 -0
- package/package.json +38 -0
- package/preload.ts +18 -0
- package/src/__tests__/init.test.ts +74 -0
- package/src/alignment-test.ts +64 -0
- package/src/check-apps.ts +16 -0
- package/src/commands/auth/decrypt.ts +123 -0
- package/src/commands/auth/encrypt.ts +81 -0
- package/src/commands/auth/index.ts +50 -0
- package/src/commands/auth/keygen.ts +41 -0
- package/src/commands/auth/status.ts +164 -0
- package/src/commands/axiom/forensic.ts +868 -0
- package/src/commands/axiom/index.ts +697 -0
- package/src/commands/build-dataset.ts +311 -0
- package/src/commands/db-status.ts +47 -0
- package/src/commands/deploys.ts +219 -0
- package/src/commands/eval-local/compare.ts +171 -0
- package/src/commands/eval-local/health.ts +212 -0
- package/src/commands/eval-local/index.ts +76 -0
- package/src/commands/eval-local/real-tools.ts +416 -0
- package/src/commands/eval-local/run.ts +1168 -0
- package/src/commands/eval-local/score-production.ts +256 -0
- package/src/commands/eval-local/seed.ts +276 -0
- package/src/commands/eval-pipeline/index.ts +53 -0
- package/src/commands/eval-pipeline/real-tools.ts +492 -0
- package/src/commands/eval-pipeline/run.ts +1316 -0
- package/src/commands/eval-pipeline/seed.ts +395 -0
- package/src/commands/eval-prompt.ts +496 -0
- package/src/commands/eval.test.ts +253 -0
- package/src/commands/eval.ts +108 -0
- package/src/commands/faq-classify.ts +460 -0
- package/src/commands/faq-cluster.ts +135 -0
- package/src/commands/faq-extract.ts +249 -0
- package/src/commands/faq-mine.ts +432 -0
- package/src/commands/faq-review.ts +426 -0
- package/src/commands/front/index.ts +351 -0
- package/src/commands/front/pull-conversations.ts +275 -0
- package/src/commands/front/tags.ts +825 -0
- package/src/commands/front-cache.ts +1277 -0
- package/src/commands/front-stats.ts +75 -0
- package/src/commands/health.test.ts +82 -0
- package/src/commands/health.ts +362 -0
- package/src/commands/init.test.ts +89 -0
- package/src/commands/init.ts +106 -0
- package/src/commands/inngest/client.ts +294 -0
- package/src/commands/inngest/events.ts +296 -0
- package/src/commands/inngest/investigate.ts +382 -0
- package/src/commands/inngest/runs.ts +149 -0
- package/src/commands/inngest/signal.ts +143 -0
- package/src/commands/kb-sync.ts +498 -0
- package/src/commands/memory/find.ts +135 -0
- package/src/commands/memory/get.ts +87 -0
- package/src/commands/memory/index.ts +97 -0
- package/src/commands/memory/stats.ts +163 -0
- package/src/commands/memory/store.ts +49 -0
- package/src/commands/memory/vote.ts +159 -0
- package/src/commands/pipeline.ts +127 -0
- package/src/commands/responses.ts +856 -0
- package/src/commands/tools.ts +293 -0
- package/src/commands/wizard.ts +319 -0
- package/src/index.ts +172 -0
- package/src/lib/crypto.ts +56 -0
- package/src/lib/env-loader.ts +206 -0
- package/src/lib/onepassword.ts +137 -0
- package/src/test-agent-local.ts +115 -0
- package/tsconfig.json +11 -0
- package/vitest.config.ts +10 -0
|
@@ -0,0 +1,856 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CLI commands for pulling agent responses for analysis
|
|
3
|
+
*
|
|
4
|
+
* Usage:
|
|
5
|
+
* skill responses list --app total-typescript --limit 50
|
|
6
|
+
* skill responses list --since 2024-01-01 --json
|
|
7
|
+
* skill responses get <actionId> --context
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import { writeFileSync } from 'fs'
|
|
11
|
+
import { createInstrumentedFrontClient } from '@skillrecordings/core/front/instrumented-client'
|
|
12
|
+
import {
|
|
13
|
+
ActionsTable,
|
|
14
|
+
AppsTable,
|
|
15
|
+
ConversationsTable,
|
|
16
|
+
and,
|
|
17
|
+
desc,
|
|
18
|
+
eq,
|
|
19
|
+
getDb,
|
|
20
|
+
gte,
|
|
21
|
+
or,
|
|
22
|
+
} from '@skillrecordings/database'
|
|
23
|
+
import { type Message } from '@skillrecordings/front-sdk'
|
|
24
|
+
import type { Command } from 'commander'
|
|
25
|
+
|
|
26
|
+
type ActionRow = typeof ActionsTable.$inferSelect
|
|
27
|
+
|
|
28
|
+
interface ResponseRecord {
|
|
29
|
+
actionId: string
|
|
30
|
+
appSlug: string
|
|
31
|
+
appName: string
|
|
32
|
+
conversationId: string
|
|
33
|
+
customerEmail: string
|
|
34
|
+
customerName?: string
|
|
35
|
+
customerDisplay: string
|
|
36
|
+
response: string
|
|
37
|
+
category: string
|
|
38
|
+
createdAt: Date
|
|
39
|
+
rating?: 'good' | 'bad'
|
|
40
|
+
ratedBy?: string
|
|
41
|
+
ratedAt?: Date
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
interface ResponseWithContext extends ResponseRecord {
|
|
45
|
+
conversationHistory?: Array<{
|
|
46
|
+
id: string
|
|
47
|
+
isInbound: boolean
|
|
48
|
+
body: string
|
|
49
|
+
createdAt: number
|
|
50
|
+
author?: string
|
|
51
|
+
}>
|
|
52
|
+
triggerMessage?: {
|
|
53
|
+
subject: string
|
|
54
|
+
body: string
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Format timestamp for display
|
|
60
|
+
*/
|
|
61
|
+
function formatDate(date: Date): string {
|
|
62
|
+
return date.toLocaleString('en-US', {
|
|
63
|
+
month: 'short',
|
|
64
|
+
day: 'numeric',
|
|
65
|
+
hour: '2-digit',
|
|
66
|
+
minute: '2-digit',
|
|
67
|
+
})
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Truncate string with ellipsis
|
|
72
|
+
*/
|
|
73
|
+
function truncate(str: string, len: number): string {
|
|
74
|
+
if (!str) return ''
|
|
75
|
+
if (str.length <= len) return str
|
|
76
|
+
return str.slice(0, len - 3) + '...'
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function normalizeParams(raw: unknown): Record<string, unknown> {
|
|
80
|
+
if (typeof raw === 'string') {
|
|
81
|
+
try {
|
|
82
|
+
let parsed: unknown = JSON.parse(raw)
|
|
83
|
+
if (typeof parsed === 'string') {
|
|
84
|
+
try {
|
|
85
|
+
parsed = JSON.parse(parsed)
|
|
86
|
+
} catch {
|
|
87
|
+
return {}
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
if (parsed && typeof parsed === 'object')
|
|
91
|
+
return parsed as Record<string, unknown>
|
|
92
|
+
} catch {
|
|
93
|
+
return {}
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
if (raw && typeof raw === 'object') {
|
|
98
|
+
if (raw instanceof Uint8Array) {
|
|
99
|
+
const text = Buffer.from(raw).toString('utf-8')
|
|
100
|
+
return normalizeParams(text)
|
|
101
|
+
}
|
|
102
|
+
return raw as Record<string, unknown>
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
return {}
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
function normalizeNested(value: unknown): Record<string, unknown> | undefined {
|
|
109
|
+
if (!value) return undefined
|
|
110
|
+
if (typeof value === 'string' || value instanceof Uint8Array) {
|
|
111
|
+
const normalized = normalizeParams(value)
|
|
112
|
+
return Object.keys(normalized).length ? normalized : undefined
|
|
113
|
+
}
|
|
114
|
+
if (typeof value === 'object') {
|
|
115
|
+
return value as Record<string, unknown>
|
|
116
|
+
}
|
|
117
|
+
return undefined
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
function extractText(value: unknown): string {
|
|
121
|
+
if (typeof value === 'string') return value
|
|
122
|
+
if (value && typeof value === 'object') {
|
|
123
|
+
const record = value as Record<string, unknown>
|
|
124
|
+
const content = record.content
|
|
125
|
+
if (typeof content === 'string') return content
|
|
126
|
+
const text = record.text
|
|
127
|
+
if (typeof text === 'string') return text
|
|
128
|
+
const body = record.body
|
|
129
|
+
if (typeof body === 'string') return body
|
|
130
|
+
const html = record.html
|
|
131
|
+
if (typeof html === 'string') return html
|
|
132
|
+
}
|
|
133
|
+
return ''
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
function getResponseText(params: Record<string, unknown>): string {
|
|
137
|
+
return (
|
|
138
|
+
extractText(params.response) ||
|
|
139
|
+
extractText(params.draft) ||
|
|
140
|
+
extractText(params.responseText) ||
|
|
141
|
+
extractText(params.draftText) ||
|
|
142
|
+
''
|
|
143
|
+
)
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
function getCategory(
|
|
147
|
+
actionCategory: string | null,
|
|
148
|
+
params: Record<string, unknown>
|
|
149
|
+
): string {
|
|
150
|
+
const context = normalizeNested(params.context)
|
|
151
|
+
const category = params.category ?? context?.category
|
|
152
|
+
return (
|
|
153
|
+
actionCategory ??
|
|
154
|
+
(typeof category === 'string' ? category : undefined) ??
|
|
155
|
+
'unknown'
|
|
156
|
+
)
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
function getCustomerEmail(
|
|
160
|
+
conversationEmail: string | null | undefined,
|
|
161
|
+
params: Record<string, unknown>
|
|
162
|
+
): string {
|
|
163
|
+
const context = normalizeNested(params.context)
|
|
164
|
+
const gatheredContext = normalizeNested(params.gatheredContext)
|
|
165
|
+
const customer =
|
|
166
|
+
(context?.customer as Record<string, unknown> | undefined) ??
|
|
167
|
+
(gatheredContext?.customer as Record<string, unknown> | undefined) ??
|
|
168
|
+
(params.customer as Record<string, unknown> | undefined)
|
|
169
|
+
|
|
170
|
+
const candidate =
|
|
171
|
+
conversationEmail ??
|
|
172
|
+
(typeof context?.customerEmail === 'string'
|
|
173
|
+
? context.customerEmail
|
|
174
|
+
: undefined) ??
|
|
175
|
+
(typeof params.customerEmail === 'string'
|
|
176
|
+
? params.customerEmail
|
|
177
|
+
: undefined) ??
|
|
178
|
+
(typeof customer?.email === 'string' ? customer.email : undefined) ??
|
|
179
|
+
(typeof params.senderEmail === 'string' ? params.senderEmail : undefined) ??
|
|
180
|
+
(typeof (params as Record<string, unknown>).sender_email === 'string'
|
|
181
|
+
? ((params as Record<string, unknown>).sender_email as string)
|
|
182
|
+
: undefined) ??
|
|
183
|
+
(typeof params.from === 'string' ? params.from : undefined)
|
|
184
|
+
|
|
185
|
+
return candidate ?? 'unknown'
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
function getCustomerName(
|
|
189
|
+
conversationName: string | null | undefined,
|
|
190
|
+
params: Record<string, unknown>
|
|
191
|
+
): string | undefined {
|
|
192
|
+
const context = normalizeNested(params.context)
|
|
193
|
+
const gatheredContext = normalizeNested(params.gatheredContext)
|
|
194
|
+
const customer =
|
|
195
|
+
(context?.customer as Record<string, unknown> | undefined) ??
|
|
196
|
+
(gatheredContext?.customer as Record<string, unknown> | undefined) ??
|
|
197
|
+
(params.customer as Record<string, unknown> | undefined)
|
|
198
|
+
|
|
199
|
+
const candidate =
|
|
200
|
+
conversationName ??
|
|
201
|
+
(typeof params.customerName === 'string'
|
|
202
|
+
? params.customerName
|
|
203
|
+
: undefined) ??
|
|
204
|
+
(typeof customer?.name === 'string' ? customer.name : undefined)
|
|
205
|
+
|
|
206
|
+
return candidate
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
function formatCustomerDisplay(email: string, name?: string): string {
|
|
210
|
+
if (name && email && email !== 'unknown') {
|
|
211
|
+
return `${name} <${email}>`
|
|
212
|
+
}
|
|
213
|
+
return name ?? email ?? 'unknown'
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
async function findResponseFallback(
|
|
217
|
+
db: ReturnType<typeof getDb>,
|
|
218
|
+
actionId: string,
|
|
219
|
+
conversationId: string
|
|
220
|
+
): Promise<{ action: ActionRow; params: Record<string, unknown> } | null> {
|
|
221
|
+
const candidates = await db
|
|
222
|
+
.select()
|
|
223
|
+
.from(ActionsTable)
|
|
224
|
+
.where(eq(ActionsTable.conversation_id, conversationId))
|
|
225
|
+
.orderBy(desc(ActionsTable.created_at))
|
|
226
|
+
.limit(10)
|
|
227
|
+
|
|
228
|
+
for (const candidate of candidates) {
|
|
229
|
+
if (candidate.id === actionId) continue
|
|
230
|
+
if (
|
|
231
|
+
candidate.type !== 'send-draft' &&
|
|
232
|
+
candidate.type !== 'draft-response'
|
|
233
|
+
) {
|
|
234
|
+
continue
|
|
235
|
+
}
|
|
236
|
+
const params = normalizeParams(candidate.parameters)
|
|
237
|
+
const responseText = getResponseText(params)
|
|
238
|
+
if (responseText) {
|
|
239
|
+
return { action: candidate, params }
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
return null
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
/**
|
|
247
|
+
* List recent agent responses
|
|
248
|
+
*/
|
|
249
|
+
async function listResponses(options: {
|
|
250
|
+
app?: string
|
|
251
|
+
limit?: number
|
|
252
|
+
since?: string
|
|
253
|
+
rating?: 'good' | 'bad' | 'unrated'
|
|
254
|
+
json?: boolean
|
|
255
|
+
}): Promise<void> {
|
|
256
|
+
const db = getDb()
|
|
257
|
+
const limit = options.limit || 20
|
|
258
|
+
|
|
259
|
+
try {
|
|
260
|
+
// Build query conditions
|
|
261
|
+
// Support both old 'draft-response' and new 'send-draft' action types
|
|
262
|
+
const conditions = [
|
|
263
|
+
or(
|
|
264
|
+
eq(ActionsTable.type, 'send-draft'),
|
|
265
|
+
eq(ActionsTable.type, 'draft-response')
|
|
266
|
+
),
|
|
267
|
+
]
|
|
268
|
+
|
|
269
|
+
if (options.app) {
|
|
270
|
+
// Lookup app by slug
|
|
271
|
+
const appResults = await db
|
|
272
|
+
.select()
|
|
273
|
+
.from(AppsTable)
|
|
274
|
+
.where(eq(AppsTable.slug, options.app))
|
|
275
|
+
.limit(1)
|
|
276
|
+
|
|
277
|
+
const foundApp = appResults[0]
|
|
278
|
+
if (!foundApp) {
|
|
279
|
+
console.error(`App not found: ${options.app}`)
|
|
280
|
+
process.exit(1)
|
|
281
|
+
}
|
|
282
|
+
conditions.push(eq(ActionsTable.app_id, foundApp.id))
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
if (options.since) {
|
|
286
|
+
const sinceDate = new Date(options.since)
|
|
287
|
+
conditions.push(gte(ActionsTable.created_at, sinceDate))
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
// Query actions with app and conversation info
|
|
291
|
+
const results = await db
|
|
292
|
+
.select({
|
|
293
|
+
action: ActionsTable,
|
|
294
|
+
app: AppsTable,
|
|
295
|
+
conversation: ConversationsTable,
|
|
296
|
+
})
|
|
297
|
+
.from(ActionsTable)
|
|
298
|
+
.leftJoin(AppsTable, eq(ActionsTable.app_id, AppsTable.id))
|
|
299
|
+
.leftJoin(
|
|
300
|
+
ConversationsTable,
|
|
301
|
+
eq(
|
|
302
|
+
ActionsTable.conversation_id,
|
|
303
|
+
ConversationsTable.front_conversation_id
|
|
304
|
+
)
|
|
305
|
+
)
|
|
306
|
+
.where(and(...conditions))
|
|
307
|
+
.orderBy(desc(ActionsTable.created_at))
|
|
308
|
+
.limit(limit)
|
|
309
|
+
|
|
310
|
+
// Transform to response records
|
|
311
|
+
const responses: ResponseRecord[] = []
|
|
312
|
+
|
|
313
|
+
for (const r of results) {
|
|
314
|
+
let params = normalizeParams(r.action.parameters)
|
|
315
|
+
let responseText = getResponseText(params)
|
|
316
|
+
let customerName = getCustomerName(r.conversation?.customer_name, params)
|
|
317
|
+
let customerEmail = getCustomerEmail(
|
|
318
|
+
r.conversation?.customer_email,
|
|
319
|
+
params
|
|
320
|
+
)
|
|
321
|
+
|
|
322
|
+
if (
|
|
323
|
+
(!responseText || customerEmail === 'unknown') &&
|
|
324
|
+
r.action.conversation_id
|
|
325
|
+
) {
|
|
326
|
+
const fallback = await findResponseFallback(
|
|
327
|
+
db,
|
|
328
|
+
r.action.id,
|
|
329
|
+
r.action.conversation_id
|
|
330
|
+
)
|
|
331
|
+
if (fallback) {
|
|
332
|
+
params = fallback.params
|
|
333
|
+
responseText = responseText || getResponseText(params)
|
|
334
|
+
customerName =
|
|
335
|
+
customerName ??
|
|
336
|
+
getCustomerName(r.conversation?.customer_name, params)
|
|
337
|
+
if (customerEmail === 'unknown') {
|
|
338
|
+
customerEmail = getCustomerEmail(
|
|
339
|
+
r.conversation?.customer_email,
|
|
340
|
+
params
|
|
341
|
+
)
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
// Determine rating from approved_by/rejected_by
|
|
347
|
+
let rating: 'good' | 'bad' | undefined
|
|
348
|
+
let ratedBy: string | undefined
|
|
349
|
+
let ratedAt: Date | undefined
|
|
350
|
+
|
|
351
|
+
if (r.action.approved_by) {
|
|
352
|
+
rating = 'good'
|
|
353
|
+
ratedBy = r.action.approved_by
|
|
354
|
+
ratedAt = r.action.approved_at ?? undefined
|
|
355
|
+
} else if (r.action.rejected_by) {
|
|
356
|
+
rating = 'bad'
|
|
357
|
+
ratedBy = r.action.rejected_by
|
|
358
|
+
ratedAt = r.action.rejected_at ?? undefined
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
responses.push({
|
|
362
|
+
actionId: r.action.id,
|
|
363
|
+
appSlug: r.app?.slug ?? 'unknown',
|
|
364
|
+
appName: r.app?.name ?? 'Unknown App',
|
|
365
|
+
conversationId: r.action.conversation_id ?? '',
|
|
366
|
+
customerEmail,
|
|
367
|
+
customerName,
|
|
368
|
+
customerDisplay: formatCustomerDisplay(customerEmail, customerName),
|
|
369
|
+
response: responseText,
|
|
370
|
+
category: getCategory(r.action.category, params),
|
|
371
|
+
createdAt: r.action.created_at ?? new Date(),
|
|
372
|
+
rating,
|
|
373
|
+
ratedBy,
|
|
374
|
+
ratedAt,
|
|
375
|
+
})
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
// Filter by rating if specified
|
|
379
|
+
let filteredResponses = responses
|
|
380
|
+
if (options.rating === 'good') {
|
|
381
|
+
filteredResponses = responses.filter((r) => r.rating === 'good')
|
|
382
|
+
} else if (options.rating === 'bad') {
|
|
383
|
+
filteredResponses = responses.filter((r) => r.rating === 'bad')
|
|
384
|
+
} else if (options.rating === 'unrated') {
|
|
385
|
+
filteredResponses = responses.filter((r) => !r.rating)
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
if (options.json) {
|
|
389
|
+
console.log(JSON.stringify(filteredResponses, null, 2))
|
|
390
|
+
return
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
// Display table
|
|
394
|
+
console.log('\n📝 Agent Responses')
|
|
395
|
+
console.log('='.repeat(80))
|
|
396
|
+
|
|
397
|
+
for (const r of filteredResponses) {
|
|
398
|
+
const ratingIcon =
|
|
399
|
+
r.rating === 'good' ? '👍' : r.rating === 'bad' ? '👎' : '⏳'
|
|
400
|
+
console.log(`\n${ratingIcon} [${formatDate(r.createdAt)}] ${r.appSlug}`)
|
|
401
|
+
console.log(` Customer: ${r.customerDisplay}`)
|
|
402
|
+
console.log(` Category: ${r.category}`)
|
|
403
|
+
console.log(
|
|
404
|
+
` Response: ${truncate(r.response.replace(/\n/g, ' '), 200)}`
|
|
405
|
+
)
|
|
406
|
+
console.log(` ID: ${r.actionId}`)
|
|
407
|
+
if (r.rating) {
|
|
408
|
+
console.log(` Rated: ${r.rating} by ${r.ratedBy}`)
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
console.log('\n' + '-'.repeat(80))
|
|
413
|
+
console.log(`Total: ${filteredResponses.length} responses`)
|
|
414
|
+
console.log(
|
|
415
|
+
` 👍 Good: ${filteredResponses.filter((r) => r.rating === 'good').length}`
|
|
416
|
+
)
|
|
417
|
+
console.log(
|
|
418
|
+
` 👎 Bad: ${filteredResponses.filter((r) => r.rating === 'bad').length}`
|
|
419
|
+
)
|
|
420
|
+
console.log(
|
|
421
|
+
` ⏳ Unrated: ${filteredResponses.filter((r) => !r.rating).length}`
|
|
422
|
+
)
|
|
423
|
+
console.log('')
|
|
424
|
+
} catch (error) {
|
|
425
|
+
console.error(
|
|
426
|
+
'Error:',
|
|
427
|
+
error instanceof Error ? error.message : 'Unknown error'
|
|
428
|
+
)
|
|
429
|
+
process.exit(1)
|
|
430
|
+
}
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
/**
|
|
434
|
+
* Get a specific response with full context
|
|
435
|
+
*/
|
|
436
|
+
async function getResponse(
|
|
437
|
+
actionId: string,
|
|
438
|
+
options: { context?: boolean; json?: boolean }
|
|
439
|
+
): Promise<void> {
|
|
440
|
+
const db = getDb()
|
|
441
|
+
|
|
442
|
+
try {
|
|
443
|
+
// Fetch the action with related data
|
|
444
|
+
const results = await db
|
|
445
|
+
.select({
|
|
446
|
+
action: ActionsTable,
|
|
447
|
+
app: AppsTable,
|
|
448
|
+
conversation: ConversationsTable,
|
|
449
|
+
})
|
|
450
|
+
.from(ActionsTable)
|
|
451
|
+
.leftJoin(AppsTable, eq(ActionsTable.app_id, AppsTable.id))
|
|
452
|
+
.leftJoin(
|
|
453
|
+
ConversationsTable,
|
|
454
|
+
eq(
|
|
455
|
+
ActionsTable.conversation_id,
|
|
456
|
+
ConversationsTable.front_conversation_id
|
|
457
|
+
)
|
|
458
|
+
)
|
|
459
|
+
.where(eq(ActionsTable.id, actionId))
|
|
460
|
+
.limit(1)
|
|
461
|
+
|
|
462
|
+
const r = results[0]
|
|
463
|
+
if (!r) {
|
|
464
|
+
console.error(`Response not found: ${actionId}`)
|
|
465
|
+
process.exit(1)
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
let params = normalizeParams(r.action.parameters)
|
|
469
|
+
let responseText = getResponseText(params)
|
|
470
|
+
let customerName = getCustomerName(r.conversation?.customer_name, params)
|
|
471
|
+
let customerEmail = getCustomerEmail(r.conversation?.customer_email, params)
|
|
472
|
+
|
|
473
|
+
if (
|
|
474
|
+
(!responseText || customerEmail === 'unknown') &&
|
|
475
|
+
r.action.conversation_id
|
|
476
|
+
) {
|
|
477
|
+
const fallback = await findResponseFallback(
|
|
478
|
+
db,
|
|
479
|
+
r.action.id,
|
|
480
|
+
r.action.conversation_id
|
|
481
|
+
)
|
|
482
|
+
if (fallback) {
|
|
483
|
+
params = fallback.params
|
|
484
|
+
responseText = responseText || getResponseText(params)
|
|
485
|
+
customerName =
|
|
486
|
+
customerName ?? getCustomerName(r.conversation?.customer_name, params)
|
|
487
|
+
if (customerEmail === 'unknown') {
|
|
488
|
+
customerEmail = getCustomerEmail(
|
|
489
|
+
r.conversation?.customer_email,
|
|
490
|
+
params
|
|
491
|
+
)
|
|
492
|
+
}
|
|
493
|
+
}
|
|
494
|
+
}
|
|
495
|
+
|
|
496
|
+
let rating: 'good' | 'bad' | undefined
|
|
497
|
+
let ratedBy: string | undefined
|
|
498
|
+
let ratedAt: Date | undefined
|
|
499
|
+
|
|
500
|
+
if (r.action.approved_by) {
|
|
501
|
+
rating = 'good'
|
|
502
|
+
ratedBy = r.action.approved_by
|
|
503
|
+
ratedAt = r.action.approved_at ?? undefined
|
|
504
|
+
} else if (r.action.rejected_by) {
|
|
505
|
+
rating = 'bad'
|
|
506
|
+
ratedBy = r.action.rejected_by
|
|
507
|
+
ratedAt = r.action.rejected_at ?? undefined
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
const response: ResponseWithContext = {
|
|
511
|
+
actionId: r.action.id,
|
|
512
|
+
appSlug: r.app?.slug ?? 'unknown',
|
|
513
|
+
appName: r.app?.name ?? 'Unknown App',
|
|
514
|
+
conversationId: r.action.conversation_id ?? '',
|
|
515
|
+
customerEmail,
|
|
516
|
+
customerName,
|
|
517
|
+
customerDisplay: formatCustomerDisplay(customerEmail, customerName),
|
|
518
|
+
response: responseText,
|
|
519
|
+
category: getCategory(r.action.category, params),
|
|
520
|
+
createdAt: r.action.created_at ?? new Date(),
|
|
521
|
+
rating,
|
|
522
|
+
ratedBy,
|
|
523
|
+
ratedAt,
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
// Fetch conversation context from Front if requested
|
|
527
|
+
if (options.context && r.action.conversation_id) {
|
|
528
|
+
const frontToken = process.env.FRONT_API_TOKEN
|
|
529
|
+
if (frontToken) {
|
|
530
|
+
try {
|
|
531
|
+
const front = createInstrumentedFrontClient({ apiToken: frontToken })
|
|
532
|
+
const messageList = (await front.conversations.listMessages(
|
|
533
|
+
r.action.conversation_id
|
|
534
|
+
)) as { _results?: Message[] }
|
|
535
|
+
const messages = messageList._results ?? []
|
|
536
|
+
|
|
537
|
+
response.conversationHistory = messages.map((m) => ({
|
|
538
|
+
id: m.id,
|
|
539
|
+
isInbound: m.is_inbound,
|
|
540
|
+
body:
|
|
541
|
+
m.text ||
|
|
542
|
+
m.body
|
|
543
|
+
.replace(/<[^>]*>/g, ' ')
|
|
544
|
+
.replace(/\s+/g, ' ')
|
|
545
|
+
.trim(),
|
|
546
|
+
createdAt: m.created_at,
|
|
547
|
+
author: m.author?.email,
|
|
548
|
+
}))
|
|
549
|
+
|
|
550
|
+
// Find the trigger message (most recent inbound before draft creation)
|
|
551
|
+
const draftTime = r.action.created_at?.getTime() ?? Date.now()
|
|
552
|
+
const inboundBefore = messages
|
|
553
|
+
.filter((m) => m.is_inbound && m.created_at * 1000 < draftTime)
|
|
554
|
+
.sort((a, b) => b.created_at - a.created_at)
|
|
555
|
+
|
|
556
|
+
const trigger = inboundBefore[0]
|
|
557
|
+
if (trigger) {
|
|
558
|
+
response.triggerMessage = {
|
|
559
|
+
subject: trigger.subject ?? '',
|
|
560
|
+
body:
|
|
561
|
+
trigger.text ??
|
|
562
|
+
trigger.body
|
|
563
|
+
.replace(/<[^>]*>/g, ' ')
|
|
564
|
+
.replace(/\s+/g, ' ')
|
|
565
|
+
.trim(),
|
|
566
|
+
}
|
|
567
|
+
}
|
|
568
|
+
} catch (err) {
|
|
569
|
+
console.error('[warn] Failed to fetch Front context:', err)
|
|
570
|
+
}
|
|
571
|
+
} else {
|
|
572
|
+
console.error('[warn] FRONT_API_TOKEN not set, skipping context fetch')
|
|
573
|
+
}
|
|
574
|
+
}
|
|
575
|
+
|
|
576
|
+
if (options.json) {
|
|
577
|
+
console.log(JSON.stringify(response, null, 2))
|
|
578
|
+
return
|
|
579
|
+
}
|
|
580
|
+
|
|
581
|
+
// Display detailed view
|
|
582
|
+
const ratingIcon =
|
|
583
|
+
response.rating === 'good'
|
|
584
|
+
? '👍'
|
|
585
|
+
: response.rating === 'bad'
|
|
586
|
+
? '👎'
|
|
587
|
+
: '⏳'
|
|
588
|
+
|
|
589
|
+
console.log('\n📝 Agent Response Details')
|
|
590
|
+
console.log('='.repeat(80))
|
|
591
|
+
console.log(`ID: ${response.actionId}`)
|
|
592
|
+
console.log(`App: ${response.appName} (${response.appSlug})`)
|
|
593
|
+
console.log(`Customer: ${response.customerDisplay}`)
|
|
594
|
+
console.log(`Category: ${response.category}`)
|
|
595
|
+
console.log(`Created: ${formatDate(response.createdAt)}`)
|
|
596
|
+
console.log(`Rating: ${ratingIcon} ${response.rating ?? 'unrated'}`)
|
|
597
|
+
if (response.ratedBy) {
|
|
598
|
+
console.log(`Rated by: ${response.ratedBy}`)
|
|
599
|
+
}
|
|
600
|
+
|
|
601
|
+
if (response.triggerMessage) {
|
|
602
|
+
console.log('\n--- Trigger Message ---')
|
|
603
|
+
if (response.triggerMessage.subject) {
|
|
604
|
+
console.log(`Subject: ${response.triggerMessage.subject}`)
|
|
605
|
+
}
|
|
606
|
+
console.log(response.triggerMessage.body)
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
console.log('\n--- Agent Response ---')
|
|
610
|
+
console.log(response.response)
|
|
611
|
+
|
|
612
|
+
if (response.conversationHistory?.length) {
|
|
613
|
+
console.log('\n--- Conversation History ---')
|
|
614
|
+
for (const msg of response.conversationHistory) {
|
|
615
|
+
const dir = msg.isInbound ? '← IN' : '→ OUT'
|
|
616
|
+
const time = new Date(msg.createdAt * 1000).toLocaleString()
|
|
617
|
+
console.log(`\n[${dir}] ${time} - ${msg.author ?? 'unknown'}`)
|
|
618
|
+
console.log(truncate(msg.body, 500))
|
|
619
|
+
}
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
console.log('')
|
|
623
|
+
} catch (error) {
|
|
624
|
+
console.error(
|
|
625
|
+
'Error:',
|
|
626
|
+
error instanceof Error ? error.message : 'Unknown error'
|
|
627
|
+
)
|
|
628
|
+
process.exit(1)
|
|
629
|
+
}
|
|
630
|
+
}
|
|
631
|
+
|
|
632
|
+
/**
|
|
633
|
+
* Export responses to a file for eval/analysis
|
|
634
|
+
*/
|
|
635
|
+
async function exportResponses(options: {
|
|
636
|
+
app?: string
|
|
637
|
+
since?: string
|
|
638
|
+
output?: string
|
|
639
|
+
rating?: 'good' | 'bad' | 'all'
|
|
640
|
+
}): Promise<void> {
|
|
641
|
+
const db = getDb()
|
|
642
|
+
|
|
643
|
+
try {
|
|
644
|
+
// Build query conditions
|
|
645
|
+
// Support both old 'draft-response' and new 'send-draft' action types
|
|
646
|
+
const conditions = [
|
|
647
|
+
or(
|
|
648
|
+
eq(ActionsTable.type, 'send-draft'),
|
|
649
|
+
eq(ActionsTable.type, 'draft-response')
|
|
650
|
+
),
|
|
651
|
+
]
|
|
652
|
+
|
|
653
|
+
if (options.app) {
|
|
654
|
+
const appResults = await db
|
|
655
|
+
.select()
|
|
656
|
+
.from(AppsTable)
|
|
657
|
+
.where(eq(AppsTable.slug, options.app))
|
|
658
|
+
.limit(1)
|
|
659
|
+
|
|
660
|
+
const foundApp = appResults[0]
|
|
661
|
+
if (!foundApp) {
|
|
662
|
+
console.error(`App not found: ${options.app}`)
|
|
663
|
+
process.exit(1)
|
|
664
|
+
}
|
|
665
|
+
conditions.push(eq(ActionsTable.app_id, foundApp.id))
|
|
666
|
+
}
|
|
667
|
+
|
|
668
|
+
if (options.since) {
|
|
669
|
+
const sinceDate = new Date(options.since)
|
|
670
|
+
conditions.push(gte(ActionsTable.created_at, sinceDate))
|
|
671
|
+
}
|
|
672
|
+
|
|
673
|
+
// Query all matching actions
|
|
674
|
+
const results = await db
|
|
675
|
+
.select({
|
|
676
|
+
action: ActionsTable,
|
|
677
|
+
app: AppsTable,
|
|
678
|
+
conversation: ConversationsTable,
|
|
679
|
+
})
|
|
680
|
+
.from(ActionsTable)
|
|
681
|
+
.leftJoin(AppsTable, eq(ActionsTable.app_id, AppsTable.id))
|
|
682
|
+
.leftJoin(
|
|
683
|
+
ConversationsTable,
|
|
684
|
+
eq(
|
|
685
|
+
ActionsTable.conversation_id,
|
|
686
|
+
ConversationsTable.front_conversation_id
|
|
687
|
+
)
|
|
688
|
+
)
|
|
689
|
+
.where(and(...conditions))
|
|
690
|
+
.orderBy(desc(ActionsTable.created_at))
|
|
691
|
+
|
|
692
|
+
// Fetch Front context for each
|
|
693
|
+
const frontToken = process.env.FRONT_API_TOKEN
|
|
694
|
+
const front = frontToken
|
|
695
|
+
? createInstrumentedFrontClient({ apiToken: frontToken })
|
|
696
|
+
: null
|
|
697
|
+
|
|
698
|
+
const exportData: ResponseWithContext[] = []
|
|
699
|
+
|
|
700
|
+
for (const r of results) {
|
|
701
|
+
let params = normalizeParams(r.action.parameters)
|
|
702
|
+
let responseText = getResponseText(params)
|
|
703
|
+
let customerName = getCustomerName(r.conversation?.customer_name, params)
|
|
704
|
+
let customerEmail = getCustomerEmail(
|
|
705
|
+
r.conversation?.customer_email,
|
|
706
|
+
params
|
|
707
|
+
)
|
|
708
|
+
|
|
709
|
+
if (
|
|
710
|
+
(!responseText || customerEmail === 'unknown') &&
|
|
711
|
+
r.action.conversation_id
|
|
712
|
+
) {
|
|
713
|
+
const fallback = await findResponseFallback(
|
|
714
|
+
db,
|
|
715
|
+
r.action.id,
|
|
716
|
+
r.action.conversation_id
|
|
717
|
+
)
|
|
718
|
+
if (fallback) {
|
|
719
|
+
params = fallback.params
|
|
720
|
+
responseText = responseText || getResponseText(params)
|
|
721
|
+
customerName =
|
|
722
|
+
customerName ??
|
|
723
|
+
getCustomerName(r.conversation?.customer_name, params)
|
|
724
|
+
if (customerEmail === 'unknown') {
|
|
725
|
+
customerEmail = getCustomerEmail(
|
|
726
|
+
r.conversation?.customer_email,
|
|
727
|
+
params
|
|
728
|
+
)
|
|
729
|
+
}
|
|
730
|
+
}
|
|
731
|
+
}
|
|
732
|
+
|
|
733
|
+
let rating: 'good' | 'bad' | undefined
|
|
734
|
+
if (r.action.approved_by) rating = 'good'
|
|
735
|
+
else if (r.action.rejected_by) rating = 'bad'
|
|
736
|
+
|
|
737
|
+
// Filter by rating
|
|
738
|
+
if (options.rating === 'good' && rating !== 'good') continue
|
|
739
|
+
if (options.rating === 'bad' && rating !== 'bad') continue
|
|
740
|
+
|
|
741
|
+
const record: ResponseWithContext = {
|
|
742
|
+
actionId: r.action.id,
|
|
743
|
+
appSlug: r.app?.slug ?? 'unknown',
|
|
744
|
+
appName: r.app?.name ?? 'Unknown App',
|
|
745
|
+
conversationId: r.action.conversation_id ?? '',
|
|
746
|
+
customerEmail,
|
|
747
|
+
customerName,
|
|
748
|
+
customerDisplay: formatCustomerDisplay(customerEmail, customerName),
|
|
749
|
+
response: responseText,
|
|
750
|
+
category: getCategory(r.action.category, params),
|
|
751
|
+
createdAt: r.action.created_at ?? new Date(),
|
|
752
|
+
rating,
|
|
753
|
+
ratedBy: r.action.approved_by ?? r.action.rejected_by ?? undefined,
|
|
754
|
+
ratedAt: r.action.approved_at ?? r.action.rejected_at ?? undefined,
|
|
755
|
+
}
|
|
756
|
+
|
|
757
|
+
// Fetch context
|
|
758
|
+
if (front && r.action.conversation_id) {
|
|
759
|
+
try {
|
|
760
|
+
const messageList = (await front.conversations.listMessages(
|
|
761
|
+
r.action.conversation_id
|
|
762
|
+
)) as { _results?: Message[] }
|
|
763
|
+
const messages = messageList._results ?? []
|
|
764
|
+
|
|
765
|
+
record.conversationHistory = messages.map((m) => ({
|
|
766
|
+
id: m.id,
|
|
767
|
+
isInbound: m.is_inbound,
|
|
768
|
+
body:
|
|
769
|
+
m.text ??
|
|
770
|
+
m.body
|
|
771
|
+
.replace(/<[^>]*>/g, ' ')
|
|
772
|
+
.replace(/\s+/g, ' ')
|
|
773
|
+
.trim(),
|
|
774
|
+
createdAt: m.created_at,
|
|
775
|
+
author: m.author?.email,
|
|
776
|
+
}))
|
|
777
|
+
|
|
778
|
+
const draftTime = r.action.created_at?.getTime() ?? Date.now()
|
|
779
|
+
const inboundBefore = messages
|
|
780
|
+
.filter((m) => m.is_inbound && m.created_at * 1000 < draftTime)
|
|
781
|
+
.sort((a, b) => b.created_at - a.created_at)
|
|
782
|
+
|
|
783
|
+
const trigger = inboundBefore[0]
|
|
784
|
+
if (trigger) {
|
|
785
|
+
record.triggerMessage = {
|
|
786
|
+
subject: trigger.subject ?? '',
|
|
787
|
+
body:
|
|
788
|
+
trigger.text ??
|
|
789
|
+
trigger.body
|
|
790
|
+
.replace(/<[^>]*>/g, ' ')
|
|
791
|
+
.replace(/\s+/g, ' ')
|
|
792
|
+
.trim(),
|
|
793
|
+
}
|
|
794
|
+
}
|
|
795
|
+
} catch {
|
|
796
|
+
// Skip context fetch failures
|
|
797
|
+
}
|
|
798
|
+
}
|
|
799
|
+
|
|
800
|
+
exportData.push(record)
|
|
801
|
+
}
|
|
802
|
+
|
|
803
|
+
const outputJson = JSON.stringify(exportData, null, 2)
|
|
804
|
+
|
|
805
|
+
if (options.output) {
|
|
806
|
+
writeFileSync(options.output, outputJson, 'utf-8')
|
|
807
|
+
console.log(
|
|
808
|
+
`Exported ${exportData.length} responses to ${options.output}`
|
|
809
|
+
)
|
|
810
|
+
} else {
|
|
811
|
+
console.log(outputJson)
|
|
812
|
+
}
|
|
813
|
+
} catch (error) {
|
|
814
|
+
console.error(
|
|
815
|
+
'Error:',
|
|
816
|
+
error instanceof Error ? error.message : 'Unknown error'
|
|
817
|
+
)
|
|
818
|
+
process.exit(1)
|
|
819
|
+
}
|
|
820
|
+
}
|
|
821
|
+
|
|
822
|
+
/**
|
|
823
|
+
* Register response commands with Commander
|
|
824
|
+
*/
|
|
825
|
+
export function registerResponseCommands(program: Command): void {
|
|
826
|
+
const responses = program
|
|
827
|
+
.command('responses')
|
|
828
|
+
.description('Pull agent responses for analysis')
|
|
829
|
+
|
|
830
|
+
responses
|
|
831
|
+
.command('list')
|
|
832
|
+
.description('List recent agent responses')
|
|
833
|
+
.option('-a, --app <slug>', 'Filter by app slug')
|
|
834
|
+
.option('-l, --limit <n>', 'Number of responses (default: 20)', parseInt)
|
|
835
|
+
.option('-s, --since <date>', 'Filter responses since date (YYYY-MM-DD)')
|
|
836
|
+
.option('-r, --rating <type>', 'Filter by rating (good, bad, unrated)')
|
|
837
|
+
.option('--json', 'Output as JSON')
|
|
838
|
+
.action(listResponses)
|
|
839
|
+
|
|
840
|
+
responses
|
|
841
|
+
.command('get')
|
|
842
|
+
.description('Get a specific response with details')
|
|
843
|
+
.argument('<actionId>', 'Action ID of the response')
|
|
844
|
+
.option('-c, --context', 'Include conversation context from Front')
|
|
845
|
+
.option('--json', 'Output as JSON')
|
|
846
|
+
.action(getResponse)
|
|
847
|
+
|
|
848
|
+
responses
|
|
849
|
+
.command('export')
|
|
850
|
+
.description('Export responses with context for analysis')
|
|
851
|
+
.option('-a, --app <slug>', 'Filter by app slug')
|
|
852
|
+
.option('-s, --since <date>', 'Filter responses since date (YYYY-MM-DD)')
|
|
853
|
+
.option('-r, --rating <type>', 'Filter by rating (good, bad, all)')
|
|
854
|
+
.option('-o, --output <file>', 'Output file path')
|
|
855
|
+
.action(exportResponses)
|
|
856
|
+
}
|