@frontera-sdk/cli 1.52.23 → 1.52.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +4 -4
- package/src/api/dataset-api.ts +8 -4
- package/src/automation-template.ts +3 -0
- package/src/blueprint/tree.ts +27 -0
- package/src/commands/local/check.ts +64 -0
- package/src/commands/local/dev.ts +301 -0
- package/src/commands/local/docs-mcp.ts +36 -0
- package/src/commands/local/index-commands.ts +477 -0
- package/src/commands/local/init.ts +78 -0
- package/src/commands/local/key.ts +108 -0
- package/src/commands/local/providers.ts +245 -0
- package/src/commands/local/schema.ts +46 -0
- package/src/commands/registry.ts +6 -4
- package/src/commands/types.ts +7 -0
- package/src/flag-help.ts +18 -0
- package/src/harness.ts +1 -1
- package/src/local/local-agent-files.ts +122 -0
- package/src/local/local-applied.ts +135 -0
- package/src/local/local-apply-error.ts +35 -0
- package/src/local/local-control.ts +143 -0
- package/src/local/local-dev-sync.ts +256 -0
- package/src/local/local-docker.ts +146 -0
- package/src/local/local-docs-search.ts +126 -0
- package/src/local/local-docs.ts +241 -0
- package/src/local/local-embeddings.ts +79 -0
- package/src/local/local-env.ts +225 -0
- package/src/local/local-keys.ts +115 -0
- package/src/local/local-operating.ts +82 -0
- package/src/local/local-ports.ts +111 -0
- package/src/local/local-process.ts +106 -0
- package/src/local/local-project.ts +54 -0
- package/src/local/local-providers.ts +268 -0
- package/src/local/local-scaffold.ts +404 -0
- package/src/local/local-schemas.ts +301 -0
- package/src/local/local-seed.ts +332 -0
- package/src/local/local-state.ts +165 -0
- package/src/local/local-studio-assets.ts +68 -0
- package/src/local/local-studio-server.ts +304 -0
- package/src/local/local-supervisor.ts +561 -0
- package/src/local/local-test-cases.ts +58 -0
- package/src/local/local-test-runner.ts +191 -0
- package/src/local/local-test-runs.ts +75 -0
- package/src/local/local-traces.ts +322 -0
- package/src/main.ts +1 -1
- package/src/packaging.ts +5 -0
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
import { existsSync, readdirSync, readFileSync } from 'node:fs'
|
|
2
|
+
import { join, relative } from 'node:path'
|
|
3
|
+
|
|
4
|
+
import { parse } from 'yaml'
|
|
5
|
+
|
|
6
|
+
import type { LocalCredentials } from './local-state'
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Run an Agent's saved regression cases against the local runtime.
|
|
10
|
+
*
|
|
11
|
+
* Each case in `agents/<slug>/evals/*.yaml` is a conversation and the reply it
|
|
12
|
+
* should end with. The runner opens a fresh conversation with the Agent's
|
|
13
|
+
* draft (or the live version), sends the person's turns in order through the
|
|
14
|
+
* same chat endpoint the platform uses, and compares the final reply with the
|
|
15
|
+
* expectation.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
export interface TestCase {
|
|
19
|
+
name: string
|
|
20
|
+
conversation: Array<Record<string, string>>
|
|
21
|
+
expected?: string
|
|
22
|
+
/** Pass when the reply contains each of these, instead of matching exactly. */
|
|
23
|
+
expectedContains?: string[]
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export interface CaseResult {
|
|
27
|
+
agent: string
|
|
28
|
+
file: string
|
|
29
|
+
name: string
|
|
30
|
+
passed: boolean
|
|
31
|
+
reply: string
|
|
32
|
+
reason?: string
|
|
33
|
+
ms: number
|
|
34
|
+
/** The Agent's config id and the conversation the case ran in, so Studio can open it. */
|
|
35
|
+
agentId?: string
|
|
36
|
+
conversationId?: string
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export interface CaseFile {
|
|
40
|
+
agent: string
|
|
41
|
+
file: string
|
|
42
|
+
cases: TestCase[]
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export function findCaseFiles(root: string, agent?: string): CaseFile[] {
|
|
46
|
+
const agentsDir = join(root, 'agents')
|
|
47
|
+
if (!existsSync(agentsDir)) return []
|
|
48
|
+
const out: CaseFile[] = []
|
|
49
|
+
for (const slug of readdirSync(agentsDir)) {
|
|
50
|
+
if (agent && slug !== agent) continue
|
|
51
|
+
const evalsDir = join(agentsDir, slug, 'evals')
|
|
52
|
+
if (!existsSync(evalsDir)) continue
|
|
53
|
+
for (const name of readdirSync(evalsDir)) {
|
|
54
|
+
if (!/\.ya?ml$/.test(name)) continue
|
|
55
|
+
const path = join(evalsDir, name)
|
|
56
|
+
const doc = parse(readFileSync(path, 'utf8')) as { cases?: TestCase[] } | null
|
|
57
|
+
out.push({ agent: slug, file: relative(root, path), cases: doc?.cases ?? [] })
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
return out
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function normalise(text: string): string {
|
|
64
|
+
return text.replace(/\s+/g, ' ').trim()
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export function judge(c: TestCase, reply: string): { passed: boolean; reason?: string } {
|
|
68
|
+
if (c.expectedContains?.length) {
|
|
69
|
+
const missing = c.expectedContains.filter((s) => !normalise(reply).toLowerCase().includes(normalise(s).toLowerCase()))
|
|
70
|
+
return missing.length ? { passed: false, reason: `reply does not mention: ${missing.join(', ')}` } : { passed: true }
|
|
71
|
+
}
|
|
72
|
+
if (c.expected === undefined) return { passed: false, reason: 'the case has no `expected` or `expectedContains`' }
|
|
73
|
+
return normalise(reply) === normalise(c.expected) ? { passed: true } : { passed: false, reason: 'reply differs from `expected`' }
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
interface Api {
|
|
77
|
+
serviceUrl: string
|
|
78
|
+
creds: LocalCredentials
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
async function call<T>(api: Api, path: string, init: { method?: string; body?: unknown } = {}): Promise<T> {
|
|
82
|
+
const res = await fetch(new URL(path, api.serviceUrl), {
|
|
83
|
+
method: init.method ?? (init.body === undefined ? 'GET' : 'POST'),
|
|
84
|
+
headers: {
|
|
85
|
+
authorization: `Bearer ${api.creds.session}`,
|
|
86
|
+
'x-org-id': api.creds.organizationId,
|
|
87
|
+
'x-workspace-id': api.creds.workspaceId,
|
|
88
|
+
'content-type': 'application/json',
|
|
89
|
+
},
|
|
90
|
+
body: init.body === undefined ? undefined : JSON.stringify(init.body),
|
|
91
|
+
signal: AbortSignal.timeout(60_000),
|
|
92
|
+
})
|
|
93
|
+
const text = await res.text()
|
|
94
|
+
if (!res.ok) throw new Error(`${path} answered ${res.status}: ${text.slice(0, 200)}`)
|
|
95
|
+
const parsed = text ? (JSON.parse(text) as { data?: T }) : {}
|
|
96
|
+
return (parsed.data ?? parsed) as T
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
const TURN_TIMEOUT_MS = 180_000
|
|
100
|
+
|
|
101
|
+
/** Send one person turn and collect the streamed reply's text. */
|
|
102
|
+
async function sendTurn(api: Api, conversationId: string, agentName: string, text: string): Promise<string> {
|
|
103
|
+
const res = await fetch(new URL('/v1/chat', api.serviceUrl), {
|
|
104
|
+
method: 'POST',
|
|
105
|
+
headers: {
|
|
106
|
+
authorization: `Bearer ${api.creds.session}`,
|
|
107
|
+
'x-org-id': api.creds.organizationId,
|
|
108
|
+
'x-workspace-id': api.creds.workspaceId,
|
|
109
|
+
'content-type': 'application/json',
|
|
110
|
+
},
|
|
111
|
+
body: JSON.stringify({
|
|
112
|
+
conversationId,
|
|
113
|
+
agentName,
|
|
114
|
+
messages: [{ id: crypto.randomUUID(), role: 'user', parts: [{ type: 'text', text }] }],
|
|
115
|
+
}),
|
|
116
|
+
// A reply that never finishes (the stream's keepalives hold it open) fails the case, not the whole run.
|
|
117
|
+
signal: AbortSignal.timeout(TURN_TIMEOUT_MS),
|
|
118
|
+
})
|
|
119
|
+
if (!res.ok || !res.body) throw new Error(`chat answered ${res.status}: ${(await res.text()).slice(0, 200)}`)
|
|
120
|
+
let reply = ''
|
|
121
|
+
let buffer = ''
|
|
122
|
+
const decoder = new TextDecoder()
|
|
123
|
+
for await (const chunk of res.body as unknown as AsyncIterable<Uint8Array>) {
|
|
124
|
+
buffer += decoder.decode(chunk, { stream: true })
|
|
125
|
+
let newline: number
|
|
126
|
+
while ((newline = buffer.indexOf('\n')) >= 0) {
|
|
127
|
+
const line = buffer.slice(0, newline).trim()
|
|
128
|
+
buffer = buffer.slice(newline + 1)
|
|
129
|
+
if (!line.startsWith('data:')) continue
|
|
130
|
+
const payload = line.slice(5).trim()
|
|
131
|
+
if (payload === '[DONE]') continue
|
|
132
|
+
try {
|
|
133
|
+
const event = JSON.parse(payload) as { type?: string; delta?: string; errorText?: string }
|
|
134
|
+
if (event.type === 'text-delta' && typeof event.delta === 'string') reply += event.delta
|
|
135
|
+
if (event.type === 'error') throw new Error(event.errorText ?? 'the turn failed')
|
|
136
|
+
} catch (err) {
|
|
137
|
+
if (err instanceof SyntaxError) continue
|
|
138
|
+
throw err
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
return reply
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
export async function runCases(args: {
|
|
146
|
+
serviceUrl: string
|
|
147
|
+
creds: LocalCredentials
|
|
148
|
+
files: CaseFile[]
|
|
149
|
+
live: boolean
|
|
150
|
+
onResult: (r: CaseResult) => void
|
|
151
|
+
}): Promise<CaseResult[]> {
|
|
152
|
+
const api: Api = { serviceUrl: args.serviceUrl, creds: args.creds }
|
|
153
|
+
const results: CaseResult[] = []
|
|
154
|
+
for (const file of args.files) {
|
|
155
|
+
let agentName = file.agent
|
|
156
|
+
let agentId: string | undefined
|
|
157
|
+
let setupError: string | undefined
|
|
158
|
+
try {
|
|
159
|
+
const agent = await call<{ id: string }>(api, `/v1/config/agents/${encodeURIComponent(file.agent)}`)
|
|
160
|
+
agentId = agent.id
|
|
161
|
+
if (!args.live) {
|
|
162
|
+
const preview = await call<{ previewAgentSlug: string }>(api, `/v1/config/agents/${agent.id}/test-session`, { body: {} })
|
|
163
|
+
agentName = preview.previewAgentSlug
|
|
164
|
+
}
|
|
165
|
+
} catch (err) {
|
|
166
|
+
// An Agent that is missing or whose draft cannot be tried fails its cases, not the run.
|
|
167
|
+
setupError = (err as Error).message
|
|
168
|
+
}
|
|
169
|
+
for (const c of file.cases) {
|
|
170
|
+
const started = Date.now()
|
|
171
|
+
let result: CaseResult
|
|
172
|
+
let conversationId: string | undefined
|
|
173
|
+
try {
|
|
174
|
+
if (setupError) throw new Error(setupError)
|
|
175
|
+
const session = await call<{ id: string }>(api, '/v1/sessions', { body: { agentName, workspaceId: args.creds.workspaceId } })
|
|
176
|
+
conversationId = session.id
|
|
177
|
+
let reply = ''
|
|
178
|
+
for (const turn of c.conversation) {
|
|
179
|
+
if (typeof turn.user === 'string') reply = await sendTurn(api, session.id, agentName, turn.user)
|
|
180
|
+
}
|
|
181
|
+
const verdict = judge(c, reply)
|
|
182
|
+
result = { agent: file.agent, file: file.file, name: c.name, passed: verdict.passed, reply, reason: verdict.reason, ms: Date.now() - started, agentId, conversationId }
|
|
183
|
+
} catch (err) {
|
|
184
|
+
result = { agent: file.agent, file: file.file, name: c.name, passed: false, reply: '', reason: (err as Error).message, ms: Date.now() - started, agentId, conversationId }
|
|
185
|
+
}
|
|
186
|
+
results.push(result)
|
|
187
|
+
args.onResult(result)
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
return results
|
|
191
|
+
}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import type { LocalProject } from './local-project'
|
|
2
|
+
import { readState } from './local-state'
|
|
3
|
+
import { findCaseFiles, runCases, type CaseResult } from './local-test-runner'
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Test runs started from Studio: the same runner as `frontera local test`,
|
|
7
|
+
* run in the supervisor so the page can follow progress. Kept in memory,
|
|
8
|
+
* newest first; the case files in the solution are the record that matters.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
export interface TestRun {
|
|
12
|
+
id: string
|
|
13
|
+
status: 'running' | 'passed' | 'failed' | 'error'
|
|
14
|
+
agent: string | null
|
|
15
|
+
live: boolean
|
|
16
|
+
startedAt: string
|
|
17
|
+
finishedAt: string | null
|
|
18
|
+
total: number
|
|
19
|
+
results: CaseResult[]
|
|
20
|
+
error: string | null
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
const KEEP = 20
|
|
24
|
+
|
|
25
|
+
export function createTestRuns(project: LocalProject, serviceUrl: string) {
|
|
26
|
+
const runs: TestRun[] = []
|
|
27
|
+
|
|
28
|
+
return {
|
|
29
|
+
list: (): TestRun[] => runs,
|
|
30
|
+
|
|
31
|
+
get: (id: string): TestRun | undefined => runs.find((r) => r.id === id),
|
|
32
|
+
|
|
33
|
+
/** Start a run; refused while another is running, so a double click cannot run everything twice. */
|
|
34
|
+
start(args: { agent?: string; live?: boolean }): { run: TestRun } | { error: string; status: number } {
|
|
35
|
+
if (runs.some((r) => r.status === 'running')) return { error: 'a test run is already running', status: 409 }
|
|
36
|
+
const creds = readState(project)?.credentials
|
|
37
|
+
if (!creds?.session) return { error: 'the runtime is not seeded yet', status: 409 }
|
|
38
|
+
let files: ReturnType<typeof findCaseFiles>
|
|
39
|
+
try {
|
|
40
|
+
files = findCaseFiles(project.root, args.agent)
|
|
41
|
+
} catch (err) {
|
|
42
|
+
return { error: `a case file could not be read: ${(err as Error).message}`, status: 422 }
|
|
43
|
+
}
|
|
44
|
+
const total = files.reduce((n, f) => n + f.cases.length, 0)
|
|
45
|
+
if (total === 0) return { error: 'no test cases to run', status: 400 }
|
|
46
|
+
|
|
47
|
+
const run: TestRun = {
|
|
48
|
+
id: crypto.randomUUID(),
|
|
49
|
+
status: 'running',
|
|
50
|
+
agent: args.agent ?? null,
|
|
51
|
+
live: !!args.live,
|
|
52
|
+
startedAt: new Date().toISOString(),
|
|
53
|
+
finishedAt: null,
|
|
54
|
+
total,
|
|
55
|
+
results: [],
|
|
56
|
+
error: null,
|
|
57
|
+
}
|
|
58
|
+
runs.unshift(run)
|
|
59
|
+
runs.splice(KEEP)
|
|
60
|
+
|
|
61
|
+
void runCases({ serviceUrl, creds, files, live: run.live, onResult: (r) => run.results.push(r) })
|
|
62
|
+
.then((results) => {
|
|
63
|
+
run.status = results.every((r) => r.passed) ? 'passed' : 'failed'
|
|
64
|
+
})
|
|
65
|
+
.catch((err: Error) => {
|
|
66
|
+
run.status = 'error'
|
|
67
|
+
run.error = err.message
|
|
68
|
+
})
|
|
69
|
+
.finally(() => {
|
|
70
|
+
run.finishedAt = new Date().toISOString()
|
|
71
|
+
})
|
|
72
|
+
return { run }
|
|
73
|
+
},
|
|
74
|
+
}
|
|
75
|
+
}
|
|
@@ -0,0 +1,322 @@
|
|
|
1
|
+
import { appendFileSync, existsSync, readFileSync, writeFileSync } from 'node:fs'
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Local traces: the part of a tracing backend the platform uses, kept on this
|
|
5
|
+
* machine. The service exports its spans (OTLP JSON, as its tracing exporter
|
|
6
|
+
* sends them) and reads traces back through the same public API it uses in a
|
|
7
|
+
* deployment, so Studio's trace views work locally with no tracing service.
|
|
8
|
+
*
|
|
9
|
+
* Spans are kept in memory and appended to a JSONL file in the runtime's
|
|
10
|
+
* state folder, which is compacted to the newest traces as it grows.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
/** Local keys the service is configured with; nothing leaves this machine. */
|
|
14
|
+
export const LOCAL_TRACE_KEYS = { publicKey: 'pk-lf-frontera-local', secretKey: 'sk-lf-frontera-local' } as const
|
|
15
|
+
|
|
16
|
+
const MAX_TRACES = 300
|
|
17
|
+
|
|
18
|
+
export interface StoredSpan {
|
|
19
|
+
traceId: string
|
|
20
|
+
spanId: string
|
|
21
|
+
parentSpanId: string | null
|
|
22
|
+
name: string
|
|
23
|
+
startMs: number
|
|
24
|
+
endMs: number | null
|
|
25
|
+
attributes: Record<string, unknown>
|
|
26
|
+
statusCode: number
|
|
27
|
+
statusMessage: string | null
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
interface OtlpValue {
|
|
31
|
+
stringValue?: string
|
|
32
|
+
boolValue?: boolean
|
|
33
|
+
intValue?: number | string
|
|
34
|
+
doubleValue?: number
|
|
35
|
+
arrayValue?: { values?: OtlpValue[] }
|
|
36
|
+
kvlistValue?: { values?: Array<{ key: string; value: OtlpValue }> }
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function otlpValue(v: OtlpValue | undefined): unknown {
|
|
40
|
+
if (!v) return null
|
|
41
|
+
if (v.stringValue !== undefined) return v.stringValue
|
|
42
|
+
if (v.boolValue !== undefined) return v.boolValue
|
|
43
|
+
if (v.intValue !== undefined) return Number(v.intValue)
|
|
44
|
+
if (v.doubleValue !== undefined) return v.doubleValue
|
|
45
|
+
if (v.arrayValue) return (v.arrayValue.values ?? []).map(otlpValue)
|
|
46
|
+
if (v.kvlistValue) return Object.fromEntries((v.kvlistValue.values ?? []).map((kv) => [kv.key, otlpValue(kv.value)]))
|
|
47
|
+
return null
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/** OTLP ids arrive as hex (the JSON exporter) or base64; traces use lowercase hex. */
|
|
51
|
+
function hexId(id: unknown): string {
|
|
52
|
+
if (typeof id !== 'string' || id === '') return ''
|
|
53
|
+
if (/^[0-9a-f]+$/i.test(id)) return id.toLowerCase()
|
|
54
|
+
return Buffer.from(id, 'base64').toString('hex')
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
const nanosToMs = (n: unknown) => (n === undefined || n === null || n === '' || n === '0' || n === 0 ? null : Number(BigInt(String(n)) / 1_000_000n))
|
|
58
|
+
|
|
59
|
+
/** The spans of an OTLP JSON export request. */
|
|
60
|
+
export function parseOtlp(body: unknown): StoredSpan[] {
|
|
61
|
+
const out: StoredSpan[] = []
|
|
62
|
+
const resources = (body as { resourceSpans?: unknown[] })?.resourceSpans ?? []
|
|
63
|
+
for (const rs of resources as Array<{ scopeSpans?: Array<{ spans?: unknown[] }> }>) {
|
|
64
|
+
for (const ss of rs.scopeSpans ?? []) {
|
|
65
|
+
for (const raw of (ss.spans ?? []) as Array<Record<string, unknown>>) {
|
|
66
|
+
const traceId = hexId(raw.traceId)
|
|
67
|
+
const spanId = hexId(raw.spanId)
|
|
68
|
+
if (!traceId || !spanId) continue
|
|
69
|
+
const attributes: Record<string, unknown> = {}
|
|
70
|
+
for (const a of (raw.attributes ?? []) as Array<{ key: string; value: OtlpValue }>) attributes[a.key] = otlpValue(a.value)
|
|
71
|
+
const status = (raw.status ?? {}) as { code?: number; message?: string }
|
|
72
|
+
out.push({
|
|
73
|
+
traceId,
|
|
74
|
+
spanId,
|
|
75
|
+
parentSpanId: hexId(raw.parentSpanId) || null,
|
|
76
|
+
name: String(raw.name ?? ''),
|
|
77
|
+
startMs: nanosToMs(raw.startTimeUnixNano) ?? Date.now(),
|
|
78
|
+
endMs: nanosToMs(raw.endTimeUnixNano),
|
|
79
|
+
attributes,
|
|
80
|
+
statusCode: Number(status.code ?? 0),
|
|
81
|
+
statusMessage: status.message ?? null,
|
|
82
|
+
})
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
return out
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/** A JSON-encoded attribute (the tracing SDK serializes objects) as its value. */
|
|
90
|
+
function decoded(value: unknown): unknown {
|
|
91
|
+
if (typeof value !== 'string') return value ?? null
|
|
92
|
+
const t = value.trim()
|
|
93
|
+
const quoted = t.length >= 2 && t.startsWith('"') && t.endsWith('"')
|
|
94
|
+
if (!(t.startsWith('{') || t.startsWith('[') || quoted)) return value
|
|
95
|
+
try {
|
|
96
|
+
return JSON.parse(t)
|
|
97
|
+
} catch {
|
|
98
|
+
return value
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
const first = (attrs: Record<string, unknown>, keys: string[]) => {
|
|
103
|
+
for (const k of keys) if (attrs[k] !== undefined && attrs[k] !== null && attrs[k] !== '') return attrs[k]
|
|
104
|
+
return undefined
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
const num = (v: unknown) => (typeof v === 'number' ? v : typeof v === 'string' && v !== '' && !Number.isNaN(Number(v)) ? Number(v) : undefined)
|
|
108
|
+
|
|
109
|
+
const prefixed = (attrs: Record<string, unknown>, prefix: string) => {
|
|
110
|
+
const out: Record<string, unknown> = {}
|
|
111
|
+
for (const [k, v] of Object.entries(attrs)) if (k.startsWith(`${prefix}.`)) out[k.slice(prefix.length + 1)] = decoded(v)
|
|
112
|
+
const whole = decoded(attrs[prefix])
|
|
113
|
+
return whole && typeof whole === 'object' && !Array.isArray(whole) ? { ...(whole as Record<string, unknown>), ...out } : out
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
const iso = (ms: number | null) => (ms === null ? null : new Date(ms).toISOString())
|
|
117
|
+
|
|
118
|
+
/** A span as an observation: a model call (generation), a tool, or a span. */
|
|
119
|
+
export function toObservation(span: StoredSpan) {
|
|
120
|
+
const a = span.attributes
|
|
121
|
+
const declared = typeof a['langfuse.observation.type'] === 'string' ? (a['langfuse.observation.type'] as string) : null
|
|
122
|
+
const model = first(a, ['langfuse.observation.model.name', 'gen_ai.response.model', 'gen_ai.request.model', 'ai.response.model', 'ai.model.id'])
|
|
123
|
+
const type = (declared ?? (model ? 'generation' : 'span')).toUpperCase()
|
|
124
|
+
const usageDetails = decoded(a['langfuse.observation.usage_details']) as Record<string, number> | null
|
|
125
|
+
const input = num(usageDetails?.input) ?? num(first(a, ['gen_ai.usage.input_tokens', 'gen_ai.usage.prompt_tokens', 'ai.usage.inputTokens', 'ai.usage.promptTokens']))
|
|
126
|
+
const output = num(usageDetails?.output) ?? num(first(a, ['gen_ai.usage.output_tokens', 'gen_ai.usage.completion_tokens', 'ai.usage.outputTokens', 'ai.usage.completionTokens']))
|
|
127
|
+
const total = num(usageDetails?.total) ?? (input !== undefined || output !== undefined ? (input ?? 0) + (output ?? 0) : undefined)
|
|
128
|
+
const costDetails = decoded(a['langfuse.observation.cost_details']) as Record<string, number> | null
|
|
129
|
+
const cost = num(costDetails?.total)
|
|
130
|
+
const level = (first(a, ['langfuse.observation.level']) as string | undefined) ?? (span.statusCode === 2 ? 'ERROR' : 'DEFAULT')
|
|
131
|
+
return {
|
|
132
|
+
id: span.spanId,
|
|
133
|
+
traceId: span.traceId,
|
|
134
|
+
type,
|
|
135
|
+
name: span.name,
|
|
136
|
+
startTime: iso(span.startMs),
|
|
137
|
+
endTime: iso(span.endMs),
|
|
138
|
+
parentObservationId: span.parentSpanId,
|
|
139
|
+
level,
|
|
140
|
+
statusMessage: (first(a, ['langfuse.observation.status_message']) as string | undefined) ?? span.statusMessage,
|
|
141
|
+
model: typeof model === 'string' ? model : null,
|
|
142
|
+
modelParameters: decoded(a['langfuse.observation.model.parameters']) ?? null,
|
|
143
|
+
input: decoded(first(a, ['langfuse.observation.input', 'ai.prompt.messages', 'ai.prompt', 'gen_ai.prompt'])) ?? null,
|
|
144
|
+
output: decoded(first(a, ['langfuse.observation.output', 'ai.response.text', 'ai.response.toolCalls', 'gen_ai.completion'])) ?? null,
|
|
145
|
+
metadata: prefixed(a, 'langfuse.observation.metadata'),
|
|
146
|
+
usage: total === undefined ? null : { input: input ?? 0, output: output ?? 0, total, unit: 'TOKENS' },
|
|
147
|
+
usageDetails: usageDetails ?? (total === undefined ? {} : { input: input ?? 0, output: output ?? 0, total }),
|
|
148
|
+
costDetails: costDetails ?? {},
|
|
149
|
+
calculatedTotalCost: cost ?? null,
|
|
150
|
+
latency: span.endMs === null ? null : (span.endMs - span.startMs) / 1000,
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/** The spans of one trace as a trace with its observations. */
|
|
155
|
+
export function toTrace(spans: StoredSpan[]) {
|
|
156
|
+
const ids = new Set(spans.map((s) => s.spanId))
|
|
157
|
+
const ordered = [...spans].sort((a, b) => a.startMs - b.startMs)
|
|
158
|
+
const roots = ordered.filter((s) => !s.parentSpanId || !ids.has(s.parentSpanId))
|
|
159
|
+
const root = roots[0] ?? ordered[0]
|
|
160
|
+
if (!root) throw new Error('a trace has at least one span')
|
|
161
|
+
// Trace-level attributes may sit on any span (they propagate to children).
|
|
162
|
+
const attr = (key: string) => {
|
|
163
|
+
for (const s of ordered) if (s.attributes[key] !== undefined) return s.attributes[key]
|
|
164
|
+
return undefined
|
|
165
|
+
}
|
|
166
|
+
const metadata: Record<string, unknown> = {}
|
|
167
|
+
for (const s of ordered) Object.assign(metadata, prefixed(s.attributes, 'langfuse.trace.metadata'))
|
|
168
|
+
const tags = new Set<string>()
|
|
169
|
+
for (const s of ordered) {
|
|
170
|
+
const t = decoded(s.attributes['langfuse.trace.tags'])
|
|
171
|
+
if (Array.isArray(t)) for (const tag of t) if (typeof tag === 'string') tags.add(tag)
|
|
172
|
+
}
|
|
173
|
+
const observations = ordered.map(toObservation)
|
|
174
|
+
const start = Math.min(...ordered.map((s) => s.startMs))
|
|
175
|
+
const end = Math.max(...ordered.map((s) => s.endMs ?? s.startMs))
|
|
176
|
+
const totalCost = observations.reduce((n, o) => n + (o.calculatedTotalCost ?? 0), 0)
|
|
177
|
+
const rootObservation = observations.find((o) => o.id === root.spanId)
|
|
178
|
+
const level = observations.some((o) => o.level === 'ERROR') ? 'ERROR' : observations.some((o) => o.level === 'WARNING') ? 'WARNING' : 'DEFAULT'
|
|
179
|
+
return {
|
|
180
|
+
id: root.traceId,
|
|
181
|
+
name: (attr('langfuse.trace.name') as string | undefined) ?? root.name,
|
|
182
|
+
timestamp: new Date(start).toISOString(),
|
|
183
|
+
userId: (attr('user.id') ?? attr('langfuse.user.id') ?? null) as string | null,
|
|
184
|
+
sessionId: (attr('session.id') ?? attr('langfuse.session.id') ?? null) as string | null,
|
|
185
|
+
tags: [...tags],
|
|
186
|
+
metadata,
|
|
187
|
+
environment: (attr('langfuse.environment') as string | undefined) ?? 'local',
|
|
188
|
+
input: decoded(attr('langfuse.trace.input')) ?? rootObservation?.input ?? null,
|
|
189
|
+
output: decoded(attr('langfuse.trace.output')) ?? rootObservation?.output ?? null,
|
|
190
|
+
latency: (end - start) / 1000,
|
|
191
|
+
totalCost,
|
|
192
|
+
level,
|
|
193
|
+
htmlPath: '',
|
|
194
|
+
observations,
|
|
195
|
+
scores: [],
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
type LocalTrace = ReturnType<typeof toTrace>
|
|
200
|
+
type Condition = { type?: string; column?: string; key?: string; operator?: string; value?: unknown }
|
|
201
|
+
|
|
202
|
+
/** The subset of the trace list filter the service sends: time bounds, a metadata key, the level. */
|
|
203
|
+
export function matchesFilter(trace: LocalTrace, filter: Condition[]): boolean {
|
|
204
|
+
for (const c of filter) {
|
|
205
|
+
if (c.column === 'timestamp' && typeof c.value === 'string') {
|
|
206
|
+
const at = Date.parse(trace.timestamp)
|
|
207
|
+
const bound = Date.parse(c.value)
|
|
208
|
+
if (c.operator === '>=' && at < bound) return false
|
|
209
|
+
if (c.operator === '<=' && at > bound) return false
|
|
210
|
+
if (c.operator === '>' && at <= bound) return false
|
|
211
|
+
if (c.operator === '<' && at >= bound) return false
|
|
212
|
+
} else if (c.column === 'metadata' && c.key && c.operator === '=') {
|
|
213
|
+
if (String(trace.metadata[c.key] ?? '') !== String(c.value)) return false
|
|
214
|
+
} else if (c.column === 'level' && c.operator === '=') {
|
|
215
|
+
if (trace.level !== c.value) return false
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
return true
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
export function createTraceStore(file: string | null) {
|
|
222
|
+
const byTrace = new Map<string, Map<string, StoredSpan>>()
|
|
223
|
+
let written = 0
|
|
224
|
+
|
|
225
|
+
const add = (span: StoredSpan) => {
|
|
226
|
+
const spans = byTrace.get(span.traceId) ?? new Map<string, StoredSpan>()
|
|
227
|
+
// Re-inserting moves the trace to the newest end of the map.
|
|
228
|
+
byTrace.delete(span.traceId)
|
|
229
|
+
spans.set(span.spanId, span)
|
|
230
|
+
byTrace.set(span.traceId, spans)
|
|
231
|
+
}
|
|
232
|
+
const prune = () => {
|
|
233
|
+
while (byTrace.size > MAX_TRACES) byTrace.delete(byTrace.keys().next().value!)
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
if (file && existsSync(file)) {
|
|
237
|
+
for (const line of readFileSync(file, 'utf8').split('\n')) {
|
|
238
|
+
if (!line.trim()) continue
|
|
239
|
+
try {
|
|
240
|
+
add(JSON.parse(line) as StoredSpan)
|
|
241
|
+
} catch {
|
|
242
|
+
// A torn last line from a crash; the rest stands.
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
prune()
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
const compact = () => {
|
|
249
|
+
if (!file) return
|
|
250
|
+
const lines: string[] = []
|
|
251
|
+
for (const spans of byTrace.values()) for (const s of spans.values()) lines.push(JSON.stringify(s))
|
|
252
|
+
writeFileSync(file, lines.length ? `${lines.join('\n')}\n` : '')
|
|
253
|
+
written = lines.length
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
return {
|
|
257
|
+
ingest(spans: StoredSpan[]) {
|
|
258
|
+
for (const s of spans) add(s)
|
|
259
|
+
prune()
|
|
260
|
+
if (!file || spans.length === 0) return
|
|
261
|
+
appendFileSync(file, `${spans.map((s) => JSON.stringify(s)).join('\n')}\n`)
|
|
262
|
+
written += spans.length
|
|
263
|
+
// Rewrite with only what is kept once the file is well past it.
|
|
264
|
+
let kept = 0
|
|
265
|
+
for (const t of byTrace.values()) kept += t.size
|
|
266
|
+
if (written > kept * 2 + 500) compact()
|
|
267
|
+
},
|
|
268
|
+
list(opts: { page?: number; limit?: number; filter?: Condition[] } = {}) {
|
|
269
|
+
const all = [...byTrace.values()].map((m) => toTrace([...m.values()])).filter((t) => matchesFilter(t, opts.filter ?? []))
|
|
270
|
+
all.sort((a, b) => Date.parse(b.timestamp) - Date.parse(a.timestamp))
|
|
271
|
+
const limit = Math.max(1, Math.min(opts.limit ?? 50, 100))
|
|
272
|
+
const page = Math.max(1, opts.page ?? 1)
|
|
273
|
+
const data = all.slice((page - 1) * limit, page * limit)
|
|
274
|
+
return { data, meta: { page, limit, totalItems: all.length, totalPages: Math.ceil(all.length / limit) } }
|
|
275
|
+
},
|
|
276
|
+
get(traceId: string): LocalTrace | null {
|
|
277
|
+
const spans = byTrace.get(traceId.toLowerCase())
|
|
278
|
+
return spans ? toTrace([...spans.values()]) : null
|
|
279
|
+
},
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
export function startLocalTraces(opts: { port: number; file: string | null; log?: (line: string) => void }) {
|
|
284
|
+
const store = createTraceStore(opts.file)
|
|
285
|
+
const json = (data: unknown, status = 200) => new Response(JSON.stringify(data), { status, headers: { 'content-type': 'application/json' } })
|
|
286
|
+
let warnedFormat = false
|
|
287
|
+
const server = Bun.serve({
|
|
288
|
+
hostname: '127.0.0.1',
|
|
289
|
+
port: opts.port,
|
|
290
|
+
async fetch(req) {
|
|
291
|
+
const url = new URL(req.url)
|
|
292
|
+
const path = url.pathname
|
|
293
|
+
if (req.method === 'POST' && path === '/api/public/otel/v1/traces') {
|
|
294
|
+
if (!(req.headers.get('content-type') ?? '').includes('json')) {
|
|
295
|
+
if (!warnedFormat) opts.log?.(`spans arrived as ${req.headers.get('content-type')}; only OTLP JSON is kept`)
|
|
296
|
+
warnedFormat = true
|
|
297
|
+
return json({ message: 'OTLP JSON only' }, 415)
|
|
298
|
+
}
|
|
299
|
+
const spans = parseOtlp(await req.json().catch(() => null))
|
|
300
|
+
store.ingest(spans)
|
|
301
|
+
return json({ partialSuccess: {} })
|
|
302
|
+
}
|
|
303
|
+
if (req.method === 'GET' && path === '/api/public/traces') {
|
|
304
|
+
let filter: Condition[] = []
|
|
305
|
+
try {
|
|
306
|
+
filter = JSON.parse(url.searchParams.get('filter') ?? '[]') as Condition[]
|
|
307
|
+
} catch {
|
|
308
|
+
filter = []
|
|
309
|
+
}
|
|
310
|
+
return json(store.list({ page: Number(url.searchParams.get('page') ?? 1), limit: Number(url.searchParams.get('limit') ?? 50), filter }))
|
|
311
|
+
}
|
|
312
|
+
const one = path.match(/^\/api\/public\/traces\/([^/]+)$/)
|
|
313
|
+
if (req.method === 'GET' && one?.[1]) {
|
|
314
|
+
const trace = store.get(decodeURIComponent(one[1]))
|
|
315
|
+
return trace ? json(trace) : json({ message: 'Trace not found' }, 404)
|
|
316
|
+
}
|
|
317
|
+
if (req.method === 'GET' && path === '/api/public/health') return json({ status: 'OK', version: 'frontera-local' })
|
|
318
|
+
return json({ message: `${req.method} ${path} is not part of local traces` }, 404)
|
|
319
|
+
},
|
|
320
|
+
})
|
|
321
|
+
return { server, store }
|
|
322
|
+
}
|
package/src/main.ts
CHANGED
|
@@ -117,7 +117,7 @@ async function main(): Promise<number> {
|
|
|
117
117
|
// `frontera agent --help`, or a bare `frontera agent`: list the noun's
|
|
118
118
|
// commands. Without this the CLI's own hints dead-ended, since a reserved
|
|
119
119
|
// verb tells the caller to run exactly this.
|
|
120
|
-
if (!command && nouns().includes(noun)) {
|
|
120
|
+
if (!command && nouns(true).includes(noun)) {
|
|
121
121
|
const askedForHelp =
|
|
122
122
|
maybeVerb === undefined ||
|
|
123
123
|
maybeVerb === '--help' ||
|
package/src/packaging.ts
CHANGED
|
@@ -45,6 +45,11 @@ export function isExcluded(rel: string): boolean {
|
|
|
45
45
|
// machine, not source: packaging it would push a few hundred KB of PNG into
|
|
46
46
|
// every `app pull` for a picture the platform already stores per version.
|
|
47
47
|
if (rel === '.frontera/last-render.png') return true
|
|
48
|
+
// The rest of `.frontera/` is this machine's tool state: the CLI's binding,
|
|
49
|
+
// and in a local project the runtime's keys, owner session and secrets
|
|
50
|
+
// (`.frontera/local/`). None of it is source and some of it is credentials,
|
|
51
|
+
// so only the render evidence the platform reads from an App's archive goes.
|
|
52
|
+
if (segments[0] === '.frontera' && segments.length > 1 && rel !== '.frontera/last-render.json') return true
|
|
48
53
|
return false
|
|
49
54
|
}
|
|
50
55
|
|