@gotcos/glasses-server 6.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.cos-profile.example.json +7 -0
- package/.env.example +44 -0
- package/CHANGELOG.md +25 -0
- package/LICENSE +21 -0
- package/README.md +78 -0
- package/bin/cli.cjs +203 -0
- package/package.json +53 -0
- package/server/env.ts +26 -0
- package/server/index.ts +211 -0
- package/server/lib/archive-budget.ts +65 -0
- package/server/lib/archive.ts +414 -0
- package/server/lib/atomic-fs.ts +50 -0
- package/server/lib/audio-enhance.ts +87 -0
- package/server/lib/claude-bridge.ts +682 -0
- package/server/lib/claude-circuit.ts +52 -0
- package/server/lib/claude-run-ledger.ts +279 -0
- package/server/lib/codex-bridge.ts +476 -0
- package/server/lib/codex-engine-sessions.ts +140 -0
- package/server/lib/codex-run-ledger.ts +298 -0
- package/server/lib/context-builder.ts +210 -0
- package/server/lib/conversation.ts +587 -0
- package/server/lib/data-dir.ts +20 -0
- package/server/lib/display-bus.ts +21 -0
- package/server/lib/display-format.ts +23 -0
- package/server/lib/fuzzy-correct.ts +286 -0
- package/server/lib/hallucination-filter.ts +469 -0
- package/server/lib/local-day.ts +13 -0
- package/server/lib/model-router.ts +38 -0
- package/server/lib/openai-key.ts +155 -0
- package/server/lib/openai-whisper-budget.ts +170 -0
- package/server/lib/profile.ts +94 -0
- package/server/lib/python-bridge.ts +84 -0
- package/server/lib/response-cache.ts +138 -0
- package/server/lib/session-cache-writer.ts +266 -0
- package/server/lib/session-log.ts +162 -0
- package/server/lib/speaker-embeddings.ts +578 -0
- package/server/lib/telegram-notify.ts +85 -0
- package/server/lib/token-audit.ts +50 -0
- package/server/lib/transcribe-audio.ts +187 -0
- package/server/lib/utils.ts +5 -0
- package/server/lib/vad-silero.ts +179 -0
- package/server/lib/whisper-local.ts +697 -0
- package/server/routes/diag.ts +115 -0
- package/server/routes/display.ts +65 -0
- package/server/routes/health.ts +128 -0
- package/server/routes/openai-compat.ts +446 -0
- package/server/routes/openai-key.ts +121 -0
- package/server/routes/query.ts +121 -0
- package/server/routes/transcribe-stream.ts +1090 -0
- package/server/routes/transcribe.ts +55 -0
- package/shared/model-preference.ts +81 -0
|
@@ -0,0 +1,682 @@
|
|
|
1
|
+
// Claude bridge — streaming interface to claude -p with context injection
|
|
2
|
+
// Uses streaming-json output, web search tools, conversation history
|
|
3
|
+
//
|
|
4
|
+
// Timeout strategy:
|
|
5
|
+
// INACTIVITY timeout (resets on any stdout data) — catches truly stalled processes
|
|
6
|
+
// WALL CLOCK max — absolute cap regardless of activity
|
|
7
|
+
// HEARTBEAT — emits progress events during silence so the display stays alive
|
|
8
|
+
|
|
9
|
+
import { spawn } from 'node:child_process'
|
|
10
|
+
import { writeFileSync, readFileSync, unlinkSync, existsSync } from 'node:fs'
|
|
11
|
+
import { join } from 'node:path'
|
|
12
|
+
import { appendFileSync } from 'node:fs'
|
|
13
|
+
import crypto from 'node:crypto'
|
|
14
|
+
import { COS_SCRIPTS_DIR } from './python-bridge.js'
|
|
15
|
+
import { logTokenAudit } from './token-audit.js'
|
|
16
|
+
import { buildSystemPrompt, buildLightweightSystemPrompt, buildPrewarmSystemPrompt, getCachedContextInstant } from './context-builder.js'
|
|
17
|
+
import { getHistory, addExchange, formatHistoryForPrompt, getOrCreateSession, isNewSession, markSessionNotified, getSessionModel, getSessionRaw, replaceLastExchangeWithSummary, type ModelPreference, type PromptReference } from './conversation.js'
|
|
18
|
+
import { notifySessionStart, notifyExchange } from './telegram-notify.js'
|
|
19
|
+
import { isClaudeModel, type ClaudeModelPreference } from '../../shared/model-preference.js'
|
|
20
|
+
import {
|
|
21
|
+
finishClaudeRun,
|
|
22
|
+
getClaudeEffortLevel,
|
|
23
|
+
startClaudeRun,
|
|
24
|
+
updateClaudeRun,
|
|
25
|
+
} from './claude-run-ledger.js'
|
|
26
|
+
|
|
27
|
+
// Inactivity = no stdout data for this long → kill (catches stalls)
|
|
28
|
+
const INACTIVITY_BY_MODEL: Record<ClaudeModelPreference, number> = {
|
|
29
|
+
opus: 180_000, // 3 minutes — Opus can gap during tool use (WebSearch, reasoning)
|
|
30
|
+
sonnet: 30_000, // 30 seconds
|
|
31
|
+
haiku: 15_000, // 15 seconds
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
// Wall clock max = absolute cap even if actively streaming
|
|
35
|
+
const WALL_MAX_BY_MODEL: Record<ClaudeModelPreference, number> = {
|
|
36
|
+
opus: 600_000, // 10 minutes
|
|
37
|
+
sonnet: 120_000, // 2 minutes
|
|
38
|
+
haiku: 60_000, // 1 minute
|
|
39
|
+
}
|
|
40
|
+
const WALL_MAX_EXTENDED_MS = 900_000 // 15 minutes for slash commands / heavy queries
|
|
41
|
+
|
|
42
|
+
// Heartbeat = emit progress status during silence so client knows we're alive
|
|
43
|
+
const HEARTBEAT_INTERVAL_MS = 6_000 // Every 6 seconds
|
|
44
|
+
|
|
45
|
+
// ─── CLI session persistence ───
|
|
46
|
+
// Maps COS session IDs + Claude model → Claude CLI session IDs for process reuse.
|
|
47
|
+
// First query in a session spawns fresh. Subsequent queries use --resume
|
|
48
|
+
// so the CLI has warm cached context (avoids re-processing system prompt).
|
|
49
|
+
const cliSessionMap = new Map<string, string>()
|
|
50
|
+
|
|
51
|
+
// ─── CLI pre-warm ───
|
|
52
|
+
// At server boot, we spawn a throwaway Haiku query to establish a CLI session.
|
|
53
|
+
// This pre-warmed session ID is used for the FIRST query in any new COS session,
|
|
54
|
+
// eliminating the 2-15s cold start. After that, each session maps to its own CLI session.
|
|
55
|
+
let preWarmedCliSessionId: string | null = null
|
|
56
|
+
let preWarmInProgress = false
|
|
57
|
+
|
|
58
|
+
// ─── CLI session disk persistence ───
|
|
59
|
+
// Persists CLI session IDs to /tmp (NOT server/data/ — iCloud sync risk).
|
|
60
|
+
// Survives server restarts. TTL: 2 hours (matches conversation SESSION_TTL_MS).
|
|
61
|
+
const CLI_SESSIONS_FILE = '/tmp/cos-cli-sessions.json'
|
|
62
|
+
const CLI_SESSION_TTL_MS = 2 * 60 * 60_000 // 2 hours
|
|
63
|
+
|
|
64
|
+
interface CliSessionsData {
|
|
65
|
+
preWarmedCliSessionId: string | null
|
|
66
|
+
cliSessionMap: Record<string, { cliSessionId: string; savedAt: number }>
|
|
67
|
+
savedAt: string
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
function cliSessionKey(cosSessionId: string, model: ClaudeModelPreference): string {
|
|
71
|
+
return `${cosSessionId}:${model}`
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function getAnyCliSessionId(cosSessionId: string): string | undefined {
|
|
75
|
+
for (const [key, cliId] of cliSessionMap) {
|
|
76
|
+
if (key === cosSessionId || key.startsWith(`${cosSessionId}:`)) return cliId
|
|
77
|
+
}
|
|
78
|
+
return undefined
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function loadCliSessions(): void {
|
|
82
|
+
// Don't restore stale sessions — they have no model affinity tag,
|
|
83
|
+
// so a Haiku session could be reused for an Opus query via --resume,
|
|
84
|
+
// causing model inheritance issues. Fresh pre-warm on boot is sufficient.
|
|
85
|
+
try {
|
|
86
|
+
if (existsSync(CLI_SESSIONS_FILE)) {
|
|
87
|
+
unlinkSync(CLI_SESSIONS_FILE)
|
|
88
|
+
console.log('[claude-bridge] Cleared stale CLI session cache (fresh start)')
|
|
89
|
+
}
|
|
90
|
+
} catch {
|
|
91
|
+
// Ignore — file may not exist
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
let cliSaveTimer: ReturnType<typeof setTimeout> | null = null
|
|
96
|
+
|
|
97
|
+
function scheduleCliSessionSave(): void {
|
|
98
|
+
if (cliSaveTimer) return
|
|
99
|
+
cliSaveTimer = setTimeout(() => {
|
|
100
|
+
cliSaveTimer = null
|
|
101
|
+
saveCliSessions()
|
|
102
|
+
}, 500)
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
function saveCliSessions(): void {
|
|
106
|
+
try {
|
|
107
|
+
const mapEntries: Record<string, { cliSessionId: string; savedAt: number }> = {}
|
|
108
|
+
const now = Date.now()
|
|
109
|
+
for (const [cosId, cliId] of cliSessionMap) {
|
|
110
|
+
mapEntries[cosId] = { cliSessionId: cliId, savedAt: now }
|
|
111
|
+
}
|
|
112
|
+
const data: CliSessionsData = {
|
|
113
|
+
preWarmedCliSessionId,
|
|
114
|
+
cliSessionMap: mapEntries,
|
|
115
|
+
savedAt: new Date().toISOString(),
|
|
116
|
+
}
|
|
117
|
+
writeFileSync(CLI_SESSIONS_FILE, JSON.stringify(data, null, 2))
|
|
118
|
+
} catch (err) {
|
|
119
|
+
console.error('[claude-bridge] Failed to save CLI sessions:', err)
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
// Load persisted sessions on module init
|
|
124
|
+
loadCliSessions()
|
|
125
|
+
|
|
126
|
+
/** Expose CLI session ID for a given COS session (used by API routes) */
|
|
127
|
+
export function getCliSessionId(cosSessionId: string): string | undefined {
|
|
128
|
+
const sessionModel = getSessionModel(cosSessionId)
|
|
129
|
+
if (sessionModel && isClaudeModel(sessionModel)) {
|
|
130
|
+
return cliSessionMap.get(cliSessionKey(cosSessionId, sessionModel)) ?? getAnyCliSessionId(cosSessionId)
|
|
131
|
+
}
|
|
132
|
+
return getAnyCliSessionId(cosSessionId)
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/** Get the best available CLI session ID — mapped session > pre-warmed > undefined */
|
|
136
|
+
export function getAvailableCliSessionId(cosSessionId?: string): string | undefined {
|
|
137
|
+
if (cosSessionId) {
|
|
138
|
+
const mapped = getCliSessionId(cosSessionId)
|
|
139
|
+
if (mapped) return mapped
|
|
140
|
+
}
|
|
141
|
+
return preWarmedCliSessionId ?? undefined
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
// ─── Latency logging ───
|
|
145
|
+
import { dataPath } from './data-dir.js'
|
|
146
|
+
const LATENCY_LOG_FILE = dataPath('latency-log.jsonl')
|
|
147
|
+
|
|
148
|
+
export interface LatencyEntry {
|
|
149
|
+
timestamp: string
|
|
150
|
+
query: string
|
|
151
|
+
ttfb_ms: number
|
|
152
|
+
total_ms: number
|
|
153
|
+
model: string
|
|
154
|
+
resumed: boolean
|
|
155
|
+
contextInjected: boolean
|
|
156
|
+
cacheHit: boolean
|
|
157
|
+
stream_requested?: boolean
|
|
158
|
+
deduped?: boolean
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
export function logLatency(entry: LatencyEntry): void {
|
|
162
|
+
try {
|
|
163
|
+
appendFileSync(LATENCY_LOG_FILE, JSON.stringify(entry) + '\n')
|
|
164
|
+
} catch { /* ignore — non-critical */ }
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* Pre-warm the Claude CLI by running a minimal Haiku query at server boot.
|
|
169
|
+
* Captures the CLI session ID so the first real query can use --resume.
|
|
170
|
+
* Called from index.ts alongside prewarmContext().
|
|
171
|
+
*/
|
|
172
|
+
export async function preWarmCLI(): Promise<void> {
|
|
173
|
+
if (preWarmInProgress) return
|
|
174
|
+
preWarmInProgress = true
|
|
175
|
+
|
|
176
|
+
const start = Date.now()
|
|
177
|
+
console.log('[claude-bridge] Pre-warming CLI session...')
|
|
178
|
+
|
|
179
|
+
return new Promise<void>((resolve) => {
|
|
180
|
+
const env = { ...process.env }
|
|
181
|
+
delete env.CLAUDECODE
|
|
182
|
+
|
|
183
|
+
const proc = spawn('claude', [
|
|
184
|
+
'-p',
|
|
185
|
+
'--model', 'opus', // Must match default query model — --resume inherits session model
|
|
186
|
+
'--effort', getClaudeEffortLevel(),
|
|
187
|
+
'--output-format', 'stream-json',
|
|
188
|
+
'--verbose',
|
|
189
|
+
'--dangerously-skip-permissions',
|
|
190
|
+
'--system-prompt', buildPrewarmSystemPrompt(),
|
|
191
|
+
], {
|
|
192
|
+
stdio: ['pipe', 'pipe', 'pipe'],
|
|
193
|
+
env,
|
|
194
|
+
cwd: COS_SCRIPTS_DIR ?? process.cwd(),
|
|
195
|
+
})
|
|
196
|
+
|
|
197
|
+
let buffer = ''
|
|
198
|
+
|
|
199
|
+
proc.stdout.on('data', (chunk: Buffer) => {
|
|
200
|
+
buffer += chunk.toString()
|
|
201
|
+
const lines = buffer.split('\n')
|
|
202
|
+
buffer = lines.pop() ?? ''
|
|
203
|
+
|
|
204
|
+
for (const line of lines) {
|
|
205
|
+
const trimmed = line.trim()
|
|
206
|
+
if (!trimmed) continue
|
|
207
|
+
try {
|
|
208
|
+
const event = JSON.parse(trimmed)
|
|
209
|
+
if (event.type === 'result' && event.session_id) {
|
|
210
|
+
preWarmedCliSessionId = event.session_id
|
|
211
|
+
scheduleCliSessionSave()
|
|
212
|
+
const elapsed = Date.now() - start
|
|
213
|
+
console.log(`[claude-bridge] CLI pre-warmed in ${elapsed}ms (session: ${event.session_id.slice(0, 12)}...)`)
|
|
214
|
+
}
|
|
215
|
+
} catch { /* ignore */ }
|
|
216
|
+
}
|
|
217
|
+
})
|
|
218
|
+
|
|
219
|
+
proc.on('close', () => {
|
|
220
|
+
preWarmInProgress = false
|
|
221
|
+
const elapsed = Date.now() - start
|
|
222
|
+
logTokenAudit({
|
|
223
|
+
source: 'g2-prewarm',
|
|
224
|
+
model: 'opus',
|
|
225
|
+
inputChars: 500, // system prompt + "ready"
|
|
226
|
+
outputChars: 50,
|
|
227
|
+
durationMs: elapsed,
|
|
228
|
+
caller: 'prewarm',
|
|
229
|
+
})
|
|
230
|
+
if (!preWarmedCliSessionId) {
|
|
231
|
+
console.warn('[claude-bridge] CLI pre-warm completed but no session ID captured')
|
|
232
|
+
}
|
|
233
|
+
resolve()
|
|
234
|
+
})
|
|
235
|
+
|
|
236
|
+
proc.on('error', (err) => {
|
|
237
|
+
preWarmInProgress = false
|
|
238
|
+
console.error('[claude-bridge] CLI pre-warm failed:', err.message)
|
|
239
|
+
resolve()
|
|
240
|
+
})
|
|
241
|
+
|
|
242
|
+
// 30s safety timeout — don't block server start forever
|
|
243
|
+
setTimeout(() => {
|
|
244
|
+
if (preWarmInProgress) {
|
|
245
|
+
proc.kill('SIGTERM')
|
|
246
|
+
preWarmInProgress = false
|
|
247
|
+
console.warn('[claude-bridge] CLI pre-warm timed out (30s)')
|
|
248
|
+
resolve()
|
|
249
|
+
}
|
|
250
|
+
}, 30_000)
|
|
251
|
+
|
|
252
|
+
// Send minimal query and close stdin
|
|
253
|
+
proc.stdin.write('ready')
|
|
254
|
+
proc.stdin.end()
|
|
255
|
+
})
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
function isExtendedQuery(query: string): boolean {
|
|
259
|
+
const trimmed = query.trim()
|
|
260
|
+
if (trimmed.startsWith('/')) return true
|
|
261
|
+
if (/\b(morning|sync|briefing|dashboard|compare|analyze|research|summarize)\b/i.test(trimmed)) return true
|
|
262
|
+
// Long queries (100+ chars) tend to be complex
|
|
263
|
+
if (trimmed.length > 100) return true
|
|
264
|
+
return false
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
export interface ModelRunMetadata {
|
|
268
|
+
codexRunId?: string
|
|
269
|
+
codexThreadId?: string
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
export interface StreamCallbacks {
|
|
273
|
+
onChunk: (text: string) => void
|
|
274
|
+
onDone: (fullText: string, model: ModelPreference, cliSessionId?: string, metadata?: ModelRunMetadata) => void
|
|
275
|
+
onError: (error: string) => void
|
|
276
|
+
onToolStatus?: (toolName: string) => void
|
|
277
|
+
onStart?: (model: ModelPreference, sessionId: string, cliSessionId?: string, metadata?: ModelRunMetadata) => void
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
type Phase = 'context' | 'thinking' | 'searching' | 'generating'
|
|
281
|
+
|
|
282
|
+
const PHASE_LABELS: Record<Phase, string> = {
|
|
283
|
+
context: 'Loading context...',
|
|
284
|
+
thinking: 'Thinking...',
|
|
285
|
+
searching: 'Searching...',
|
|
286
|
+
generating: 'Writing...',
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
/**
|
|
290
|
+
* Call Claude with streaming output, conversation history, and COS context.
|
|
291
|
+
* Returns the session ID for multi-turn tracking.
|
|
292
|
+
*/
|
|
293
|
+
export interface CallOptions {
|
|
294
|
+
lightweight?: boolean // Skip async context fetch — use cached context instantly (G2 speed path)
|
|
295
|
+
abortSignal?: AbortSignal
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
export async function callClaudeStreaming(
|
|
299
|
+
query: string,
|
|
300
|
+
sessionId: string | undefined,
|
|
301
|
+
callbacks: StreamCallbacks,
|
|
302
|
+
model?: ClaudeModelPreference,
|
|
303
|
+
images?: string[],
|
|
304
|
+
reference?: PromptReference,
|
|
305
|
+
globalMsgNum?: number,
|
|
306
|
+
options?: CallOptions,
|
|
307
|
+
): Promise<string> {
|
|
308
|
+
// Get or create session
|
|
309
|
+
const sid = getOrCreateSession(sessionId)
|
|
310
|
+
const history = getHistory(sid)
|
|
311
|
+
const session = getSessionRaw(sid)
|
|
312
|
+
const contextBreaks = session?.contextBreaks ?? []
|
|
313
|
+
const historyPrompt = formatHistoryForPrompt(history, contextBreaks, reference)
|
|
314
|
+
const contextPrompt = historyPrompt
|
|
315
|
+
|
|
316
|
+
// Resolve model: per-message > session preference > opus default
|
|
317
|
+
const sessionModel = getSessionModel(sid)
|
|
318
|
+
const resolvedModel: ClaudeModelPreference = model ?? (sessionModel && isClaudeModel(sessionModel) ? sessionModel : 'opus')
|
|
319
|
+
|
|
320
|
+
// Notify client immediately — model is known before any async work
|
|
321
|
+
// Pass existing CLI session ID if resuming (new sessions get it after first result)
|
|
322
|
+
const resolvedCliKey = cliSessionKey(sid, resolvedModel)
|
|
323
|
+
let existingCliSession = cliSessionMap.get(resolvedCliKey)
|
|
324
|
+
callbacks.onStart?.(resolvedModel, sid, existingCliSession)
|
|
325
|
+
|
|
326
|
+
// Phase: context loading (skipped in lightweight mode)
|
|
327
|
+
let phase: Phase = 'context'
|
|
328
|
+
|
|
329
|
+
let systemPrompt: string
|
|
330
|
+
if (options?.lightweight) {
|
|
331
|
+
// G2 speed path — minimal system prompt, context only when needed
|
|
332
|
+
systemPrompt = buildLightweightSystemPrompt(query, contextPrompt)
|
|
333
|
+
} else {
|
|
334
|
+
callbacks.onToolStatus?.('Loading context...')
|
|
335
|
+
// Full COS path — async context with Python subprocess calls
|
|
336
|
+
systemPrompt = await buildSystemPrompt(contextPrompt)
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
// Phase: thinking (waiting for Claude to start)
|
|
340
|
+
phase = 'thinking'
|
|
341
|
+
callbacks.onToolStatus?.('Thinking...')
|
|
342
|
+
|
|
343
|
+
// ── Vision: save temp image files if provided ──
|
|
344
|
+
const imagePaths: string[] = []
|
|
345
|
+
if (images && images.length > 0) {
|
|
346
|
+
for (const img of images) {
|
|
347
|
+
const id = crypto.randomUUID().slice(0, 8)
|
|
348
|
+
const p = join('/tmp', `cos-vision-${id}.jpg`)
|
|
349
|
+
writeFileSync(p, Buffer.from(img, 'base64'))
|
|
350
|
+
imagePaths.push(p)
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
// Check if this is the first query in a new session (before adding exchange)
|
|
355
|
+
const isFirstQuery = isNewSession(sid)
|
|
356
|
+
|
|
357
|
+
// Record user message (with [Photo]/[N Photos] prefix for vision queries)
|
|
358
|
+
const photoPrefix = imagePaths.length === 1 ? '[Photo]' : imagePaths.length > 1 ? `[${imagePaths.length} Photos]` : ''
|
|
359
|
+
const historyQuery = photoPrefix ? `${photoPrefix} ${query || 'What do you see?'}` : query
|
|
360
|
+
addExchange(sid, 'user', historyQuery, globalMsgNum)
|
|
361
|
+
|
|
362
|
+
// Vision queries need the Read tool to see the image files
|
|
363
|
+
const tools = imagePaths.length > 0 ? 'WebSearch,WebFetch,Read' : 'WebSearch,WebFetch'
|
|
364
|
+
|
|
365
|
+
// Prepend image instruction when photos are attached
|
|
366
|
+
let fullQuery: string
|
|
367
|
+
if (imagePaths.length === 1) {
|
|
368
|
+
fullQuery = `The user has shared a photo from their phone camera. First, read the image file at ${imagePaths[0]} to see it. Then respond to their request: ${query || 'Describe what you see in this image concisely.'}`
|
|
369
|
+
} else if (imagePaths.length > 1) {
|
|
370
|
+
const fileList = imagePaths.map((p, i) => `${i + 1}. ${p}`).join('\n')
|
|
371
|
+
fullQuery = `The user has shared ${imagePaths.length} photos. Read each image file:\n${fileList}\nThen respond to their request: ${query || 'Describe what you see in these images concisely.'}`
|
|
372
|
+
} else {
|
|
373
|
+
fullQuery = query
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
// Check if we have a prior CLI session for this COS session.
|
|
377
|
+
// If not, use the pre-warmed session (eliminates 2-15s cold start on first query).
|
|
378
|
+
if (!existingCliSession && preWarmedCliSessionId && resolvedModel === 'opus') {
|
|
379
|
+
// Only Opus queries consume the pre-warmed session (pre-warmed with Opus).
|
|
380
|
+
// Hey Even (Haiku) cold-starts its own session to avoid model contamination.
|
|
381
|
+
existingCliSession = preWarmedCliSessionId
|
|
382
|
+
// Consume the pre-warmed session — next new session will cold start
|
|
383
|
+
// (but by then the first session's CLI session exists for --resume)
|
|
384
|
+
preWarmedCliSessionId = null
|
|
385
|
+
scheduleCliSessionSave()
|
|
386
|
+
console.log(`[claude-bridge] Using pre-warmed CLI session for first query (session: ${sid.slice(0, 8)}...)`)
|
|
387
|
+
|
|
388
|
+
// Fire-and-forget: pre-warm a fresh session for the NEXT new COS session
|
|
389
|
+
preWarmCLI().catch(() => {})
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
const args = [
|
|
393
|
+
'-p',
|
|
394
|
+
'--model', resolvedModel,
|
|
395
|
+
'--effort', getClaudeEffortLevel(),
|
|
396
|
+
'--output-format', 'stream-json',
|
|
397
|
+
'--verbose', // Required: stream-json requires --verbose
|
|
398
|
+
'--dangerously-skip-permissions', // Required: headless CLI mode with no TTY for user prompts
|
|
399
|
+
'--system-prompt', systemPrompt,
|
|
400
|
+
]
|
|
401
|
+
|
|
402
|
+
// Full COS path gets tools + partial messages; lightweight gets web search only
|
|
403
|
+
if (options?.lightweight) {
|
|
404
|
+
if (imagePaths.length > 0) {
|
|
405
|
+
args.push('--allowedTools', tools)
|
|
406
|
+
} else {
|
|
407
|
+
// Lightweight: web search for general questions, no Bash/Read/Write (saves 5-10s)
|
|
408
|
+
args.push('--allowedTools', 'WebSearch,WebFetch')
|
|
409
|
+
}
|
|
410
|
+
} else {
|
|
411
|
+
args.push('--allowedTools', tools, '--include-partial-messages')
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
if (existingCliSession) {
|
|
415
|
+
// Resume prior CLI session — reuses cached context, avoids cold start
|
|
416
|
+
args.push('--resume', existingCliSession)
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
// Strip CLAUDECODE env var so claude -p doesn't think it's nested
|
|
420
|
+
const env = { ...process.env }
|
|
421
|
+
delete env.CLAUDECODE
|
|
422
|
+
const cliCwd = COS_SCRIPTS_DIR ?? process.cwd()
|
|
423
|
+
const inactivityMs = INACTIVITY_BY_MODEL[resolvedModel]
|
|
424
|
+
const defaultWallMax = WALL_MAX_BY_MODEL[resolvedModel]
|
|
425
|
+
const wallMax = isExtendedQuery(query) ? WALL_MAX_EXTENDED_MS : defaultWallMax
|
|
426
|
+
const startTime = Date.now()
|
|
427
|
+
const run = startClaudeRun({
|
|
428
|
+
cosSessionId: sid,
|
|
429
|
+
model: resolvedModel,
|
|
430
|
+
cwd: cliCwd,
|
|
431
|
+
resumed: !!existingCliSession,
|
|
432
|
+
cliSessionId: existingCliSession,
|
|
433
|
+
timeoutMs: inactivityMs,
|
|
434
|
+
wallMaxMs: wallMax,
|
|
435
|
+
query: fullQuery,
|
|
436
|
+
})
|
|
437
|
+
|
|
438
|
+
const proc = spawn('claude', args, {
|
|
439
|
+
stdio: ['pipe', 'pipe', 'pipe'],
|
|
440
|
+
env,
|
|
441
|
+
cwd: cliCwd,
|
|
442
|
+
})
|
|
443
|
+
|
|
444
|
+
let fullText = ''
|
|
445
|
+
let stderr = ''
|
|
446
|
+
let buffer = ''
|
|
447
|
+
let finalized = false // Guard against double onDone/onError
|
|
448
|
+
let lastActivity = Date.now() // Tracks last stdout data for inactivity timeout
|
|
449
|
+
let receivedStreamEvents = false // Track if CLI emits stream_event (vs older assistant-only format)
|
|
450
|
+
|
|
451
|
+
function cleanupImages() {
|
|
452
|
+
for (const p of imagePaths) {
|
|
453
|
+
try { unlinkSync(p) } catch { /* ignore */ }
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
function finalize(text: string) {
|
|
458
|
+
if (finalized) return
|
|
459
|
+
finalized = true
|
|
460
|
+
cleanup()
|
|
461
|
+
cleanupImages()
|
|
462
|
+
|
|
463
|
+
// Token audit — log every completed claude -p call
|
|
464
|
+
const totalMs = Date.now() - startTime
|
|
465
|
+
const inputEstimate = systemPrompt.length + fullQuery.length + contextPrompt.length
|
|
466
|
+
logTokenAudit({
|
|
467
|
+
source: options?.lightweight ? 'g2-voice' : 'g2-query',
|
|
468
|
+
model: resolvedModel,
|
|
469
|
+
inputChars: inputEstimate,
|
|
470
|
+
outputChars: text.length,
|
|
471
|
+
durationMs: totalMs,
|
|
472
|
+
caller: options?.lightweight ? 'voice_query' : 'full_query',
|
|
473
|
+
})
|
|
474
|
+
|
|
475
|
+
addExchange(sid, 'assistant', text, globalMsgNum)
|
|
476
|
+
|
|
477
|
+
// Replace photo exchanges with condensed summaries to prevent context rot
|
|
478
|
+
// while preserving enough context for follow-up questions
|
|
479
|
+
if (imagePaths.length > 0) {
|
|
480
|
+
replaceLastExchangeWithSummary(sid, query, text, imagePaths.length)
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
finishClaudeRun(run.runId, {
|
|
484
|
+
status: 'completed',
|
|
485
|
+
startedAtMs: startTime,
|
|
486
|
+
output: text,
|
|
487
|
+
exitCode: 0,
|
|
488
|
+
})
|
|
489
|
+
|
|
490
|
+
callbacks.onDone(text, resolvedModel, cliSessionMap.get(resolvedCliKey))
|
|
491
|
+
|
|
492
|
+
// Telegram notifications — fire and forget
|
|
493
|
+
if (isFirstQuery) {
|
|
494
|
+
notifySessionStart(sid, query)
|
|
495
|
+
markSessionNotified(sid)
|
|
496
|
+
}
|
|
497
|
+
notifyExchange(sid, query, text)
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
function finalizeError(msg: string, exitCode?: number | null) {
|
|
501
|
+
if (finalized) return
|
|
502
|
+
finalized = true
|
|
503
|
+
cleanup()
|
|
504
|
+
cleanupImages()
|
|
505
|
+
finishClaudeRun(run.runId, {
|
|
506
|
+
status: 'failed',
|
|
507
|
+
startedAtMs: startTime,
|
|
508
|
+
error: msg,
|
|
509
|
+
exitCode,
|
|
510
|
+
})
|
|
511
|
+
callbacks.onError(msg)
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
// ─── Heartbeat: emit phase status during silence ───
|
|
515
|
+
|
|
516
|
+
const heartbeat = setInterval(() => {
|
|
517
|
+
if (finalized) return
|
|
518
|
+
// The client owns elapsed-time rendering so repeated heartbeats do not
|
|
519
|
+
// recreate visual pulses or duplicate "(72s)" suffixes.
|
|
520
|
+
const msg = PHASE_LABELS[phase] ?? 'Processing...'
|
|
521
|
+
callbacks.onToolStatus?.(msg)
|
|
522
|
+
}, HEARTBEAT_INTERVAL_MS)
|
|
523
|
+
|
|
524
|
+
// ─── Model-aware timeouts ───
|
|
525
|
+
|
|
526
|
+
// ─── Inactivity timeout: resets on any stdout data ───
|
|
527
|
+
|
|
528
|
+
let inactivityTimer = setTimeout(() => {
|
|
529
|
+
proc.kill('SIGTERM')
|
|
530
|
+
const elapsed = Math.round((Date.now() - startTime) / 1000)
|
|
531
|
+
finalizeError(`No output for ${inactivityMs / 1000}s (${elapsed}s total). Process killed.`)
|
|
532
|
+
}, inactivityMs)
|
|
533
|
+
|
|
534
|
+
function resetInactivity() {
|
|
535
|
+
lastActivity = Date.now()
|
|
536
|
+
clearTimeout(inactivityTimer)
|
|
537
|
+
inactivityTimer = setTimeout(() => {
|
|
538
|
+
proc.kill('SIGTERM')
|
|
539
|
+
const elapsed = Math.round((Date.now() - startTime) / 1000)
|
|
540
|
+
finalizeError(`No output for ${inactivityMs / 1000}s (${elapsed}s total). Process killed.`)
|
|
541
|
+
}, inactivityMs)
|
|
542
|
+
}
|
|
543
|
+
|
|
544
|
+
// ─── Wall clock max: absolute cap ───
|
|
545
|
+
|
|
546
|
+
const wallTimer = setTimeout(() => {
|
|
547
|
+
proc.kill('SIGTERM')
|
|
548
|
+
if (fullText) {
|
|
549
|
+
// Got partial output — deliver what we have
|
|
550
|
+
finalize(fullText)
|
|
551
|
+
} else {
|
|
552
|
+
finalizeError(`Wall clock limit reached (${wallMax / 1000}s). Process killed.`)
|
|
553
|
+
}
|
|
554
|
+
}, wallMax)
|
|
555
|
+
|
|
556
|
+
function cleanup() {
|
|
557
|
+
clearInterval(heartbeat)
|
|
558
|
+
clearTimeout(inactivityTimer)
|
|
559
|
+
clearTimeout(wallTimer)
|
|
560
|
+
}
|
|
561
|
+
|
|
562
|
+
// ─── Process stdout ───
|
|
563
|
+
|
|
564
|
+
proc.stdout.on('data', (chunk: Buffer) => {
|
|
565
|
+
resetInactivity()
|
|
566
|
+
buffer += chunk.toString()
|
|
567
|
+
|
|
568
|
+
// Process complete JSON lines
|
|
569
|
+
const lines = buffer.split('\n')
|
|
570
|
+
buffer = lines.pop() ?? '' // Keep incomplete line in buffer
|
|
571
|
+
|
|
572
|
+
for (const line of lines) {
|
|
573
|
+
const trimmed = line.trim()
|
|
574
|
+
if (!trimmed) continue
|
|
575
|
+
|
|
576
|
+
try {
|
|
577
|
+
const event = JSON.parse(trimmed)
|
|
578
|
+
|
|
579
|
+
if (event.type === 'stream_event') {
|
|
580
|
+
// Real-time token streaming — fires every few tokens during generation
|
|
581
|
+
receivedStreamEvents = true
|
|
582
|
+
const inner = event.event
|
|
583
|
+
if (inner?.type === 'content_block_delta' && inner.delta?.type === 'text_delta' && inner.delta.text) {
|
|
584
|
+
phase = 'generating'
|
|
585
|
+
fullText += inner.delta.text
|
|
586
|
+
callbacks.onChunk(inner.delta.text)
|
|
587
|
+
} else if (inner?.type === 'content_block_start' && inner.content_block?.type === 'thinking') {
|
|
588
|
+
phase = 'thinking'
|
|
589
|
+
callbacks.onToolStatus?.('Reasoning...')
|
|
590
|
+
} else if (inner?.type === 'content_block_start' && inner.content_block?.type === 'tool_use' && inner.content_block.name) {
|
|
591
|
+
const toolName = inner.content_block.name
|
|
592
|
+
if (toolName === 'WebSearch' || toolName === 'WebFetch') {
|
|
593
|
+
phase = 'searching'
|
|
594
|
+
}
|
|
595
|
+
callbacks.onToolStatus?.(toolName)
|
|
596
|
+
if (toolName === 'Read') {
|
|
597
|
+
callbacks.onToolStatus?.('Analyzing photo...')
|
|
598
|
+
}
|
|
599
|
+
}
|
|
600
|
+
} else if (event.type === 'assistant') {
|
|
601
|
+
// Fallback: only used if CLI doesn't emit stream_events (older CLI compatibility)
|
|
602
|
+
// Format A (streaming-json): { type: "assistant", message: { content: [{ type: "text", text: "..." }] } }
|
|
603
|
+
// Format B (legacy/chunk): { type: "assistant", subtype: "text", content: "..." }
|
|
604
|
+
if (!receivedStreamEvents) {
|
|
605
|
+
let text = ''
|
|
606
|
+
if (event.message?.content) {
|
|
607
|
+
for (const block of event.message.content) {
|
|
608
|
+
if (block.type === 'text' && block.text) text += block.text
|
|
609
|
+
if (block.type === 'tool_use' && block.name) {
|
|
610
|
+
if (block.name === 'WebSearch' || block.name === 'WebFetch') {
|
|
611
|
+
phase = 'searching'
|
|
612
|
+
}
|
|
613
|
+
callbacks.onToolStatus?.(block.name)
|
|
614
|
+
if (block.name === 'Read') {
|
|
615
|
+
callbacks.onToolStatus?.('Analyzing photo...')
|
|
616
|
+
}
|
|
617
|
+
}
|
|
618
|
+
}
|
|
619
|
+
} else if (event.subtype === 'text' && typeof event.content === 'string') {
|
|
620
|
+
text = event.content
|
|
621
|
+
}
|
|
622
|
+
if (text) {
|
|
623
|
+
phase = 'generating'
|
|
624
|
+
fullText += text
|
|
625
|
+
callbacks.onChunk(text)
|
|
626
|
+
}
|
|
627
|
+
}
|
|
628
|
+
} else if (event.type === 'result') {
|
|
629
|
+
// Capture CLI session ID for future --resume (avoids cold start on next query)
|
|
630
|
+
if (event.session_id) {
|
|
631
|
+
cliSessionMap.set(resolvedCliKey, event.session_id)
|
|
632
|
+
scheduleCliSessionSave()
|
|
633
|
+
updateClaudeRun(run.runId, { cliSessionId: event.session_id })
|
|
634
|
+
}
|
|
635
|
+
// Final result — use accumulated text (more reliable than result.result)
|
|
636
|
+
finalize(fullText || event.result || '')
|
|
637
|
+
}
|
|
638
|
+
// tool_use/tool_result/other events still reset inactivity (we got stdout data)
|
|
639
|
+
} catch {
|
|
640
|
+
// Not valid JSON — ignore partial lines
|
|
641
|
+
}
|
|
642
|
+
}
|
|
643
|
+
})
|
|
644
|
+
|
|
645
|
+
proc.stderr.on('data', (chunk: Buffer) => {
|
|
646
|
+
// stderr activity also counts — Claude CLI logs progress there
|
|
647
|
+
resetInactivity()
|
|
648
|
+
stderr += chunk.toString()
|
|
649
|
+
})
|
|
650
|
+
|
|
651
|
+
proc.on('close', (code) => {
|
|
652
|
+
// Process any remaining buffer
|
|
653
|
+
if (buffer.trim()) {
|
|
654
|
+
try {
|
|
655
|
+
const event = JSON.parse(buffer.trim())
|
|
656
|
+
if (event.type === 'result') {
|
|
657
|
+
finalize(fullText || event.result || '')
|
|
658
|
+
return
|
|
659
|
+
}
|
|
660
|
+
} catch { /* ignore */ }
|
|
661
|
+
}
|
|
662
|
+
|
|
663
|
+
if (code !== 0 && !fullText) {
|
|
664
|
+
finalizeError(`claude-bridge: exit ${code} — ${stderr.trim().slice(0, 200)}`, code)
|
|
665
|
+
} else if (fullText) {
|
|
666
|
+
// If we got text but no explicit result event, still finalize
|
|
667
|
+
finalize(fullText)
|
|
668
|
+
} else {
|
|
669
|
+
cleanup() // No output, no error — just clean up timers
|
|
670
|
+
}
|
|
671
|
+
})
|
|
672
|
+
|
|
673
|
+
proc.on('error', (err) => {
|
|
674
|
+
finalizeError(`claude-bridge: ${err.message}`, null)
|
|
675
|
+
})
|
|
676
|
+
|
|
677
|
+
// Send query via stdin (fullQuery includes image instruction when vision)
|
|
678
|
+
proc.stdin.write(fullQuery)
|
|
679
|
+
proc.stdin.end()
|
|
680
|
+
|
|
681
|
+
return sid
|
|
682
|
+
}
|