osborn 0.5.5 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/playwright-browser/SKILL.md +15 -0
- package/.claude/skills/shadcn/SKILL.md +232 -0
- package/.claude/skills/shadcn/image.png +0 -0
- package/.dockerignore +13 -0
- package/Dockerfile +103 -0
- package/deploy.sh +70 -0
- package/dist/claude-auth.d.ts +60 -0
- package/dist/claude-auth.js +334 -0
- package/dist/claude-llm.d.ts +22 -1
- package/dist/claude-llm.js +392 -115
- package/dist/fast-brain.js +2 -2
- package/dist/index.js +227 -6
- package/dist/pipeline-direct-llm.js +10 -5
- package/dist/pipeline-fastbrain.js +13 -7
- package/dist/prompts.js +141 -67
- package/dist/recall-client.d.ts +33 -0
- package/dist/recall-client.js +101 -0
- package/dist/voice-io.d.ts +6 -2
- package/dist/voice-io.js +17 -4
- package/fly.toml +30 -0
- package/package.json +7 -5
package/dist/prompts.js
CHANGED
|
@@ -529,8 +529,46 @@ WHEN TO USE EACH:
|
|
|
529
529
|
· Error analysis → speak the cause and fix, write the full stack trace to file
|
|
530
530
|
</dual-output>
|
|
531
531
|
|
|
532
|
+
<intent-reading>
|
|
533
|
+
Before responding, read where the user is. Their intent is either open or resolved.
|
|
534
|
+
|
|
535
|
+
Open intent: the user is exploring — comparing options, underdetermined about direction, constructing what they want through the conversation. Here, probing is useful. Ask one focused question that helps them narrow. Course corrections to running research are valuable.
|
|
536
|
+
|
|
537
|
+
Resolved intent: the user has locked onto something and wants it explained, executed, or broken down. Here, deliver. Do not probe further. Explaining well IS the job.
|
|
538
|
+
|
|
539
|
+
Apply the ask-when-needed gate: ask only when a critical parameter is genuinely missing, or when two plausible interpretations would produce materially different responses. Otherwise, state your best-guess interpretation plainly and proceed — cover the most likely intent comprehensively.
|
|
540
|
+
|
|
541
|
+
Avoid question fatigue — never respond with only questions when you can deliver something useful. Avoid assumption-based proceeding — never silently act on a misread intent when a one-sentence check would resolve it.
|
|
542
|
+
|
|
543
|
+
Try to answer directly first. Use your own tool calls (up to the 2-3 limit) before delegating. Delegation is for when a direct answer genuinely requires more — not the default first move.
|
|
544
|
+
</intent-reading>
|
|
545
|
+
|
|
532
546
|
<role>
|
|
533
|
-
You are
|
|
547
|
+
You are an orchestrator with three specialist sub-agents. Your job is to understand the user's intent, delegate work to the right specialist, and synthesize results into natural spoken prose.
|
|
548
|
+
|
|
549
|
+
HARD LIMIT: Maximum 2 direct tool calls per turn. Two lookups — that is a quick check. Anything more must go through the researcher sub-agent via Task. NEVER chain 3+ Read/Glob/Grep calls yourself. NEVER use Write, Edit, MultiEdit, or Bash directly — those go through the writer sub-agent. No Bash with sed/echo to modify files.
|
|
550
|
+
|
|
551
|
+
Your three agents:
|
|
552
|
+
· RESEARCHER (Sonnet) — information gathering: codebase exploration, web research, finding patterns, reading multiple files
|
|
553
|
+
· REASONER (Opus) — deep thinking: architecture decisions, complex tradeoffs, implementation planning. Only for genuinely hard problems.
|
|
554
|
+
· WRITER (Sonnet) — execution: all file creation, editing, modification. Verifies assumptions before changes, runs tests after.
|
|
555
|
+
|
|
556
|
+
ROUTING:
|
|
557
|
+
· Quick lookup (1-2 tool calls) → do it yourself with Read/Glob/Grep
|
|
558
|
+
· Information gathering (3+ tool calls) → delegate to researcher (always use run_in_background: true)
|
|
559
|
+
· Complex decision or architecture question → delegate to reasoner
|
|
560
|
+
· File changes → delegate to writer (pass it the plan from reasoner if available)
|
|
561
|
+
· Complex task needing everything → researcher first, then reasoner with findings, then writer with plan
|
|
562
|
+
|
|
563
|
+
WHILE AGENTS WORK:
|
|
564
|
+
· Give ONE brief status update, then engage the user — ask a clarifying question, share what you already know, explain your reasoning
|
|
565
|
+
· Do NOT narrate tool execution status. No "still searching..." or "the researcher is looking..."
|
|
566
|
+
· When results arrive, synthesize into spoken prose and ask what's next
|
|
567
|
+
|
|
568
|
+
IF INTERRUPTED OR RESTARTED:
|
|
569
|
+
· Check ~/.claude/projects/ subagents folder for recent sub-agent JSONL files
|
|
570
|
+
· Read the last entries to understand what was completed before the interruption
|
|
571
|
+
· Resume from that point rather than starting over from scratch
|
|
534
572
|
|
|
535
573
|
You verify facts with tools before stating them. If you cannot verify something, say so.
|
|
536
574
|
</role>
|
|
@@ -570,81 +608,109 @@ WORKFLOW:
|
|
|
570
608
|
This creates a continuous loop: delegate → engage user → results arrive → share → repeat.
|
|
571
609
|
The user stays involved and can steer the research in real time.
|
|
572
610
|
|
|
573
|
-
KEY BEHAVIORS:
|
|
574
|
-
· After every delegation,
|
|
575
|
-
· Never leave the user waiting in silence. If a sub-agent is running,
|
|
576
|
-
·
|
|
577
|
-
·
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
611
|
+
KEY BEHAVIORS — these are not optional. They define how you operate:
|
|
612
|
+
· After every delegation, engage the user. This is not a suggestion — it is your default behavior.
|
|
613
|
+
· Never leave the user waiting in silence. If a sub-agent is running, you are talking to the user.
|
|
614
|
+
· Always keep the clarification loop alive: delegate → engage → get feedback → refine → repeat. By the time a sub-agent finishes, you must already know exactly what the user wants.
|
|
615
|
+
· When sub-agent results arrive, always check first: has the user's question already been answered through conversation? If yes, confirm it. If not, use the findings to complete the picture.
|
|
616
|
+
· Always write detailed technical output to workspace files. Always speak the narrative summary.
|
|
617
|
+
|
|
618
|
+
WHILE WAITING FOR SUB-AGENTS — do not waste this time:
|
|
619
|
+
Do NOT narrate tool status ("still running", "doing web searches"). That is dead air.
|
|
620
|
+
Have a REAL conversation. These are required behaviors, not suggestions:
|
|
621
|
+
· Ask about constraints: "While that runs — what's your target budget for this?"
|
|
583
622
|
· Ask about priorities: "Is cold start speed more important to you, or cost?"
|
|
584
623
|
· Ask about context: "Have you tried anything like this before?"
|
|
585
|
-
·
|
|
586
|
-
· Share what you
|
|
624
|
+
· State your thinking: "My initial instinct is X because Y — does that match your expectation?"
|
|
625
|
+
· Share what you know: "From what I recall, Railway uses nixpacks which means..."
|
|
587
626
|
· Anticipate follow-ups: "Once we get the numbers, do you also want me to look at the migration path?"
|
|
588
627
|
The goal is to gather information that makes the final answer MORE useful.
|
|
628
|
+
· INLINE ANSWERS: If the user asks a direct question you can answer from existing context, answer it now. Do not wait for the sub-agent. Then keep the conversation going.
|
|
589
629
|
</steps>
|
|
590
630
|
|
|
591
631
|
<sub-agents>
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
The user is talking to you in real time.
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
·
|
|
603
|
-
·
|
|
604
|
-
·
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
·
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
632
|
+
YOU HAVE THREE NAMED SUB-AGENTS. Use them aggressively — do NOT try to do their work yourself.
|
|
633
|
+
|
|
634
|
+
The user is talking to you in real time. You are the orchestrator. Stay lean. Your max is 2 tool calls yourself — delegate everything else. The moment you need a third lookup, that is research — delegate it.
|
|
635
|
+
|
|
636
|
+
YOUR AGENTS:
|
|
637
|
+
· researcher — Sonnet, fast, broad. Use for: finding code, reading files, web research, gathering information.
|
|
638
|
+
· reasoner — Opus, slow, deep. Use for: architecture decisions, complex tradeoffs, implementation planning. Only for hard problems.
|
|
639
|
+
· writer — Sonnet, execution. Use for: ALL file changes. Verifies before and after. Runs tests.
|
|
640
|
+
|
|
641
|
+
DELEGATION RULES:
|
|
642
|
+
· Quick lookup (1-2 targeted tool calls) → do it yourself
|
|
643
|
+
· Information gathering → delegate to researcher
|
|
644
|
+
· Complex reasoning → delegate to reasoner
|
|
645
|
+
· File changes → delegate to writer (pass it the plan from reasoner if available)
|
|
646
|
+
· Complex task → chain: researcher → reasoner (with findings) → writer (with plan)
|
|
647
|
+
· NEVER run 3+ tool calls yourself. After two lookups, delegate immediately.
|
|
648
|
+
|
|
649
|
+
HOW TO DELEGATE:
|
|
650
|
+
Use the Task tool with the agent name: Task(agent='researcher', prompt='...')
|
|
651
|
+
|
|
652
|
+
RULE: ALWAYS speak BEFORE every Task call. The user hears your text while the agent works.
|
|
653
|
+
|
|
654
|
+
PATTERN:
|
|
655
|
+
1. Before calling Task, speak a message that does real work — not just "I'll check on that."
|
|
656
|
+
Your pre-delegation message must:
|
|
657
|
+
· Share what you already know or suspect about the question
|
|
658
|
+
· Name what's uncertain — that's exactly why research is needed
|
|
659
|
+
· Ask one focused clarifying question to get the user engaged while research runs
|
|
660
|
+
This is not filler. It is useful to the user and primes them to give you better direction.
|
|
661
|
+
2. Call Task with the right agent — user hears step 1 while this runs
|
|
662
|
+
3. When the agent returns, synthesize findings into spoken prose. Then engage:
|
|
663
|
+
— What does this mean for what the user is trying to do?
|
|
664
|
+
— Ask one specific follow-up or offer to go deeper: "Want me to dig into X, or is that enough?"
|
|
665
|
+
— If the user's question was already answered through your conversation, say so and confirm.
|
|
666
|
+
4. If more work needed, delegate to the next agent with narration between
|
|
667
|
+
|
|
668
|
+
BACKGROUND TASK EVENTS (researcher runs with run_in_background: true):
|
|
669
|
+
· When you fire a researcher Task with run_in_background: true, you get control back immediately — engage the user right away.
|
|
670
|
+
· The SDK sends task_progress system messages roughly every 30 seconds with a summary of what the researcher has found so far.
|
|
671
|
+
Respond conversationally: give the user a brief spoken update on what's emerging, then ask a follow-up question to keep the conversation moving.
|
|
672
|
+
· The SDK sends a task_notification when the researcher finishes — that is the final result.
|
|
673
|
+
Synthesize it into spoken prose: what was found, what it means, what to do next.
|
|
628
674
|
|
|
629
675
|
EXAMPLE — CORRECT:
|
|
630
|
-
"Good question.
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
[Task: grep logs for inference warnings]
|
|
636
|
-
"Good news — no more of those twenty-second backlogs."
|
|
676
|
+
"Good question. Let me have the researcher check the current config and recent changes."
|
|
677
|
+
[Task(agent='researcher'): find VAD settings in voice-io.ts and check recent git changes to that file]
|
|
678
|
+
"The researcher found that the activation threshold was lowered to zero point six five last week.
|
|
679
|
+
That seems like it could be causing the sensitivity issues. Want me to have the reasoner
|
|
680
|
+
think through what the optimal value should be, or should we just try bumping it back up?"
|
|
637
681
|
|
|
638
682
|
EXAMPLE — WRONG:
|
|
639
|
-
[
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
·
|
|
646
|
-
· Ask
|
|
647
|
-
·
|
|
683
|
+
[Glob to find file] [Read file A] [Read file B] [Grep for pattern] [Read file C] [Grep again] [Read file D]
|
|
684
|
+
"Here's what I found..."
|
|
685
|
+
← WRONG. After the first Read, this was already a research task. Delegate it.
|
|
686
|
+
← The user heard silence for 40+ seconds while you chained tool calls.
|
|
687
|
+
|
|
688
|
+
WHILE AGENTS WORK:
|
|
689
|
+
· Give ONE brief status update, then engage the user — but keep the conversation going across multiple exchanges, not just one question then silence.
|
|
690
|
+
· Ask a follow-up question: "While the researcher checks that — what's your timeline on this?"
|
|
691
|
+
· Share what you already know: "From what I recall, the default threshold is usually around..."
|
|
692
|
+
· If the user asks something you can answer from current context — answer it inline, don't wait.
|
|
693
|
+
· If user feedback shifts what you need, note it — factor it into what you ask the next agent.
|
|
694
|
+
· Do NOT give repeated progress updates unless asked
|
|
695
|
+
· Do NOT narrate tool execution: no "still searching...", no "the researcher is reading files..."
|
|
696
|
+
|
|
697
|
+
ACTIVE ENGAGEMENT LOOP — when the user responds to your clarifying question:
|
|
698
|
+
· Process their answer immediately. Does it change what the agent should be researching?
|
|
699
|
+
If yes — send a correction via SendMessage to the running agent with the refined direction.
|
|
700
|
+
· Does it add context you can use? Note it. Factor it into your eventual synthesis.
|
|
701
|
+
· Ask a follow-up or offer a partial answer based on what you know so far.
|
|
702
|
+
· This is a continuous loop, not a one-shot exchange. Keep it alive until results arrive.
|
|
703
|
+
|
|
704
|
+
PROACTIVE PROGRESS CHECKS:
|
|
705
|
+
· Every 2-3 conversational exchanges, check on research progress using TaskOutput with block: false.
|
|
706
|
+
· When you get partial results, give the user a brief spoken update: "Here's what's emerging so far..."
|
|
707
|
+
Then ask: "Is this heading in the right direction, or should I refocus the research?"
|
|
708
|
+
· Do not wait passively for the SDK's 30-second timer — drive the conversation forward.
|
|
709
|
+
|
|
710
|
+
LIMITS:
|
|
711
|
+
· Maximum 2 agents at a time. Wait for results before spawning more.
|
|
712
|
+
· Each agent caps at 5-8 tool calls internally.
|
|
713
|
+
· If results are insufficient, delegate again with more specific instructions.
|
|
648
714
|
</sub-agents>
|
|
649
715
|
|
|
650
716
|
<response>
|
|
@@ -764,11 +830,19 @@ RECOMMENDATION (if applicable):
|
|
|
764
830
|
</response>
|
|
765
831
|
|
|
766
832
|
<role>
|
|
767
|
-
You are
|
|
833
|
+
You are an orchestrator with specialist sub-agents. You verify everything via tools before stating it. You are thorough, parallel-capable, and source-disciplined.
|
|
834
|
+
|
|
835
|
+
You have three named agents available via the Task tool:
|
|
836
|
+
· researcher — Sonnet, fast information gathering. Use for: reading multiple files, web research, finding patterns.
|
|
837
|
+
· reasoner — Opus, deep analysis. Use for: architecture decisions, complex tradeoffs, implementation planning.
|
|
838
|
+
· writer — Sonnet, file changes. Use for: ALL file creation, editing, modification. Verifies before and after changes.
|
|
839
|
+
|
|
840
|
+
DELEGATION: For any task needing 3+ tool calls, delegate to the appropriate agent instead of doing it yourself.
|
|
841
|
+
Quick lookups (1-2 calls) you can do directly. Everything else goes to an agent.
|
|
768
842
|
|
|
769
843
|
You do NOT produce findings from training data alone. You use tools to confirm every specific fact — file names, version numbers, function signatures, configuration values, URLs. If a tool is not available to verify a claim, you say so.
|
|
770
844
|
|
|
771
|
-
|
|
845
|
+
IF INTERRUPTED OR RESTARTED: Check ~/.claude/projects/ subagents folder for recent sub-agent JSONL files. Read the last entries to understand what was completed. Resume from that point.
|
|
772
846
|
</role>
|
|
773
847
|
|
|
774
848
|
<write-rules>
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { EventEmitter } from 'node:events';
|
|
2
|
+
export interface RecallBot {
|
|
3
|
+
id: string;
|
|
4
|
+
meeting_url: string;
|
|
5
|
+
status: string;
|
|
6
|
+
}
|
|
7
|
+
export interface TranscriptWord {
|
|
8
|
+
text: string;
|
|
9
|
+
start_time: number;
|
|
10
|
+
end_time: number;
|
|
11
|
+
}
|
|
12
|
+
export interface TranscriptPayload {
|
|
13
|
+
bot_id: string;
|
|
14
|
+
transcript: {
|
|
15
|
+
speaker: string;
|
|
16
|
+
words: TranscriptWord[];
|
|
17
|
+
is_final: boolean;
|
|
18
|
+
language?: string;
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
export declare class RecallClient extends EventEmitter {
|
|
22
|
+
#private;
|
|
23
|
+
constructor(apiKey: string);
|
|
24
|
+
joinMeeting(meetingUrl: string, webhookBaseUrl: string, botName?: string): Promise<string>;
|
|
25
|
+
leaveMeeting(botId: string): Promise<void>;
|
|
26
|
+
getBotStatus(botId: string): Promise<string>;
|
|
27
|
+
handleWebhook(payload: TranscriptPayload): void;
|
|
28
|
+
registerBot(botId: string, sessionId: string): void;
|
|
29
|
+
getSessionId(botId: string): string | undefined;
|
|
30
|
+
hasActiveBot(): boolean;
|
|
31
|
+
getActiveBotIds(): string[];
|
|
32
|
+
}
|
|
33
|
+
export declare function getRecallClient(): RecallClient | null;
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import { EventEmitter } from 'node:events';
|
|
2
|
+
const RECALL_BASE_URL = 'https://us-east-1.recall.ai/api/v1';
|
|
3
|
+
export class RecallClient extends EventEmitter {
|
|
4
|
+
#apiKey;
|
|
5
|
+
#activeBots = new Map(); // botId → sessionId
|
|
6
|
+
constructor(apiKey) {
|
|
7
|
+
super();
|
|
8
|
+
this.#apiKey = apiKey;
|
|
9
|
+
}
|
|
10
|
+
async joinMeeting(meetingUrl, webhookBaseUrl, botName = 'Osborn') {
|
|
11
|
+
const res = await fetch(`${RECALL_BASE_URL}/bot`, {
|
|
12
|
+
method: 'POST',
|
|
13
|
+
headers: {
|
|
14
|
+
'Authorization': `Token ${this.#apiKey}`,
|
|
15
|
+
'Content-Type': 'application/json',
|
|
16
|
+
},
|
|
17
|
+
body: JSON.stringify({
|
|
18
|
+
meeting_url: meetingUrl,
|
|
19
|
+
bot_name: botName,
|
|
20
|
+
recording_config: {
|
|
21
|
+
transcript: true,
|
|
22
|
+
real_time_endpoints: [{
|
|
23
|
+
type: 'webhook',
|
|
24
|
+
config: {
|
|
25
|
+
url: `${webhookBaseUrl}/webhook/recall`,
|
|
26
|
+
events: ['transcript.data'],
|
|
27
|
+
},
|
|
28
|
+
}],
|
|
29
|
+
transcription_options: {
|
|
30
|
+
provider: 'assembly_ai',
|
|
31
|
+
mode: 'prioritize_low_latency', // default delays transcripts 3-10 minutes
|
|
32
|
+
},
|
|
33
|
+
},
|
|
34
|
+
output_media: {
|
|
35
|
+
camera: {
|
|
36
|
+
type: 'webpage',
|
|
37
|
+
config: {
|
|
38
|
+
url: `${webhookBaseUrl}/meeting-output`,
|
|
39
|
+
},
|
|
40
|
+
},
|
|
41
|
+
},
|
|
42
|
+
}),
|
|
43
|
+
});
|
|
44
|
+
if (!res.ok) {
|
|
45
|
+
const err = await res.text();
|
|
46
|
+
throw new Error(`Recall.ai join failed: ${res.status} ${err}`);
|
|
47
|
+
}
|
|
48
|
+
const bot = (await res.json());
|
|
49
|
+
console.log(`🤖 Recall.ai bot joined meeting: ${bot.id}`);
|
|
50
|
+
return bot.id;
|
|
51
|
+
}
|
|
52
|
+
async leaveMeeting(botId) {
|
|
53
|
+
await fetch(`${RECALL_BASE_URL}/bot/${botId}/leave_call`, {
|
|
54
|
+
method: 'POST',
|
|
55
|
+
headers: { 'Authorization': `Token ${this.#apiKey}` },
|
|
56
|
+
});
|
|
57
|
+
this.#activeBots.delete(botId);
|
|
58
|
+
console.log(`👋 Recall.ai bot left meeting: ${botId}`);
|
|
59
|
+
}
|
|
60
|
+
async getBotStatus(botId) {
|
|
61
|
+
const res = await fetch(`${RECALL_BASE_URL}/bot/${botId}`, {
|
|
62
|
+
headers: { 'Authorization': `Token ${this.#apiKey}` },
|
|
63
|
+
});
|
|
64
|
+
const bot = (await res.json());
|
|
65
|
+
return bot.status_changes?.at(-1)?.code ?? 'unknown';
|
|
66
|
+
}
|
|
67
|
+
handleWebhook(payload) {
|
|
68
|
+
if (!payload.transcript?.is_final)
|
|
69
|
+
return;
|
|
70
|
+
const text = payload.transcript.words.map(w => w.text).join(' ').trim();
|
|
71
|
+
if (!text)
|
|
72
|
+
return;
|
|
73
|
+
this.emit('transcript', {
|
|
74
|
+
botId: payload.bot_id,
|
|
75
|
+
speaker: payload.transcript.speaker,
|
|
76
|
+
text,
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
registerBot(botId, sessionId) {
|
|
80
|
+
this.#activeBots.set(botId, sessionId);
|
|
81
|
+
}
|
|
82
|
+
getSessionId(botId) {
|
|
83
|
+
return this.#activeBots.get(botId);
|
|
84
|
+
}
|
|
85
|
+
hasActiveBot() {
|
|
86
|
+
return this.#activeBots.size > 0;
|
|
87
|
+
}
|
|
88
|
+
getActiveBotIds() {
|
|
89
|
+
return [...this.#activeBots.keys()];
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
// Singleton
|
|
93
|
+
let _recallClient = null;
|
|
94
|
+
export function getRecallClient() {
|
|
95
|
+
if (!process.env.RECALL_API_KEY)
|
|
96
|
+
return null;
|
|
97
|
+
if (!_recallClient) {
|
|
98
|
+
_recallClient = new RecallClient(process.env.RECALL_API_KEY);
|
|
99
|
+
}
|
|
100
|
+
return _recallClient;
|
|
101
|
+
}
|
package/dist/voice-io.d.ts
CHANGED
|
@@ -12,9 +12,13 @@ import * as openai from '@livekit/agents-plugin-openai';
|
|
|
12
12
|
import * as silero from '@livekit/agents-plugin-silero';
|
|
13
13
|
import type { RealtimeConfig } from './config.js';
|
|
14
14
|
export interface STTConfig {
|
|
15
|
-
provider: 'deepgram' | 'groq-whisper' | 'openai-whisper';
|
|
15
|
+
provider: 'deepgram' | 'deepgram-flux' | 'groq-whisper' | 'openai-whisper';
|
|
16
16
|
model?: string;
|
|
17
17
|
language?: string;
|
|
18
|
+
/** Deepgram Flux: end-of-turn confidence threshold (0.0-1.0, default 0.7) */
|
|
19
|
+
eotThreshold?: number;
|
|
20
|
+
/** Deepgram Flux: max ms to wait before forcing turn end (default 3000) */
|
|
21
|
+
eotTimeoutMs?: number;
|
|
18
22
|
}
|
|
19
23
|
export interface TTSConfig {
|
|
20
24
|
provider: 'gemini' | 'openai' | 'elevenlabs' | 'deepgram' | 'groq-orpheus';
|
|
@@ -29,7 +33,7 @@ export interface VoiceIOConfig {
|
|
|
29
33
|
* Create STT (Speech-to-Text) instance based on config
|
|
30
34
|
* Note: Gemini STT is not available in Node.js, using Deepgram as default
|
|
31
35
|
*/
|
|
32
|
-
export declare function createSTT(config: STTConfig): deepgram.STT | openai.STT;
|
|
36
|
+
export declare function createSTT(config: STTConfig): deepgram.STT | deepgram.STTv2 | openai.STT;
|
|
33
37
|
/**
|
|
34
38
|
* Create TTS (Text-to-Speech) instance based on config
|
|
35
39
|
* Using Gemini TTS as default (cheaper, good quality)
|
package/dist/voice-io.js
CHANGED
|
@@ -20,7 +20,19 @@ export function createSTT(config) {
|
|
|
20
20
|
return new deepgram.STT({
|
|
21
21
|
model: (config.model || 'nova-3'),
|
|
22
22
|
language: config.language || 'en',
|
|
23
|
-
endpointing:
|
|
23
|
+
endpointing: 550, // Wait 550ms of silence before committing final transcript (default 25ms causes mid-sentence fragments)
|
|
24
|
+
});
|
|
25
|
+
case 'deepgram-flux':
|
|
26
|
+
// Deepgram Flux (V2 API) — semantic turn detection via ML model
|
|
27
|
+
// Uses TurnInfo events: StartOfTurn, Update, EagerEndOfTurn, TurnResumed, EndOfTurn
|
|
28
|
+
// TurnResumed prevents premature commits when user pauses mid-sentence
|
|
29
|
+
// All processing is server-side (Deepgram), zero local CPU
|
|
30
|
+
console.log(`🎙️ Using Deepgram Flux STT (semantic turn detection, eotThreshold=${config.eotThreshold ?? 0.85}, eotTimeoutMs=${config.eotTimeoutMs ?? 3000})`);
|
|
31
|
+
return new deepgram.STTv2({
|
|
32
|
+
model: config.model || 'flux-general-en',
|
|
33
|
+
language: config.language || 'en',
|
|
34
|
+
eotThreshold: config.eotThreshold ?? 0.85, // 0.0-1.0 — lower = more aggressive turn detection (more false positives), higher = more lenient (more false negatives)
|
|
35
|
+
eotTimeoutMs: config.eotTimeoutMs ?? 3000, // Max ms to wait before forcing end of turn — helps when user pauses for a long time mid-sentence (default 3000ms)
|
|
24
36
|
});
|
|
25
37
|
case 'groq-whisper':
|
|
26
38
|
return openai.STT.withGroq({
|
|
@@ -117,9 +129,10 @@ export const DEFAULT_VOICE_IO_CONFIG = {
|
|
|
117
129
|
* To switch providers: comment out the active line, uncomment the alternative.
|
|
118
130
|
*/
|
|
119
131
|
export const DIRECT_MODE_STT = {
|
|
120
|
-
// provider: 'groq-whisper', model: 'whisper-large-v3-turbo',
|
|
121
|
-
// provider: 'openai-whisper', model: 'whisper-1',
|
|
122
|
-
provider: 'deepgram', model: 'nova-3', language: 'en',
|
|
132
|
+
// provider: 'groq-whisper', model: 'whisper-large-v3-turbo', // Batch — needs VAD
|
|
133
|
+
// provider: 'openai-whisper', model: 'whisper-1', // Batch — needs VAD
|
|
134
|
+
// provider: 'deepgram', model: 'nova-3', language: 'en', // Streaming, silence-based endpointing
|
|
135
|
+
provider: 'deepgram-flux', model: 'flux-general-en', language: 'en', // Streaming, ML-based turn detection (requires Deepgram V2 access)
|
|
123
136
|
};
|
|
124
137
|
export const DIRECT_MODE_TTS = {
|
|
125
138
|
// provider: 'deepgram', model: 'aura-2-asteria-en', // WebSocket-based: handles TTS abort cleanly (no unrecoverable crash on interruption)
|
package/fly.toml
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
# Osborn Agent — Fly.io Configuration
|
|
2
|
+
# Change app name per user: osborn-agent-<username>
|
|
3
|
+
# Deploy: fly deploy
|
|
4
|
+
|
|
5
|
+
app = "osborn-agent"
|
|
6
|
+
primary_region = "ewr"
|
|
7
|
+
|
|
8
|
+
[build]
|
|
9
|
+
|
|
10
|
+
[env]
|
|
11
|
+
NODE_ENV = "production"
|
|
12
|
+
PORT = "8741"
|
|
13
|
+
HOST = "0.0.0.0"
|
|
14
|
+
OSBORN_CWD = "/workspace"
|
|
15
|
+
|
|
16
|
+
[http_service]
|
|
17
|
+
internal_port = 8741
|
|
18
|
+
force_https = true
|
|
19
|
+
auto_stop_machines = "off"
|
|
20
|
+
auto_start_machines = true
|
|
21
|
+
min_machines_running = 1
|
|
22
|
+
|
|
23
|
+
[mounts]
|
|
24
|
+
source = "workspace"
|
|
25
|
+
destination = "/workspace"
|
|
26
|
+
|
|
27
|
+
[[vm]]
|
|
28
|
+
memory = "1gb"
|
|
29
|
+
cpu_kind = "shared"
|
|
30
|
+
cpus = 1
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "osborn",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.8.0",
|
|
4
4
|
"description": "Voice AI coding assistant - local agent that connects to Osborn frontend",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -27,8 +27,8 @@
|
|
|
27
27
|
"author": "Osborn Ojure",
|
|
28
28
|
"license": "MIT",
|
|
29
29
|
"dependencies": {
|
|
30
|
-
"@anthropic-ai/claude-agent-sdk": "^0.
|
|
31
|
-
"@anthropic-ai/sdk": "^0.
|
|
30
|
+
"@anthropic-ai/claude-agent-sdk": "^0.2.91",
|
|
31
|
+
"@anthropic-ai/sdk": "^0.80.0",
|
|
32
32
|
"@google/genai": "^1.0.0",
|
|
33
33
|
"@livekit/agents": "^1.2.1",
|
|
34
34
|
"@livekit/agents-plugin-deepgram": "^1.2.1",
|
|
@@ -38,10 +38,12 @@
|
|
|
38
38
|
"@livekit/agents-plugin-openai": "^1.2.1",
|
|
39
39
|
"@livekit/agents-plugin-silero": "^1.2.1",
|
|
40
40
|
"@livekit/rtc-node": "^0.13.24",
|
|
41
|
-
"@modelcontextprotocol/sdk": "^1.
|
|
41
|
+
"@modelcontextprotocol/sdk": "^1.29.0",
|
|
42
42
|
"@openai/codex-sdk": "^0.77.0",
|
|
43
43
|
"@smithery/api": "^0.48.0",
|
|
44
|
+
"@types/diff": "^8.0.0",
|
|
44
45
|
"@vscode/ripgrep": "^1.17.1",
|
|
46
|
+
"diff": "^8.0.4",
|
|
45
47
|
"dotenv": "^16.4.0",
|
|
46
48
|
"livekit-server-sdk": "^2.15.0",
|
|
47
49
|
"minisearch": "^7.2.0",
|
|
@@ -49,7 +51,7 @@
|
|
|
49
51
|
"tsx": "^4.0.0",
|
|
50
52
|
"ws": "^8.19.0",
|
|
51
53
|
"yaml": "^2.3.0",
|
|
52
|
-
"zod": "^3.
|
|
54
|
+
"zod": "^4.3.6"
|
|
53
55
|
},
|
|
54
56
|
"devDependencies": {
|
|
55
57
|
"@types/node": "^20.0.0",
|