@praveencs/agent 0.9.30 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +55 -9
- package/ROADMAP.md +42 -50
- package/agent-skills/README.md +192 -0
- package/agent-skills/plugins/aws/plugin.json +22 -0
- package/agent-skills/plugins/aws/skills/aws-manager.md +17 -0
- package/agent-skills/plugins/aws/tools/aws.js +152 -0
- package/agent-skills/plugins/discord/plugin.json +22 -0
- package/agent-skills/plugins/discord/skills/discord-manager.md +15 -0
- package/agent-skills/plugins/discord/tools/discord.js +150 -0
- package/agent-skills/plugins/docker/plugin.json +21 -0
- package/agent-skills/plugins/docker/skills/docker-manager.md +15 -0
- package/agent-skills/plugins/docker/tools/docker.js +135 -0
- package/agent-skills/plugins/firebase/plugin.json +22 -0
- package/agent-skills/plugins/firebase/skills/firebase-manager.md +14 -0
- package/agent-skills/plugins/firebase/tools/firebase.js +157 -0
- package/agent-skills/plugins/github/plugin.json +23 -0
- package/agent-skills/plugins/github/skills/github-manager.md +15 -0
- package/agent-skills/plugins/github/tools/github.js +133 -0
- package/agent-skills/plugins/huggingface/plugin.json +22 -0
- package/agent-skills/plugins/huggingface/skills/huggingface-manager.md +16 -0
- package/agent-skills/plugins/huggingface/tools/huggingface.js +149 -0
- package/agent-skills/plugins/linear/plugin.json +22 -0
- package/agent-skills/plugins/linear/skills/linear-manager.md +16 -0
- package/agent-skills/plugins/linear/tools/linear.js +199 -0
- package/agent-skills/plugins/mongodb/plugin.json +21 -0
- package/agent-skills/plugins/mongodb/skills/mongodb-manager.md +14 -0
- package/agent-skills/plugins/mongodb/tools/mongodb.js +123 -0
- package/agent-skills/plugins/notion/plugin.json +22 -0
- package/agent-skills/plugins/notion/skills/notion-manager.md +16 -0
- package/agent-skills/plugins/notion/tools/notion.js +158 -0
- package/agent-skills/plugins/openai/plugin.json +23 -0
- package/agent-skills/plugins/openai/skills/openai-manager.md +15 -0
- package/agent-skills/plugins/openai/tools/openai.js +137 -0
- package/agent-skills/plugins/resend/plugin.json +21 -0
- package/agent-skills/plugins/resend/skills/resend-manager.md +15 -0
- package/agent-skills/plugins/resend/tools/resend.js +102 -0
- package/agent-skills/plugins/slack/plugin.json +22 -0
- package/agent-skills/plugins/slack/skills/slack-manager.md +15 -0
- package/agent-skills/plugins/slack/tools/slack.js +168 -0
- package/agent-skills/plugins/stripe/plugin.json +22 -0
- package/agent-skills/plugins/stripe/skills/stripe-manager.md +16 -0
- package/agent-skills/plugins/stripe/tools/stripe.js +174 -0
- package/agent-skills/plugins/supabase/plugin.json +22 -0
- package/agent-skills/plugins/supabase/skills/supabase-manager.md +16 -0
- package/agent-skills/plugins/supabase/tools/supabase.js +153 -0
- package/agent-skills/plugins/telegram/plugin.json +21 -0
- package/agent-skills/plugins/telegram/skills/telegram-manager.md +14 -0
- package/agent-skills/plugins/telegram/tools/telegram.js +131 -0
- package/agent-skills/plugins/vercel/plugin.json +22 -0
- package/agent-skills/plugins/vercel/skills/vercel-manager.md +15 -0
- package/agent-skills/plugins/vercel/tools/vercel.js +145 -0
- package/agent-skills/registry.json +675 -0
- package/agent-skills/skills/api-tester/prompt.md +27 -0
- package/agent-skills/skills/api-tester/skill.json +16 -0
- package/agent-skills/skills/backup/prompt.md +12 -0
- package/agent-skills/skills/backup/skill.json +1 -0
- package/agent-skills/skills/code-review/prompt.md +29 -0
- package/agent-skills/skills/code-review/skill.json +18 -0
- package/agent-skills/skills/create-note/prompt.md +21 -0
- package/agent-skills/skills/create-note/skill.json +15 -0
- package/agent-skills/skills/cron-scheduler/prompt.md +25 -0
- package/agent-skills/skills/cron-scheduler/skill.json +1 -0
- package/agent-skills/skills/db-query/prompt.md +12 -0
- package/agent-skills/skills/db-query/skill.json +1 -0
- package/agent-skills/skills/docker-deploy/prompt.md +21 -0
- package/agent-skills/skills/docker-deploy/skill.json +16 -0
- package/agent-skills/skills/file-organizer/prompt.md +32 -0
- package/agent-skills/skills/file-organizer/skill.json +19 -0
- package/agent-skills/skills/git-commit/prompt.md +21 -0
- package/agent-skills/skills/git-commit/skill.json +17 -0
- package/agent-skills/skills/log-analyzer/prompt.md +16 -0
- package/agent-skills/skills/log-analyzer/skill.json +1 -0
- package/agent-skills/skills/npm-publish/prompt.md +24 -0
- package/agent-skills/skills/npm-publish/skill.json +20 -0
- package/agent-skills/skills/open-vscode/prompt.md +8 -0
- package/agent-skills/skills/open-vscode/skill.json +15 -0
- package/agent-skills/skills/project-scaffold/prompt.md +43 -0
- package/agent-skills/skills/project-scaffold/skill.json +17 -0
- package/agent-skills/skills/send-email/prompt.md +19 -0
- package/agent-skills/skills/send-email/send.js +60 -0
- package/agent-skills/skills/send-email/skill.json +16 -0
- package/agent-skills/skills/system-monitor/prompt.md +27 -0
- package/agent-skills/skills/system-monitor/skill.json +15 -0
- package/agent-skills/skills/web-search/prompt.md +26 -0
- package/agent-skills/skills/web-search/skill.json +16 -0
- package/dist/src/cli/commands/desktop.d.ts +3 -0
- package/dist/src/cli/commands/desktop.d.ts.map +1 -0
- package/dist/src/cli/commands/desktop.js +80 -0
- package/dist/src/cli/commands/desktop.js.map +1 -0
- package/dist/src/cli/commands/multimodal.d.ts +3 -0
- package/dist/src/cli/commands/multimodal.d.ts.map +1 -0
- package/dist/src/cli/commands/multimodal.js +78 -0
- package/dist/src/cli/commands/multimodal.js.map +1 -0
- package/dist/src/cli/commands/sandbox.d.ts +3 -0
- package/dist/src/cli/commands/sandbox.d.ts.map +1 -0
- package/dist/src/cli/commands/sandbox.js +89 -0
- package/dist/src/cli/commands/sandbox.js.map +1 -0
- package/dist/src/cli/commands/swarm.d.ts +3 -0
- package/dist/src/cli/commands/swarm.d.ts.map +1 -0
- package/dist/src/cli/commands/swarm.js +120 -0
- package/dist/src/cli/commands/swarm.js.map +1 -0
- package/dist/src/cli/index.d.ts.map +1 -1
- package/dist/src/cli/index.js +8 -0
- package/dist/src/cli/index.js.map +1 -1
- package/dist/src/config/defaults.d.ts.map +1 -1
- package/dist/src/config/defaults.js +42 -0
- package/dist/src/config/defaults.js.map +1 -1
- package/dist/src/config/schema.d.ts +261 -0
- package/dist/src/config/schema.d.ts.map +1 -1
- package/dist/src/config/schema.js +58 -0
- package/dist/src/config/schema.js.map +1 -1
- package/dist/src/desktop/engine.d.ts +42 -0
- package/dist/src/desktop/engine.d.ts.map +1 -0
- package/dist/src/desktop/engine.js +77 -0
- package/dist/src/desktop/engine.js.map +1 -0
- package/dist/src/desktop/index.d.ts +6 -0
- package/dist/src/desktop/index.d.ts.map +1 -0
- package/dist/src/desktop/index.js +6 -0
- package/dist/src/desktop/index.js.map +1 -0
- package/dist/src/desktop/input.d.ts +20 -0
- package/dist/src/desktop/input.d.ts.map +1 -0
- package/dist/src/desktop/input.js +160 -0
- package/dist/src/desktop/input.js.map +1 -0
- package/dist/src/desktop/screen.d.ts +17 -0
- package/dist/src/desktop/screen.d.ts.map +1 -0
- package/dist/src/desktop/screen.js +120 -0
- package/dist/src/desktop/screen.js.map +1 -0
- package/dist/src/desktop/types.d.ts +50 -0
- package/dist/src/desktop/types.d.ts.map +1 -0
- package/dist/src/desktop/types.js +10 -0
- package/dist/src/desktop/types.js.map +1 -0
- package/dist/src/multimodal/engine.d.ts +33 -0
- package/dist/src/multimodal/engine.d.ts.map +1 -0
- package/dist/src/multimodal/engine.js +63 -0
- package/dist/src/multimodal/engine.js.map +1 -0
- package/dist/src/multimodal/index.d.ts +7 -0
- package/dist/src/multimodal/index.d.ts.map +1 -0
- package/dist/src/multimodal/index.js +7 -0
- package/dist/src/multimodal/index.js.map +1 -0
- package/dist/src/multimodal/tts.d.ts +11 -0
- package/dist/src/multimodal/tts.d.ts.map +1 -0
- package/dist/src/multimodal/tts.js +48 -0
- package/dist/src/multimodal/tts.js.map +1 -0
- package/dist/src/multimodal/types.d.ts +55 -0
- package/dist/src/multimodal/types.d.ts.map +1 -0
- package/dist/src/multimodal/types.js +20 -0
- package/dist/src/multimodal/types.js.map +1 -0
- package/dist/src/multimodal/vision.d.ts +15 -0
- package/dist/src/multimodal/vision.d.ts.map +1 -0
- package/dist/src/multimodal/vision.js +98 -0
- package/dist/src/multimodal/vision.js.map +1 -0
- package/dist/src/multimodal/voice.d.ts +11 -0
- package/dist/src/multimodal/voice.d.ts.map +1 -0
- package/dist/src/multimodal/voice.js +43 -0
- package/dist/src/multimodal/voice.js.map +1 -0
- package/dist/src/plans/types.d.ts +18 -18
- package/dist/src/sandbox/docker.d.ts +42 -0
- package/dist/src/sandbox/docker.d.ts.map +1 -0
- package/dist/src/sandbox/docker.js +131 -0
- package/dist/src/sandbox/docker.js.map +1 -0
- package/dist/src/sandbox/engine.d.ts +50 -0
- package/dist/src/sandbox/engine.d.ts.map +1 -0
- package/dist/src/sandbox/engine.js +133 -0
- package/dist/src/sandbox/engine.js.map +1 -0
- package/dist/src/sandbox/index.d.ts +5 -0
- package/dist/src/sandbox/index.d.ts.map +1 -0
- package/dist/src/sandbox/index.js +5 -0
- package/dist/src/sandbox/index.js.map +1 -0
- package/dist/src/sandbox/types.d.ts +41 -0
- package/dist/src/sandbox/types.d.ts.map +1 -0
- package/dist/src/sandbox/types.js +12 -0
- package/dist/src/sandbox/types.js.map +1 -0
- package/dist/src/scripts/types.d.ts +2 -2
- package/dist/src/server/app.d.ts.map +1 -1
- package/dist/src/server/app.js +133 -0
- package/dist/src/server/app.js.map +1 -1
- package/dist/src/swarm/bus.d.ts +27 -0
- package/dist/src/swarm/bus.d.ts.map +1 -0
- package/dist/src/swarm/bus.js +64 -0
- package/dist/src/swarm/bus.js.map +1 -0
- package/dist/src/swarm/index.d.ts +6 -0
- package/dist/src/swarm/index.d.ts.map +1 -0
- package/dist/src/swarm/index.js +6 -0
- package/dist/src/swarm/index.js.map +1 -0
- package/dist/src/swarm/orchestrator.d.ts +79 -0
- package/dist/src/swarm/orchestrator.d.ts.map +1 -0
- package/dist/src/swarm/orchestrator.js +271 -0
- package/dist/src/swarm/orchestrator.js.map +1 -0
- package/dist/src/swarm/roles.d.ts +18 -0
- package/dist/src/swarm/roles.d.ts.map +1 -0
- package/dist/src/swarm/roles.js +83 -0
- package/dist/src/swarm/roles.js.map +1 -0
- package/dist/src/swarm/types.d.ts +58 -0
- package/dist/src/swarm/types.d.ts.map +1 -0
- package/dist/src/swarm/types.js +10 -0
- package/dist/src/swarm/types.js.map +1 -0
- package/dist/src/tools/core/cmd.d.ts.map +1 -1
- package/dist/src/tools/core/cmd.js +28 -0
- package/dist/src/tools/core/cmd.js.map +1 -1
- package/docs/DOCUMENTATION.md +137 -0
- package/package.json +3 -1
- package/studio/README.md +73 -0
- package/studio/eslint.config.js +23 -0
- package/studio/index.html +13 -0
- package/studio/package-lock.json +4350 -0
- package/studio/package.json +44 -0
- package/studio/postcss.config.js +6 -0
- package/studio/public/vite.svg +1 -0
- package/studio/src/App.tsx +243 -0
- package/studio/src/assets/react.svg +1 -0
- package/studio/src/components/Capabilities.tsx +80 -0
- package/studio/src/components/CommandsManager.tsx +94 -0
- package/studio/src/components/CostDashboard.tsx +182 -0
- package/studio/src/components/CredentialCapture.tsx +196 -0
- package/studio/src/components/CredentialsManager.tsx +257 -0
- package/studio/src/components/DaemonPanel.tsx +91 -0
- package/studio/src/components/DesktopPanel.tsx +118 -0
- package/studio/src/components/GoalTemplates.tsx +190 -0
- package/studio/src/components/GoalsPanel.tsx +235 -0
- package/studio/src/components/MemoryExplorer.tsx +152 -0
- package/studio/src/components/MultimodalPanel.tsx +150 -0
- package/studio/src/components/NotificationsPanel.tsx +175 -0
- package/studio/src/components/PluginsManager.tsx +60 -0
- package/studio/src/components/SandboxPanel.tsx +118 -0
- package/studio/src/components/ScriptsManager.tsx +269 -0
- package/studio/src/components/SkillsManager.tsx +123 -0
- package/studio/src/components/SwarmPanel.tsx +149 -0
- package/studio/src/components/TaskStreaming.tsx +189 -0
- package/studio/src/components/Terminal.tsx +200 -0
- package/studio/src/index.css +51 -0
- package/studio/src/main.tsx +13 -0
- package/studio/tailwind.config.js +47 -0
- package/studio/tsconfig.app.json +28 -0
- package/studio/tsconfig.json +7 -0
- package/studio/tsconfig.node.json +26 -0
- package/studio/vite.config.ts +7 -0
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
export interface MultimodalConfig {
|
|
2
|
+
/** Enable multimodal features */
|
|
3
|
+
enabled: boolean;
|
|
4
|
+
/** OpenAI API key (for Whisper, TTS, Vision) */
|
|
5
|
+
apiKey?: string;
|
|
6
|
+
/** Voice input settings */
|
|
7
|
+
voice: VoiceConfig;
|
|
8
|
+
/** Vision/image analysis settings */
|
|
9
|
+
vision: VisionConfig;
|
|
10
|
+
/** Text-to-speech settings */
|
|
11
|
+
tts: TTSConfig;
|
|
12
|
+
}
|
|
13
|
+
export interface VoiceConfig {
|
|
14
|
+
/** Whisper model to use */
|
|
15
|
+
model: string;
|
|
16
|
+
/** Language code (auto-detect if omitted) */
|
|
17
|
+
language?: string;
|
|
18
|
+
/** Audio format for recording */
|
|
19
|
+
format: 'wav' | 'mp3' | 'webm';
|
|
20
|
+
}
|
|
21
|
+
export interface VisionConfig {
|
|
22
|
+
/** Model for image analysis */
|
|
23
|
+
model: string;
|
|
24
|
+
/** Max tokens for vision response */
|
|
25
|
+
maxTokens: number;
|
|
26
|
+
/** Detail level: low, high, auto */
|
|
27
|
+
detail: 'low' | 'high' | 'auto';
|
|
28
|
+
}
|
|
29
|
+
export interface TTSConfig {
|
|
30
|
+
/** TTS model */
|
|
31
|
+
model: string;
|
|
32
|
+
/** Voice: alloy, echo, fable, onyx, nova, shimmer */
|
|
33
|
+
voice: string;
|
|
34
|
+
/** Output format */
|
|
35
|
+
format: 'mp3' | 'opus' | 'aac' | 'flac';
|
|
36
|
+
/** Speed multiplier (0.25 - 4.0) */
|
|
37
|
+
speed: number;
|
|
38
|
+
}
|
|
39
|
+
export declare const DEFAULT_MULTIMODAL_CONFIG: MultimodalConfig;
|
|
40
|
+
export interface TranscriptionResult {
|
|
41
|
+
text: string;
|
|
42
|
+
language?: string;
|
|
43
|
+
duration?: number;
|
|
44
|
+
}
|
|
45
|
+
export interface VisionAnalysisResult {
|
|
46
|
+
description: string;
|
|
47
|
+
model: string;
|
|
48
|
+
tokensUsed: number;
|
|
49
|
+
}
|
|
50
|
+
export interface SpeechResult {
|
|
51
|
+
audioPath: string;
|
|
52
|
+
format: string;
|
|
53
|
+
text: string;
|
|
54
|
+
}
|
|
55
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/multimodal/types.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,gBAAgB;IAC7B,iCAAiC;IACjC,OAAO,EAAE,OAAO,CAAC;IACjB,gDAAgD;IAChD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,2BAA2B;IAC3B,KAAK,EAAE,WAAW,CAAC;IACnB,qCAAqC;IACrC,MAAM,EAAE,YAAY,CAAC;IACrB,8BAA8B;IAC9B,GAAG,EAAE,SAAS,CAAC;CAClB;AAED,MAAM,WAAW,WAAW;IACxB,2BAA2B;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,6CAA6C;IAC7C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,iCAAiC;IACjC,MAAM,EAAE,KAAK,GAAG,KAAK,GAAG,MAAM,CAAC;CAClC;AAED,MAAM,WAAW,YAAY;IACzB,+BAA+B;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,qCAAqC;IACrC,SAAS,EAAE,MAAM,CAAC;IAClB,oCAAoC;IACpC,MAAM,EAAE,KAAK,GAAG,MAAM,GAAG,MAAM,CAAC;CACnC;AAED,MAAM,WAAW,SAAS;IACtB,gBAAgB;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,qDAAqD;IACrD,KAAK,EAAE,MAAM,CAAC;IACd,oBAAoB;IACpB,MAAM,EAAE,KAAK,GAAG,MAAM,GAAG,KAAK,GAAG,MAAM,CAAC;IACxC,oCAAoC;IACpC,KAAK,EAAE,MAAM,CAAC;CACjB;AAED,eAAO,MAAM,yBAAyB,EAAE,gBAiBvC,CAAC;AAEF,MAAM,WAAW,mBAAmB;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,oBAAoB;IACjC,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,YAAY;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;CAChB"}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
// ─── Multimodal Types ───
|
|
2
|
+
export const DEFAULT_MULTIMODAL_CONFIG = {
|
|
3
|
+
enabled: false,
|
|
4
|
+
voice: {
|
|
5
|
+
model: 'whisper-1',
|
|
6
|
+
format: 'wav',
|
|
7
|
+
},
|
|
8
|
+
vision: {
|
|
9
|
+
model: 'gpt-4o',
|
|
10
|
+
maxTokens: 1024,
|
|
11
|
+
detail: 'auto',
|
|
12
|
+
},
|
|
13
|
+
tts: {
|
|
14
|
+
model: 'tts-1',
|
|
15
|
+
voice: 'alloy',
|
|
16
|
+
format: 'mp3',
|
|
17
|
+
speed: 1.0,
|
|
18
|
+
},
|
|
19
|
+
};
|
|
20
|
+
//# sourceMappingURL=types.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/multimodal/types.ts"],"names":[],"mappings":"AAAA,2BAA2B;AA4C3B,MAAM,CAAC,MAAM,yBAAyB,GAAqB;IACvD,OAAO,EAAE,KAAK;IACd,KAAK,EAAE;QACH,KAAK,EAAE,WAAW;QAClB,MAAM,EAAE,KAAK;KAChB;IACD,MAAM,EAAE;QACJ,KAAK,EAAE,QAAQ;QACf,SAAS,EAAE,IAAI;QACf,MAAM,EAAE,MAAM;KACjB;IACD,GAAG,EAAE;QACD,KAAK,EAAE,OAAO;QACd,KAAK,EAAE,OAAO;QACd,MAAM,EAAE,KAAK;QACb,KAAK,EAAE,GAAG;KACb;CACJ,CAAC"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { VisionConfig, VisionAnalysisResult } from './types.js';
|
|
2
|
+
export declare class VisionAnalyzer {
|
|
3
|
+
private config;
|
|
4
|
+
private apiKey;
|
|
5
|
+
constructor(config: VisionConfig, apiKey: string);
|
|
6
|
+
/**
|
|
7
|
+
* Analyze an image with a prompt.
|
|
8
|
+
*/
|
|
9
|
+
analyze(imagePath: string, prompt?: string): Promise<VisionAnalysisResult>;
|
|
10
|
+
/**
|
|
11
|
+
* Analyze an image from a URL.
|
|
12
|
+
*/
|
|
13
|
+
analyzeUrl(imageUrl: string, prompt?: string): Promise<VisionAnalysisResult>;
|
|
14
|
+
}
|
|
15
|
+
//# sourceMappingURL=vision.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"vision.d.ts","sourceRoot":"","sources":["../../../src/multimodal/vision.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,YAAY,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAErE,qBAAa,cAAc;IACvB,OAAO,CAAC,MAAM,CAAe;IAC7B,OAAO,CAAC,MAAM,CAAS;gBAEX,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM;IAKhD;;OAEG;IACG,OAAO,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,GAAE,MAAyC,GAAG,OAAO,CAAC,oBAAoB,CAAC;IAoDlH;;OAEG;IACG,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,GAAE,MAA+B,GAAG,OAAO,CAAC,oBAAoB,CAAC;CAyC7G"}
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
// ─── Vision (Image Analysis) ───
|
|
2
|
+
// Uses OpenAI GPT-4V / GPT-4o for image understanding.
|
|
3
|
+
import { readFileSync } from 'node:fs';
|
|
4
|
+
import { extname } from 'node:path';
|
|
5
|
+
export class VisionAnalyzer {
|
|
6
|
+
config;
|
|
7
|
+
apiKey;
|
|
8
|
+
constructor(config, apiKey) {
|
|
9
|
+
this.config = config;
|
|
10
|
+
this.apiKey = apiKey;
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Analyze an image with a prompt.
|
|
14
|
+
*/
|
|
15
|
+
async analyze(imagePath, prompt = 'Describe this image in detail.') {
|
|
16
|
+
// Read and base64-encode the image
|
|
17
|
+
const imageBuffer = readFileSync(imagePath);
|
|
18
|
+
const base64 = imageBuffer.toString('base64');
|
|
19
|
+
const ext = extname(imagePath).slice(1) || 'png';
|
|
20
|
+
const mimeType = ext === 'jpg' ? 'image/jpeg' : `image/${ext}`;
|
|
21
|
+
const dataUrl = `data:${mimeType};base64,${base64}`;
|
|
22
|
+
const res = await fetch('https://api.openai.com/v1/chat/completions', {
|
|
23
|
+
method: 'POST',
|
|
24
|
+
headers: {
|
|
25
|
+
'Content-Type': 'application/json',
|
|
26
|
+
'Authorization': `Bearer ${this.apiKey}`,
|
|
27
|
+
},
|
|
28
|
+
body: JSON.stringify({
|
|
29
|
+
model: this.config.model,
|
|
30
|
+
max_tokens: this.config.maxTokens,
|
|
31
|
+
messages: [
|
|
32
|
+
{
|
|
33
|
+
role: 'user',
|
|
34
|
+
content: [
|
|
35
|
+
{ type: 'text', text: prompt },
|
|
36
|
+
{
|
|
37
|
+
type: 'image_url',
|
|
38
|
+
image_url: {
|
|
39
|
+
url: dataUrl,
|
|
40
|
+
detail: this.config.detail,
|
|
41
|
+
},
|
|
42
|
+
},
|
|
43
|
+
],
|
|
44
|
+
},
|
|
45
|
+
],
|
|
46
|
+
}),
|
|
47
|
+
});
|
|
48
|
+
if (!res.ok) {
|
|
49
|
+
const err = await res.json();
|
|
50
|
+
throw new Error(`Vision API Error: ${err.error?.message || JSON.stringify(err)}`);
|
|
51
|
+
}
|
|
52
|
+
const data = await res.json();
|
|
53
|
+
return {
|
|
54
|
+
description: data.choices[0]?.message?.content || '',
|
|
55
|
+
model: this.config.model,
|
|
56
|
+
tokensUsed: data.usage?.total_tokens || 0,
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Analyze an image from a URL.
|
|
61
|
+
*/
|
|
62
|
+
async analyzeUrl(imageUrl, prompt = 'Describe this image.') {
|
|
63
|
+
const res = await fetch('https://api.openai.com/v1/chat/completions', {
|
|
64
|
+
method: 'POST',
|
|
65
|
+
headers: {
|
|
66
|
+
'Content-Type': 'application/json',
|
|
67
|
+
'Authorization': `Bearer ${this.apiKey}`,
|
|
68
|
+
},
|
|
69
|
+
body: JSON.stringify({
|
|
70
|
+
model: this.config.model,
|
|
71
|
+
max_tokens: this.config.maxTokens,
|
|
72
|
+
messages: [
|
|
73
|
+
{
|
|
74
|
+
role: 'user',
|
|
75
|
+
content: [
|
|
76
|
+
{ type: 'text', text: prompt },
|
|
77
|
+
{
|
|
78
|
+
type: 'image_url',
|
|
79
|
+
image_url: { url: imageUrl, detail: this.config.detail },
|
|
80
|
+
},
|
|
81
|
+
],
|
|
82
|
+
},
|
|
83
|
+
],
|
|
84
|
+
}),
|
|
85
|
+
});
|
|
86
|
+
if (!res.ok) {
|
|
87
|
+
const err = await res.json();
|
|
88
|
+
throw new Error(`Vision API Error: ${err.error?.message || JSON.stringify(err)}`);
|
|
89
|
+
}
|
|
90
|
+
const data = await res.json();
|
|
91
|
+
return {
|
|
92
|
+
description: data.choices[0]?.message?.content || '',
|
|
93
|
+
model: this.config.model,
|
|
94
|
+
tokensUsed: data.usage?.total_tokens || 0,
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
//# sourceMappingURL=vision.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"vision.js","sourceRoot":"","sources":["../../../src/multimodal/vision.ts"],"names":[],"mappings":"AAAA,kCAAkC;AAClC,uDAAuD;AAEvD,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAGpC,MAAM,OAAO,cAAc;IACf,MAAM,CAAe;IACrB,MAAM,CAAS;IAEvB,YAAY,MAAoB,EAAE,MAAc;QAC5C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACzB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO,CAAC,SAAiB,EAAE,SAAiB,gCAAgC;QAC9E,mCAAmC;QACnC,MAAM,WAAW,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC;QAC5C,MAAM,MAAM,GAAG,WAAW,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAC9C,MAAM,GAAG,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC;QACjD,MAAM,QAAQ,GAAG,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS,GAAG,EAAE,CAAC;QAC/D,MAAM,OAAO,GAAG,QAAQ,QAAQ,WAAW,MAAM,EAAE,CAAC;QAEpD,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,4CAA4C,EAAE;YAClE,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACL,cAAc,EAAE,kBAAkB;gBAClC,eAAe,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;aAC3C;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACjB,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK;gBACxB,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS;gBACjC,QAAQ,EAAE;oBACN;wBACI,IAAI,EAAE,MAAM;wBACZ,OAAO,EAAE;4BACL,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE;4BAC9B;gCACI,IAAI,EAAE,WAAW;gCACjB,SAAS,EAAE;oCACP,GAAG,EAAE,OAAO;oCACZ,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;iCAC7B;6BACJ;yBACJ;qBACJ;iBACJ;aACJ,CAAC;SACL,CAAC,CAAC;QAEH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACV,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,IAAI,EAAS,CAAC;YACpC,MAAM,IAAI,KAAK,CAAC,qBAAqB,GAAG,CAAC,KAAK,EAAE,OAAO,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACtF,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAG1B,CAAC;QAEF,OAAO;YACH,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,IAAI,EAAE;YACpD,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK;YACxB,UAAU,EAAE,IAAI,CAAC,KAAK,EAAE,YAAY,IAAI,CAAC;SAC5C,CAAC;IACN,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,QAAgB,EAAE,SAAiB,sBAAsB;QACtE,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,4CAA4C,EAAE;YAClE,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACL,cAAc,EAAE,kBAAkB;gBAClC,eAAe,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;aAC3C;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACjB,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK;gBACxB,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS;gBACjC,QAAQ,EAAE;oBACN;wBACI,IAAI,EAAE,MAAM;wBACZ,OAAO,EAAE;4BACL,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE;4BAC9B;gCACI,IAAI,EAAE,WAAW;gCACjB,SAAS,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE;6BAC3D;yBACJ;qBACJ;iBACJ;aACJ,CAAC;SACL,CAAC,CAAC;QAEH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACV,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,IAAI,EAAS,CAAC;YACpC,MAAM,IAAI,KAAK,CAAC,qBAAqB,GAAG,CAAC,KAAK,EAAE,OAAO,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACtF,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAG1B,CAAC;QAEF,OAAO;YACH,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,IAAI,EAAE;YACpD,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK;YACxB,UAAU,EAAE,IAAI,CAAC,KAAK,EAAE,YAAY,IAAI,CAAC;SAC5C,CAAC;IACN,CAAC;CACJ"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { VoiceConfig, TranscriptionResult } from './types.js';
|
|
2
|
+
export declare class VoiceInput {
|
|
3
|
+
private config;
|
|
4
|
+
private apiKey;
|
|
5
|
+
constructor(config: VoiceConfig, apiKey: string);
|
|
6
|
+
/**
|
|
7
|
+
* Transcribe an audio file using Whisper.
|
|
8
|
+
*/
|
|
9
|
+
transcribe(audioPath: string): Promise<TranscriptionResult>;
|
|
10
|
+
}
|
|
11
|
+
//# sourceMappingURL=voice.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"voice.d.ts","sourceRoot":"","sources":["../../../src/multimodal/voice.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,WAAW,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AAEnE,qBAAa,UAAU;IACnB,OAAO,CAAC,MAAM,CAAc;IAC5B,OAAO,CAAC,MAAM,CAAS;gBAEX,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM;IAK/C;;OAEG;IACG,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,mBAAmB,CAAC;CAiCpE"}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
// ─── Voice Input (Speech-to-Text) ───
|
|
2
|
+
// Uses OpenAI Whisper API for transcription.
|
|
3
|
+
import { readFileSync } from 'node:fs';
|
|
4
|
+
export class VoiceInput {
|
|
5
|
+
config;
|
|
6
|
+
apiKey;
|
|
7
|
+
constructor(config, apiKey) {
|
|
8
|
+
this.config = config;
|
|
9
|
+
this.apiKey = apiKey;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Transcribe an audio file using Whisper.
|
|
13
|
+
*/
|
|
14
|
+
async transcribe(audioPath) {
|
|
15
|
+
const audioData = readFileSync(audioPath);
|
|
16
|
+
const blob = new Blob([audioData]);
|
|
17
|
+
const formData = new FormData();
|
|
18
|
+
formData.append('file', blob, `audio.${this.config.format}`);
|
|
19
|
+
formData.append('model', this.config.model);
|
|
20
|
+
if (this.config.language) {
|
|
21
|
+
formData.append('language', this.config.language);
|
|
22
|
+
}
|
|
23
|
+
formData.append('response_format', 'verbose_json');
|
|
24
|
+
const res = await fetch('https://api.openai.com/v1/audio/transcriptions', {
|
|
25
|
+
method: 'POST',
|
|
26
|
+
headers: {
|
|
27
|
+
'Authorization': `Bearer ${this.apiKey}`,
|
|
28
|
+
},
|
|
29
|
+
body: formData,
|
|
30
|
+
});
|
|
31
|
+
if (!res.ok) {
|
|
32
|
+
const err = await res.json();
|
|
33
|
+
throw new Error(`Whisper API Error: ${err.error?.message || JSON.stringify(err)}`);
|
|
34
|
+
}
|
|
35
|
+
const data = await res.json();
|
|
36
|
+
return {
|
|
37
|
+
text: data.text,
|
|
38
|
+
language: data.language,
|
|
39
|
+
duration: data.duration,
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
//# sourceMappingURL=voice.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"voice.js","sourceRoot":"","sources":["../../../src/multimodal/voice.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,6CAA6C;AAE7C,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAGvC,MAAM,OAAO,UAAU;IACX,MAAM,CAAc;IACpB,MAAM,CAAS;IAEvB,YAAY,MAAmB,EAAE,MAAc;QAC3C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACzB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,SAAiB;QAC9B,MAAM,SAAS,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC;QAC1C,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;QAEnC,MAAM,QAAQ,GAAG,IAAI,QAAQ,EAAE,CAAC;QAChC,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,SAAS,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QAC7D,QAAQ,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC5C,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;YACvB,QAAQ,CAAC,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACtD,CAAC;QACD,QAAQ,CAAC,MAAM,CAAC,iBAAiB,EAAE,cAAc,CAAC,CAAC;QAEnD,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,gDAAgD,EAAE;YACtE,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACL,eAAe,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;aAC3C;YACD,IAAI,EAAE,QAAQ;SACjB,CAAC,CAAC;QAEH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACV,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,IAAI,EAAS,CAAC;YACpC,MAAM,IAAI,KAAK,CAAC,sBAAsB,GAAG,CAAC,KAAK,EAAE,OAAO,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACvF,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAA4D,CAAC;QAExF,OAAO;YACH,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,QAAQ,EAAE,IAAI,CAAC,QAAQ;SAC1B,CAAC;IACN,CAAC;CACJ"}
|
|
@@ -83,12 +83,12 @@ export declare const GoalSchema: z.ZodObject<{
|
|
|
83
83
|
description: string;
|
|
84
84
|
id: string;
|
|
85
85
|
successCriteria: string[];
|
|
86
|
-
riskLevel: "low" | "
|
|
86
|
+
riskLevel: "low" | "high" | "medium";
|
|
87
87
|
}, {
|
|
88
88
|
description: string;
|
|
89
89
|
id: string;
|
|
90
90
|
successCriteria?: string[] | undefined;
|
|
91
|
-
riskLevel?: "low" | "
|
|
91
|
+
riskLevel?: "low" | "high" | "medium" | undefined;
|
|
92
92
|
}>;
|
|
93
93
|
export declare const VerifySchema: z.ZodObject<{
|
|
94
94
|
command: z.ZodOptional<z.ZodString>;
|
|
@@ -96,13 +96,13 @@ export declare const VerifySchema: z.ZodObject<{
|
|
|
96
96
|
exitCode: z.ZodOptional<z.ZodNumber>;
|
|
97
97
|
contains: z.ZodOptional<z.ZodString>;
|
|
98
98
|
}, "strip", z.ZodTypeAny, {
|
|
99
|
-
command?: string | undefined;
|
|
100
99
|
exitCode?: number | undefined;
|
|
100
|
+
command?: string | undefined;
|
|
101
101
|
fileExists?: string | undefined;
|
|
102
102
|
contains?: string | undefined;
|
|
103
103
|
}, {
|
|
104
|
-
command?: string | undefined;
|
|
105
104
|
exitCode?: number | undefined;
|
|
105
|
+
command?: string | undefined;
|
|
106
106
|
fileExists?: string | undefined;
|
|
107
107
|
contains?: string | undefined;
|
|
108
108
|
}>;
|
|
@@ -119,13 +119,13 @@ export declare const StepSchema: z.ZodObject<{
|
|
|
119
119
|
exitCode: z.ZodOptional<z.ZodNumber>;
|
|
120
120
|
contains: z.ZodOptional<z.ZodString>;
|
|
121
121
|
}, "strip", z.ZodTypeAny, {
|
|
122
|
-
command?: string | undefined;
|
|
123
122
|
exitCode?: number | undefined;
|
|
123
|
+
command?: string | undefined;
|
|
124
124
|
fileExists?: string | undefined;
|
|
125
125
|
contains?: string | undefined;
|
|
126
126
|
}, {
|
|
127
|
-
command?: string | undefined;
|
|
128
127
|
exitCode?: number | undefined;
|
|
128
|
+
command?: string | undefined;
|
|
129
129
|
fileExists?: string | undefined;
|
|
130
130
|
contains?: string | undefined;
|
|
131
131
|
}>>;
|
|
@@ -142,8 +142,8 @@ export declare const StepSchema: z.ZodObject<{
|
|
|
142
142
|
tool?: string | undefined;
|
|
143
143
|
skill?: string | undefined;
|
|
144
144
|
verify?: {
|
|
145
|
-
command?: string | undefined;
|
|
146
145
|
exitCode?: number | undefined;
|
|
146
|
+
command?: string | undefined;
|
|
147
147
|
fileExists?: string | undefined;
|
|
148
148
|
contains?: string | undefined;
|
|
149
149
|
} | undefined;
|
|
@@ -156,8 +156,8 @@ export declare const StepSchema: z.ZodObject<{
|
|
|
156
156
|
tool?: string | undefined;
|
|
157
157
|
skill?: string | undefined;
|
|
158
158
|
verify?: {
|
|
159
|
-
command?: string | undefined;
|
|
160
159
|
exitCode?: number | undefined;
|
|
160
|
+
command?: string | undefined;
|
|
161
161
|
fileExists?: string | undefined;
|
|
162
162
|
contains?: string | undefined;
|
|
163
163
|
} | undefined;
|
|
@@ -235,12 +235,12 @@ export declare const PlanSchema: z.ZodObject<{
|
|
|
235
235
|
description: string;
|
|
236
236
|
id: string;
|
|
237
237
|
successCriteria: string[];
|
|
238
|
-
riskLevel: "low" | "
|
|
238
|
+
riskLevel: "low" | "high" | "medium";
|
|
239
239
|
}, {
|
|
240
240
|
description: string;
|
|
241
241
|
id: string;
|
|
242
242
|
successCriteria?: string[] | undefined;
|
|
243
|
-
riskLevel?: "low" | "
|
|
243
|
+
riskLevel?: "low" | "high" | "medium" | undefined;
|
|
244
244
|
}>, "many">;
|
|
245
245
|
steps: z.ZodArray<z.ZodObject<{
|
|
246
246
|
id: z.ZodString;
|
|
@@ -255,13 +255,13 @@ export declare const PlanSchema: z.ZodObject<{
|
|
|
255
255
|
exitCode: z.ZodOptional<z.ZodNumber>;
|
|
256
256
|
contains: z.ZodOptional<z.ZodString>;
|
|
257
257
|
}, "strip", z.ZodTypeAny, {
|
|
258
|
-
command?: string | undefined;
|
|
259
258
|
exitCode?: number | undefined;
|
|
259
|
+
command?: string | undefined;
|
|
260
260
|
fileExists?: string | undefined;
|
|
261
261
|
contains?: string | undefined;
|
|
262
262
|
}, {
|
|
263
|
-
command?: string | undefined;
|
|
264
263
|
exitCode?: number | undefined;
|
|
264
|
+
command?: string | undefined;
|
|
265
265
|
fileExists?: string | undefined;
|
|
266
266
|
contains?: string | undefined;
|
|
267
267
|
}>>;
|
|
@@ -278,8 +278,8 @@ export declare const PlanSchema: z.ZodObject<{
|
|
|
278
278
|
tool?: string | undefined;
|
|
279
279
|
skill?: string | undefined;
|
|
280
280
|
verify?: {
|
|
281
|
-
command?: string | undefined;
|
|
282
281
|
exitCode?: number | undefined;
|
|
282
|
+
command?: string | undefined;
|
|
283
283
|
fileExists?: string | undefined;
|
|
284
284
|
contains?: string | undefined;
|
|
285
285
|
} | undefined;
|
|
@@ -292,8 +292,8 @@ export declare const PlanSchema: z.ZodObject<{
|
|
|
292
292
|
tool?: string | undefined;
|
|
293
293
|
skill?: string | undefined;
|
|
294
294
|
verify?: {
|
|
295
|
-
command?: string | undefined;
|
|
296
295
|
exitCode?: number | undefined;
|
|
296
|
+
command?: string | undefined;
|
|
297
297
|
fileExists?: string | undefined;
|
|
298
298
|
contains?: string | undefined;
|
|
299
299
|
} | undefined;
|
|
@@ -373,7 +373,7 @@ export declare const PlanSchema: z.ZodObject<{
|
|
|
373
373
|
description: string;
|
|
374
374
|
id: string;
|
|
375
375
|
successCriteria: string[];
|
|
376
|
-
riskLevel: "low" | "
|
|
376
|
+
riskLevel: "low" | "high" | "medium";
|
|
377
377
|
}[];
|
|
378
378
|
steps: {
|
|
379
379
|
name: string;
|
|
@@ -385,8 +385,8 @@ export declare const PlanSchema: z.ZodObject<{
|
|
|
385
385
|
tool?: string | undefined;
|
|
386
386
|
skill?: string | undefined;
|
|
387
387
|
verify?: {
|
|
388
|
-
command?: string | undefined;
|
|
389
388
|
exitCode?: number | undefined;
|
|
389
|
+
command?: string | undefined;
|
|
390
390
|
fileExists?: string | undefined;
|
|
391
391
|
contains?: string | undefined;
|
|
392
392
|
} | undefined;
|
|
@@ -411,7 +411,7 @@ export declare const PlanSchema: z.ZodObject<{
|
|
|
411
411
|
description: string;
|
|
412
412
|
id: string;
|
|
413
413
|
successCriteria?: string[] | undefined;
|
|
414
|
-
riskLevel?: "low" | "
|
|
414
|
+
riskLevel?: "low" | "high" | "medium" | undefined;
|
|
415
415
|
}[];
|
|
416
416
|
steps: {
|
|
417
417
|
name: string;
|
|
@@ -421,8 +421,8 @@ export declare const PlanSchema: z.ZodObject<{
|
|
|
421
421
|
tool?: string | undefined;
|
|
422
422
|
skill?: string | undefined;
|
|
423
423
|
verify?: {
|
|
424
|
-
command?: string | undefined;
|
|
425
424
|
exitCode?: number | undefined;
|
|
425
|
+
command?: string | undefined;
|
|
426
426
|
fileExists?: string | undefined;
|
|
427
427
|
contains?: string | undefined;
|
|
428
428
|
} | undefined;
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import type { ContainerState, SandboxMount } from './types.js';
|
|
2
|
+
export declare class DockerClient {
|
|
3
|
+
/**
|
|
4
|
+
* Check if Docker is available on the system.
|
|
5
|
+
*/
|
|
6
|
+
isAvailable(): Promise<boolean>;
|
|
7
|
+
/**
|
|
8
|
+
* Pull a Docker image if not already present.
|
|
9
|
+
*/
|
|
10
|
+
pullImage(image: string): Promise<void>;
|
|
11
|
+
/**
|
|
12
|
+
* Create and start a new container.
|
|
13
|
+
*/
|
|
14
|
+
createContainer(options: {
|
|
15
|
+
name: string;
|
|
16
|
+
image: string;
|
|
17
|
+
mounts: SandboxMount[];
|
|
18
|
+
network: string;
|
|
19
|
+
projectRoot: string;
|
|
20
|
+
}): Promise<ContainerState>;
|
|
21
|
+
/**
|
|
22
|
+
* Execute a command inside a running container.
|
|
23
|
+
*/
|
|
24
|
+
exec(containerId: string, command: string, cmdArgs?: string[], options?: {
|
|
25
|
+
cwd?: string;
|
|
26
|
+
timeout?: number;
|
|
27
|
+
env?: Record<string, string>;
|
|
28
|
+
}): Promise<{
|
|
29
|
+
stdout: string;
|
|
30
|
+
stderr: string;
|
|
31
|
+
exitCode: number;
|
|
32
|
+
}>;
|
|
33
|
+
/**
|
|
34
|
+
* Check if a container is running.
|
|
35
|
+
*/
|
|
36
|
+
isRunning(containerId: string): Promise<boolean>;
|
|
37
|
+
/**
|
|
38
|
+
* Stop and remove a container.
|
|
39
|
+
*/
|
|
40
|
+
destroyContainer(containerId: string): Promise<void>;
|
|
41
|
+
}
|
|
42
|
+
//# sourceMappingURL=docker.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"docker.d.ts","sourceRoot":"","sources":["../../../src/sandbox/docker.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAI/D,qBAAa,YAAY;IACrB;;OAEG;IACG,WAAW,IAAI,OAAO,CAAC,OAAO,CAAC;IAWrC;;OAEG;IACG,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAY7C;;OAEG;IACG,eAAe,CAAC,OAAO,EAAE;QAC3B,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,YAAY,EAAE,CAAC;QACvB,OAAO,EAAE,MAAM,CAAC;QAChB,WAAW,EAAE,MAAM,CAAC;KACvB,GAAG,OAAO,CAAC,cAAc,CAAC;IAoC3B;;OAEG;IACG,IAAI,CACN,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,MAAM,EACf,OAAO,GAAE,MAAM,EAAO,EACtB,OAAO,GAAE;QACL,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAC3B,GACP,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC;IAkChE;;OAEG;IACG,SAAS,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAWtD;;OAEG;IACG,gBAAgB,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;CAS7D"}
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
// ─── Docker Client ───
|
|
2
|
+
// Lightweight Docker Engine API client using fetch over Unix socket.
|
|
3
|
+
// No npm dependencies — talks directly to /var/run/docker.sock.
|
|
4
|
+
import { execFile } from 'node:child_process';
|
|
5
|
+
import { promisify } from 'node:util';
|
|
6
|
+
const execFileAsync = promisify(execFile);
|
|
7
|
+
export class DockerClient {
|
|
8
|
+
/**
|
|
9
|
+
* Check if Docker is available on the system.
|
|
10
|
+
*/
|
|
11
|
+
async isAvailable() {
|
|
12
|
+
try {
|
|
13
|
+
await execFileAsync('docker', ['info', '--format', '{{.ServerVersion}}'], {
|
|
14
|
+
timeout: 5000,
|
|
15
|
+
});
|
|
16
|
+
return true;
|
|
17
|
+
}
|
|
18
|
+
catch {
|
|
19
|
+
return false;
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Pull a Docker image if not already present.
|
|
24
|
+
*/
|
|
25
|
+
async pullImage(image) {
|
|
26
|
+
try {
|
|
27
|
+
// Check if image exists locally
|
|
28
|
+
await execFileAsync('docker', ['image', 'inspect', image], { timeout: 5000 });
|
|
29
|
+
}
|
|
30
|
+
catch {
|
|
31
|
+
// Image not found locally — pull it
|
|
32
|
+
await execFileAsync('docker', ['pull', image], {
|
|
33
|
+
timeout: 120000, // 2 min for large images
|
|
34
|
+
});
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Create and start a new container.
|
|
39
|
+
*/
|
|
40
|
+
async createContainer(options) {
|
|
41
|
+
const args = [
|
|
42
|
+
'run', '-d',
|
|
43
|
+
'--name', options.name,
|
|
44
|
+
'--network', options.network,
|
|
45
|
+
'--workdir', '/project',
|
|
46
|
+
// Keep the container alive with a sleep process
|
|
47
|
+
'--entrypoint', 'tail',
|
|
48
|
+
];
|
|
49
|
+
// Add volume mounts
|
|
50
|
+
for (const mount of options.mounts) {
|
|
51
|
+
const hostPath = mount.hostPath === '.'
|
|
52
|
+
? options.projectRoot
|
|
53
|
+
: mount.hostPath;
|
|
54
|
+
const flag = mount.readOnly ? ':ro' : '';
|
|
55
|
+
args.push('-v', `${hostPath}:${mount.containerPath}${flag}`);
|
|
56
|
+
}
|
|
57
|
+
args.push(options.image, '-f', '/dev/null');
|
|
58
|
+
const { stdout } = await execFileAsync('docker', args, {
|
|
59
|
+
timeout: 30000,
|
|
60
|
+
});
|
|
61
|
+
const containerId = stdout.trim();
|
|
62
|
+
return {
|
|
63
|
+
containerId,
|
|
64
|
+
name: options.name,
|
|
65
|
+
running: true,
|
|
66
|
+
image: options.image,
|
|
67
|
+
createdAt: new Date(),
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Execute a command inside a running container.
|
|
72
|
+
*/
|
|
73
|
+
async exec(containerId, command, cmdArgs = [], options = {}) {
|
|
74
|
+
const args = ['exec'];
|
|
75
|
+
// Working directory
|
|
76
|
+
if (options.cwd) {
|
|
77
|
+
args.push('-w', options.cwd);
|
|
78
|
+
}
|
|
79
|
+
// Environment variables
|
|
80
|
+
if (options.env) {
|
|
81
|
+
for (const [key, val] of Object.entries(options.env)) {
|
|
82
|
+
args.push('-e', `${key}=${val}`);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
args.push(containerId, command, ...cmdArgs);
|
|
86
|
+
try {
|
|
87
|
+
const { stdout, stderr } = await execFileAsync('docker', args, {
|
|
88
|
+
timeout: options.timeout || 30000,
|
|
89
|
+
maxBuffer: 10 * 1024 * 1024,
|
|
90
|
+
shell: true,
|
|
91
|
+
});
|
|
92
|
+
return { stdout: stdout.toString(), stderr: stderr.toString(), exitCode: 0 };
|
|
93
|
+
}
|
|
94
|
+
catch (err) {
|
|
95
|
+
const error = err;
|
|
96
|
+
return {
|
|
97
|
+
stdout: error.stdout?.toString() ?? '',
|
|
98
|
+
stderr: error.stderr?.toString() ?? error.message ?? '',
|
|
99
|
+
exitCode: error.code ?? 1,
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* Check if a container is running.
|
|
105
|
+
*/
|
|
106
|
+
async isRunning(containerId) {
|
|
107
|
+
try {
|
|
108
|
+
const { stdout } = await execFileAsync('docker', [
|
|
109
|
+
'inspect', '--format', '{{.State.Running}}', containerId,
|
|
110
|
+
], { timeout: 5000 });
|
|
111
|
+
return stdout.trim() === 'true';
|
|
112
|
+
}
|
|
113
|
+
catch {
|
|
114
|
+
return false;
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
/**
|
|
118
|
+
* Stop and remove a container.
|
|
119
|
+
*/
|
|
120
|
+
async destroyContainer(containerId) {
|
|
121
|
+
try {
|
|
122
|
+
await execFileAsync('docker', ['rm', '-f', containerId], {
|
|
123
|
+
timeout: 10000,
|
|
124
|
+
});
|
|
125
|
+
}
|
|
126
|
+
catch {
|
|
127
|
+
// Container may already be gone — ignore
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
//# sourceMappingURL=docker.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"docker.js","sourceRoot":"","sources":["../../../src/sandbox/docker.ts"],"names":[],"mappings":"AAAA,wBAAwB;AACxB,qEAAqE;AACrE,gEAAgE;AAEhE,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAGtC,MAAM,aAAa,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;AAE1C,MAAM,OAAO,YAAY;IACrB;;OAEG;IACH,KAAK,CAAC,WAAW;QACb,IAAI,CAAC;YACD,MAAM,aAAa,CAAC,QAAQ,EAAE,CAAC,MAAM,EAAE,UAAU,EAAE,oBAAoB,CAAC,EAAE;gBACtE,OAAO,EAAE,IAAI;aAChB,CAAC,CAAC;YACH,OAAO,IAAI,CAAC;QAChB,CAAC;QAAC,MAAM,CAAC;YACL,OAAO,KAAK,CAAC;QACjB,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,SAAS,CAAC,KAAa;QACzB,IAAI,CAAC;YACD,gCAAgC;YAChC,MAAM,aAAa,CAAC,QAAQ,EAAE,CAAC,OAAO,EAAE,SAAS,EAAE,KAAK,CAAC,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QAClF,CAAC;QAAC,MAAM,CAAC;YACL,oCAAoC;YACpC,MAAM,aAAa,CAAC,QAAQ,EAAE,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE;gBAC3C,OAAO,EAAE,MAAM,EAAE,yBAAyB;aAC7C,CAAC,CAAC;QACP,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,eAAe,CAAC,OAMrB;QACG,MAAM,IAAI,GAAG;YACT,KAAK,EAAE,IAAI;YACX,QAAQ,EAAE,OAAO,CAAC,IAAI;YACtB,WAAW,EAAE,OAAO,CAAC,OAAO;YAC5B,WAAW,EAAE,UAAU;YACvB,gDAAgD;YAChD,cAAc,EAAE,MAAM;SACzB,CAAC;QAEF,oBAAoB;QACpB,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACjC,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,KAAK,GAAG;gBACnC,CAAC,CAAC,OAAO,CAAC,WAAW;gBACrB,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC;YACrB,MAAM,IAAI,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;YACzC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,QAAQ,IAAI,KAAK,CAAC,aAAa,GAAG,IAAI,EAAE,CAAC,CAAC;QACjE,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,EAAE,WAAW,CAAC,CAAC;QAE5C,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,aAAa,CAAC,QAAQ,EAAE,IAAI,EAAE;YACnD,OAAO,EAAE,KAAK;SACjB,CAAC,CAAC;QAEH,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC;QAElC,OAAO;YACH,WAAW;YACX,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,OAAO,EAAE,IAAI;YACb,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,SAAS,EAAE,IAAI,IAAI,EAAE;SACxB,CAAC;IACN,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI,CACN,WAAmB,EACnB,OAAe,EACf,UAAoB,EAAE,EACtB,UAII,EAAE;QAEN,MAAM,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;QAEtB,oBAAoB;QACpB,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;YACd,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC;QACjC,CAAC;QAED,wBAAwB;QACxB,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;YACd,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;gBACnD,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,GAAG,IAAI,GAAG,EAAE,CAAC,CAAC;YACrC,CAAC;QACL,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,CAAC;QAE5C,IAAI,CAAC;YACD,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,aAAa,CAAC,QAAQ,EAAE,IAAI,EAAE;gBAC3D,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,KAAK;gBACjC,SAAS,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI;gBAC3B,KAAK,EAAE,IAAI;aACd,CAAC,CAAC;YACH,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;QACjF,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,MAAM,KAAK,GAAG,GAA4E,CAAC;YAC3F,OAAO;gBACH,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE;gBACtC,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,QAAQ,EAAE,IAAI,KAAK,CAAC,OAAO,IAAI,EAAE;gBACvD,QAAQ,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC;aAC5B,CAAC;QACN,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,SAAS,CAAC,WAAmB;QAC/B,IAAI,CAAC;YACD,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,aAAa,CAAC,QAAQ,EAAE;gBAC7C,SAAS,EAAE,UAAU,EAAE,oBAAoB,EAAE,WAAW;aAC3D,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;YACtB,OAAO,MAAM,CAAC,IAAI,EAAE,KAAK,MAAM,CAAC;QACpC,CAAC;QAAC,MAAM,CAAC;YACL,OAAO,KAAK,CAAC;QACjB,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,gBAAgB,CAAC,WAAmB;QACtC,IAAI,CAAC;YACD,MAAM,aAAa,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,WAAW,CAAC,EAAE;gBACrD,OAAO,EAAE,KAAK;aACjB,CAAC,CAAC;QACP,CAAC;QAAC,MAAM,CAAC;YACL,yCAAyC;QAC7C,CAAC;IACL,CAAC;CACJ"}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import type { SandboxConfig, ContainerState, SandboxExecResult } from './types.js';
|
|
2
|
+
export declare class SandboxEngine {
|
|
3
|
+
private docker;
|
|
4
|
+
private config;
|
|
5
|
+
private container;
|
|
6
|
+
private projectRoot;
|
|
7
|
+
constructor(projectRoot: string, config?: Partial<SandboxConfig>);
|
|
8
|
+
/** Whether sandbox mode is enabled */
|
|
9
|
+
get enabled(): boolean;
|
|
10
|
+
/** Whether the sandbox container is currently running */
|
|
11
|
+
get isActive(): boolean;
|
|
12
|
+
/** Current container info */
|
|
13
|
+
get containerInfo(): ContainerState | null;
|
|
14
|
+
/**
|
|
15
|
+
* Start the sandbox: pull image, create container.
|
|
16
|
+
* Call this when the daemon starts or when the user runs `agent sandbox start`.
|
|
17
|
+
*/
|
|
18
|
+
start(): Promise<ContainerState>;
|
|
19
|
+
/**
|
|
20
|
+
* Execute a command inside the sandbox.
|
|
21
|
+
* This is the main entry point used by cmd.run when sandbox is enabled.
|
|
22
|
+
*/
|
|
23
|
+
exec(command: string, args?: string[], options?: {
|
|
24
|
+
cwd?: string;
|
|
25
|
+
timeout?: number;
|
|
26
|
+
env?: Record<string, string>;
|
|
27
|
+
}): Promise<SandboxExecResult>;
|
|
28
|
+
/**
|
|
29
|
+
* Stop and destroy the sandbox container.
|
|
30
|
+
*/
|
|
31
|
+
stop(): Promise<void>;
|
|
32
|
+
/**
|
|
33
|
+
* Get sandbox status for CLI/Studio display.
|
|
34
|
+
*/
|
|
35
|
+
status(): Promise<{
|
|
36
|
+
enabled: boolean;
|
|
37
|
+
running: boolean;
|
|
38
|
+
image: string;
|
|
39
|
+
containerId?: string;
|
|
40
|
+
uptime?: number;
|
|
41
|
+
}>;
|
|
42
|
+
/**
|
|
43
|
+
* Map a host filesystem path to the equivalent container path
|
|
44
|
+
* based on the configured volume mounts.
|
|
45
|
+
*/
|
|
46
|
+
private mapHostPathToContainer;
|
|
47
|
+
}
|
|
48
|
+
export declare function getSandboxEngine(): SandboxEngine | null;
|
|
49
|
+
export declare function initSandboxEngine(projectRoot: string, config?: Partial<SandboxConfig>): SandboxEngine;
|
|
50
|
+
//# sourceMappingURL=engine.d.ts.map
|