mycontext-cli 4.2.8 → 4.2.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. package/README.md +13 -4
  2. package/dist/README.md +13 -4
  3. package/dist/agents/implementations/ArchitectAgent.js.map +1 -1
  4. package/dist/agents/implementations/BuildStrategyAgent.js.map +1 -1
  5. package/dist/agents/implementations/CodeGenSubAgent.js.map +1 -1
  6. package/dist/agents/implementations/DemoRecorderAgent.d.ts +73 -0
  7. package/dist/agents/implementations/DemoRecorderAgent.d.ts.map +1 -0
  8. package/dist/agents/implementations/DemoRecorderAgent.js +485 -0
  9. package/dist/agents/implementations/DemoRecorderAgent.js.map +1 -0
  10. package/dist/agents/implementations/DesignPipelineAgent.d.ts +4 -0
  11. package/dist/agents/implementations/DesignPipelineAgent.d.ts.map +1 -1
  12. package/dist/agents/implementations/DesignPipelineAgent.js +82 -10
  13. package/dist/agents/implementations/DesignPipelineAgent.js.map +1 -1
  14. package/dist/agents/implementations/DocsSubAgent.js.map +1 -1
  15. package/dist/agents/implementations/FeaturePlannerAgent.d.ts +10 -0
  16. package/dist/agents/implementations/FeaturePlannerAgent.d.ts.map +1 -0
  17. package/dist/agents/implementations/FeaturePlannerAgent.js +35 -0
  18. package/dist/agents/implementations/FeaturePlannerAgent.js.map +1 -0
  19. package/dist/agents/implementations/QASubAgent.js.map +1 -1
  20. package/dist/agents/implementations/SecurityAgent.js.map +1 -1
  21. package/dist/agents/implementations/VOScriptExtractorAgent.d.ts +84 -0
  22. package/dist/agents/implementations/VOScriptExtractorAgent.d.ts.map +1 -0
  23. package/dist/agents/implementations/VOScriptExtractorAgent.js +414 -0
  24. package/dist/agents/implementations/VOScriptExtractorAgent.js.map +1 -0
  25. package/dist/agents/implementations/VisionNavigatorAgent.d.ts +52 -0
  26. package/dist/agents/implementations/VisionNavigatorAgent.d.ts.map +1 -0
  27. package/dist/agents/implementations/VisionNavigatorAgent.js +313 -0
  28. package/dist/agents/implementations/VisionNavigatorAgent.js.map +1 -0
  29. package/dist/agents/implementations/VisionTestCoordinator.d.ts +51 -0
  30. package/dist/agents/implementations/VisionTestCoordinator.d.ts.map +1 -0
  31. package/dist/agents/implementations/VisionTestCoordinator.js +348 -0
  32. package/dist/agents/implementations/VisionTestCoordinator.js.map +1 -0
  33. package/dist/agents/implementations/VisualValidatorAgent.d.ts +83 -0
  34. package/dist/agents/implementations/VisualValidatorAgent.d.ts.map +1 -0
  35. package/dist/agents/implementations/VisualValidatorAgent.js +472 -0
  36. package/dist/agents/implementations/VisualValidatorAgent.js.map +1 -0
  37. package/dist/agents/orchestrator/SubAgentOrchestrator.js.map +1 -1
  38. package/dist/cli.js +3 -0
  39. package/dist/cli.js.map +1 -1
  40. package/dist/clients/ClaudeSDKClient.d.ts +4 -0
  41. package/dist/clients/ClaudeSDKClient.d.ts.map +1 -1
  42. package/dist/clients/ClaudeSDKClient.js +8 -0
  43. package/dist/clients/ClaudeSDKClient.js.map +1 -1
  44. package/dist/clients/MyContextAIClient.d.ts +4 -0
  45. package/dist/clients/MyContextAIClient.d.ts.map +1 -1
  46. package/dist/clients/MyContextAIClient.js +69 -0
  47. package/dist/clients/MyContextAIClient.js.map +1 -1
  48. package/dist/clients/ProviderChain.d.ts +1 -0
  49. package/dist/clients/ProviderChain.d.ts.map +1 -1
  50. package/dist/clients/ProviderChain.js +3 -0
  51. package/dist/clients/ProviderChain.js.map +1 -1
  52. package/dist/clients/XAIClient.d.ts +4 -0
  53. package/dist/clients/XAIClient.d.ts.map +1 -1
  54. package/dist/clients/XAIClient.js +8 -0
  55. package/dist/clients/XAIClient.js.map +1 -1
  56. package/dist/commands/agent.d.ts.map +1 -1
  57. package/dist/commands/agent.js +10 -1
  58. package/dist/commands/agent.js.map +1 -1
  59. package/dist/commands/init.d.ts.map +1 -1
  60. package/dist/commands/init.js +110 -204
  61. package/dist/commands/init.js.map +1 -1
  62. package/dist/commands/plan-feature.d.ts +3 -0
  63. package/dist/commands/plan-feature.d.ts.map +1 -0
  64. package/dist/commands/plan-feature.js +57 -0
  65. package/dist/commands/plan-feature.js.map +1 -0
  66. package/dist/commands/sanitize.d.ts.map +1 -1
  67. package/dist/commands/sanitize.js +95 -7
  68. package/dist/commands/sanitize.js.map +1 -1
  69. package/dist/commands/vision-test.d.ts +67 -0
  70. package/dist/commands/vision-test.d.ts.map +1 -0
  71. package/dist/commands/vision-test.js +347 -0
  72. package/dist/commands/vision-test.js.map +1 -0
  73. package/dist/core/ai/AICore.d.ts +13 -0
  74. package/dist/core/ai/AICore.d.ts.map +1 -1
  75. package/dist/core/ai/AICore.js +65 -0
  76. package/dist/core/ai/AICore.js.map +1 -1
  77. package/dist/doctor/rules/node-rules.d.ts.map +1 -1
  78. package/dist/doctor/rules/node-rules.js +41 -0
  79. package/dist/doctor/rules/node-rules.js.map +1 -1
  80. package/dist/interfaces/AIClient.d.ts +1 -0
  81. package/dist/interfaces/AIClient.d.ts.map +1 -1
  82. package/dist/interfaces/AIClient.js.map +1 -1
  83. package/dist/mcp/vision-test-runner.d.ts +57 -0
  84. package/dist/mcp/vision-test-runner.d.ts.map +1 -0
  85. package/dist/mcp/vision-test-runner.js +337 -0
  86. package/dist/mcp/vision-test-runner.js.map +1 -0
  87. package/dist/mcp/vision-testing-server.d.ts +38 -0
  88. package/dist/mcp/vision-testing-server.d.ts.map +1 -0
  89. package/dist/mcp/vision-testing-server.js +637 -0
  90. package/dist/mcp/vision-testing-server.js.map +1 -0
  91. package/dist/package.json +1 -1
  92. package/dist/services/ProjectScanner.d.ts.map +1 -1
  93. package/dist/services/ProjectScanner.js +8 -5
  94. package/dist/services/ProjectScanner.js.map +1 -1
  95. package/dist/services/gemini-vision.d.ts +3 -11
  96. package/dist/services/gemini-vision.d.ts.map +1 -1
  97. package/dist/services/gemini-vision.js +13 -132
  98. package/dist/services/gemini-vision.js.map +1 -1
  99. package/dist/services/llmClarifier.d.ts +9 -0
  100. package/dist/services/llmClarifier.d.ts.map +1 -0
  101. package/dist/services/llmClarifier.js +22 -0
  102. package/dist/services/llmClarifier.js.map +1 -0
  103. package/dist/services/vision-element-finder.d.ts +64 -0
  104. package/dist/services/vision-element-finder.d.ts.map +1 -0
  105. package/dist/services/vision-element-finder.js +446 -0
  106. package/dist/services/vision-element-finder.js.map +1 -0
  107. package/dist/types/flow-testing.d.ts +1 -1
  108. package/dist/types/flow-testing.d.ts.map +1 -1
  109. package/dist/types/vision-testing.d.ts +382 -0
  110. package/dist/types/vision-testing.d.ts.map +1 -0
  111. package/dist/types/vision-testing.js +9 -0
  112. package/dist/types/vision-testing.js.map +1 -0
  113. package/dist/utils/claudeAgentClient.d.ts +4 -0
  114. package/dist/utils/claudeAgentClient.d.ts.map +1 -1
  115. package/dist/utils/claudeAgentClient.js +8 -0
  116. package/dist/utils/claudeAgentClient.js.map +1 -1
  117. package/dist/utils/geminiClient.d.ts +41 -6
  118. package/dist/utils/geminiClient.d.ts.map +1 -1
  119. package/dist/utils/geminiClient.js +124 -18
  120. package/dist/utils/geminiClient.js.map +1 -1
  121. package/dist/utils/githubModelsClient.d.ts +19 -6
  122. package/dist/utils/githubModelsClient.d.ts.map +1 -1
  123. package/dist/utils/githubModelsClient.js +94 -7
  124. package/dist/utils/githubModelsClient.js.map +1 -1
  125. package/dist/utils/openRouterClient.d.ts +12 -3
  126. package/dist/utils/openRouterClient.d.ts.map +1 -1
  127. package/dist/utils/openRouterClient.js +91 -2
  128. package/dist/utils/openRouterClient.js.map +1 -1
  129. package/dist/utils/unifiedDesignContextLoader.d.ts.map +1 -1
  130. package/dist/utils/unifiedDesignContextLoader.js +15 -0
  131. package/dist/utils/unifiedDesignContextLoader.js.map +1 -1
  132. package/package.json +1 -1
@@ -0,0 +1,84 @@
1
+ /**
2
+ * Voice-Over Script Extractor Agent
3
+ *
4
+ * Extracts text from screenshots and generates professional narration scripts
5
+ * with timestamps. Supports multiple narration styles and optional TTS integration.
6
+ *
7
+ * Communicates via Living Brain (context.json).
8
+ */
9
+ import { SubAgent } from "../interfaces/SubAgent";
10
+ import { VisionTestStep, VOScript, DemoConfig } from "../../types/vision-testing";
11
+ export interface VOScriptExtractorInput {
12
+ steps: VisionTestStep[];
13
+ missionName: string;
14
+ missionDescription?: string;
15
+ narrationStyle?: "technical" | "casual" | "marketing" | "educational";
16
+ projectPath: string;
17
+ outputPath: string;
18
+ generateAudio?: boolean;
19
+ ttsConfig?: DemoConfig["ttsConfig"];
20
+ }
21
+ export interface VOScriptExtractorOutput {
22
+ script: VOScript;
23
+ scriptPath: string;
24
+ audioPath?: string;
25
+ success: boolean;
26
+ error?: string;
27
+ }
28
+ export declare class VOScriptExtractorAgent implements SubAgent<VOScriptExtractorInput, VOScriptExtractorOutput> {
29
+ name: string;
30
+ description: string;
31
+ personality: string;
32
+ llmProvider: string;
33
+ expertise: string[];
34
+ private visionService;
35
+ private contextService?;
36
+ constructor();
37
+ run(input: VOScriptExtractorInput): Promise<VOScriptExtractorOutput>;
38
+ /**
39
+ * Generate a single narration segment
40
+ */
41
+ private generateSegment;
42
+ /**
43
+ * Extract text from screenshot using vision AI
44
+ */
45
+ private extractTextFromScreenshot;
46
+ /**
47
+ * Generate narration for a step
48
+ */
49
+ private generateNarration;
50
+ /**
51
+ * Generate fallback narration
52
+ */
53
+ private generateFallbackNarration;
54
+ /**
55
+ * Build full script markdown
56
+ */
57
+ private buildFullScript;
58
+ /**
59
+ * Generate audio using TTS
60
+ */
61
+ private generateAudio;
62
+ /**
63
+ * Format timestamp as MM:SS
64
+ */
65
+ private formatTimestamp;
66
+ /**
67
+ * Format duration as human-readable string
68
+ */
69
+ private formatDuration;
70
+ /**
71
+ * Write update to Living Brain
72
+ */
73
+ private writeLivingBrain;
74
+ validate(input: VOScriptExtractorInput): Promise<boolean>;
75
+ getStatus(): Promise<{
76
+ name: string;
77
+ status: "idle" | "running" | "completed" | "error";
78
+ lastRun?: Date;
79
+ executionTime?: number;
80
+ errorCount: number;
81
+ successCount: number;
82
+ }>;
83
+ }
84
+ //# sourceMappingURL=VOScriptExtractorAgent.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"VOScriptExtractorAgent.d.ts","sourceRoot":"","sources":["../../../src/agents/implementations/VOScriptExtractorAgent.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EACL,QAAQ,EACT,MAAM,wBAAwB,CAAC;AAGhC,OAAO,EACL,cAAc,EACd,QAAQ,EAER,UAAU,EAEX,MAAM,4BAA4B,CAAC;AAQpC,MAAM,WAAW,sBAAsB;IACrC,KAAK,EAAE,cAAc,EAAE,CAAC;IACxB,WAAW,EAAE,MAAM,CAAC;IACpB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,cAAc,CAAC,EAAE,WAAW,GAAG,QAAQ,GAAG,WAAW,GAAG,aAAa,CAAC;IACtE,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,SAAS,CAAC,EAAE,UAAU,CAAC,WAAW,CAAC,CAAC;CACrC;AAED,MAAM,WAAW,uBAAuB;IACtC,MAAM,EAAE,QAAQ,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,qBAAa,sBACX,YAAW,QAAQ,CAAC,sBAAsB,EAAE,uBAAuB,CAAC;IAEpE,IAAI,SAA4B;IAChC,WAAW,SACqE;IAChF,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,EAAE,CAAC;IAEpB,OAAO,CAAC,aAAa,CAAsB;IAC3C,OAAO,CAAC,cAAc,CAAC,CAAiB;;IAoBlC,GAAG,CAAC,KAAK,EAAE,sBAAsB,GAAG,OAAO,CAAC,uBAAuB,CAAC;IAmI1E;;OAEG;YACW,eAAe;IAmE7B;;OAEG;YACW,yBAAyB;IAsDvC;;OAEG;YACW,iBAAiB;IA6C/B;;OAEG;IACH,OAAO,CAAC,yBAAyB;IAgBjC;;OAEG;IACH,OAAO,CAAC,eAAe;IAsCvB;;OAEG;YACW,aAAa;IA6B3B;;OAEG;IACH,OAAO,CAAC,eAAe;IAMvB;;OAEG;IACH,OAAO,CAAC,cAAc;IAUtB;;OAEG;YACW,gBAAgB;IAyBxB,QAAQ,CAAC,KAAK,EAAE,sBAAsB,GAAG,OAAO,CAAC,OAAO,CAAC;IASzD,SAAS,IAAI,OAAO,CAAC;QACzB,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,EAAE,MAAM,GAAG,SAAS,GAAG,WAAW,GAAG,OAAO,CAAC;QACnD,OAAO,CAAC,EAAE,IAAI,CAAC;QACf,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,UAAU,EAAE,MAAM,CAAC;QACnB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;CAQH"}
@@ -0,0 +1,414 @@
1
+ "use strict";
2
+ /**
3
+ * Voice-Over Script Extractor Agent
4
+ *
5
+ * Extracts text from screenshots and generates professional narration scripts
6
+ * with timestamps. Supports multiple narration styles and optional TTS integration.
7
+ *
8
+ * Communicates via Living Brain (context.json).
9
+ */
10
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
11
+ if (k2 === undefined) k2 = k;
12
+ var desc = Object.getOwnPropertyDescriptor(m, k);
13
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
14
+ desc = { enumerable: true, get: function() { return m[k]; } };
15
+ }
16
+ Object.defineProperty(o, k2, desc);
17
+ }) : (function(o, m, k, k2) {
18
+ if (k2 === undefined) k2 = k;
19
+ o[k2] = m[k];
20
+ }));
21
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
22
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
23
+ }) : function(o, v) {
24
+ o["default"] = v;
25
+ });
26
+ var __importStar = (this && this.__importStar) || (function () {
27
+ var ownKeys = function(o) {
28
+ ownKeys = Object.getOwnPropertyNames || function (o) {
29
+ var ar = [];
30
+ for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
31
+ return ar;
32
+ };
33
+ return ownKeys(o);
34
+ };
35
+ return function (mod) {
36
+ if (mod && mod.__esModule) return mod;
37
+ var result = {};
38
+ if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
39
+ __setModuleDefault(result, mod);
40
+ return result;
41
+ };
42
+ })();
43
+ var __importDefault = (this && this.__importDefault) || function (mod) {
44
+ return (mod && mod.__esModule) ? mod : { "default": mod };
45
+ };
46
+ Object.defineProperty(exports, "__esModule", { value: true });
47
+ exports.VOScriptExtractorAgent = void 0;
48
+ const gemini_vision_1 = require("../../services/gemini-vision");
49
+ const ContextService_1 = require("../../services/ContextService");
50
+ const AICore_1 = require("../../core/ai/AICore");
51
+ const uuid_1 = require("uuid");
52
+ const fs = __importStar(require("fs-extra"));
53
+ const path = __importStar(require("path"));
54
+ const chalk_1 = __importDefault(require("chalk"));
55
+ class VOScriptExtractorAgent {
56
+ constructor() {
57
+ this.name = "VOScriptExtractorAgent";
58
+ this.description = "Extracts text from screenshots and generates professional narration scripts";
59
+ this.personality = `You are a professional voice-over artist and scriptwriter.
60
+ You write engaging, clear narration that guides viewers through demos smoothly. Your scripts
61
+ are natural, informative, and adapt to different audiences - from technical developers to
62
+ business stakeholders.`;
63
+ this.llmProvider = "gemini-vision";
64
+ this.expertise = [
65
+ "Script writing",
66
+ "Text extraction",
67
+ "Narration generation",
68
+ "Timing coordination",
69
+ "TTS integration",
70
+ ];
71
+ this.visionService = new gemini_vision_1.GeminiVisionService();
72
+ }
73
+ async run(input) {
74
+ const { steps, missionName, missionDescription, narrationStyle = "technical", projectPath, outputPath, generateAudio, ttsConfig, } = input;
75
+ // Initialize context service
76
+ this.contextService = new ContextService_1.ContextService(projectPath);
77
+ await this.contextService.initialize();
78
+ console.log(chalk_1.default.blue(`\n🎙️ ${this.name} generating narration...`));
79
+ console.log(chalk_1.default.gray(`Style: ${narrationStyle}`));
80
+ console.log(chalk_1.default.gray(`Steps: ${steps.length}`));
81
+ try {
82
+ // Generate VO segments for each step
83
+ const segments = [];
84
+ let totalDuration = 0;
85
+ for (let i = 0; i < steps.length; i++) {
86
+ const step = steps[i];
87
+ if (!step)
88
+ continue;
89
+ console.log(chalk_1.default.cyan(`\n 📝 Segment ${i + 1}/${steps.length}`));
90
+ const segment = await this.generateSegment(step, i, narrationStyle, totalDuration);
91
+ segments.push(segment);
92
+ totalDuration += segment.duration;
93
+ console.log(chalk_1.default.gray(` "${segment.narration}"`));
94
+ }
95
+ // Build full script
96
+ const fullScript = this.buildFullScript(segments, missionName, missionDescription, narrationStyle);
97
+ // Create VOScript object
98
+ const voScript = {
99
+ title: missionName,
100
+ totalDuration,
101
+ segments,
102
+ fullScript,
103
+ };
104
+ // Save script to file
105
+ await fs.ensureDir(path.dirname(outputPath));
106
+ await fs.writeFile(outputPath, JSON.stringify(voScript, null, 2), "utf-8");
107
+ // Also save as human-readable markdown
108
+ const mdPath = outputPath.replace(".json", ".md");
109
+ await fs.writeFile(mdPath, fullScript, "utf-8");
110
+ console.log(chalk_1.default.green(`\n ✓ Script saved to ${outputPath}`));
111
+ // Generate audio if requested
112
+ let audioPath;
113
+ if (generateAudio && ttsConfig) {
114
+ console.log(chalk_1.default.cyan(`\n 🔊 Generating audio...`));
115
+ audioPath = await this.generateAudio(voScript, ttsConfig, outputPath);
116
+ console.log(chalk_1.default.green(` ✓ Audio saved to ${audioPath}`));
117
+ }
118
+ // Write to Living Brain
119
+ await this.writeLivingBrain({
120
+ timestamp: new Date().toISOString(),
121
+ agentName: this.name,
122
+ updateType: "demo-generated",
123
+ data: {
124
+ scriptPath: outputPath,
125
+ audioPath,
126
+ segmentCount: segments.length,
127
+ totalDuration,
128
+ },
129
+ });
130
+ console.log(chalk_1.default.blue(`\n🎉 Narration script complete!`));
131
+ return {
132
+ script: voScript,
133
+ scriptPath: outputPath,
134
+ audioPath,
135
+ success: true,
136
+ };
137
+ }
138
+ catch (error) {
139
+ console.error(chalk_1.default.red(`\n❌ Script generation error: ${error.message}`));
140
+ // Write error to Living Brain
141
+ await this.writeLivingBrain({
142
+ timestamp: new Date().toISOString(),
143
+ agentName: this.name,
144
+ updateType: "error",
145
+ data: {
146
+ error: error.message,
147
+ stack: error.stack,
148
+ },
149
+ });
150
+ return {
151
+ script: {
152
+ title: missionName,
153
+ totalDuration: 0,
154
+ segments: [],
155
+ fullScript: "",
156
+ },
157
+ scriptPath: outputPath,
158
+ success: false,
159
+ error: error.message,
160
+ };
161
+ }
162
+ }
163
+ /**
164
+ * Generate a single narration segment
165
+ */
166
+ async generateSegment(step, index, narrationStyle, currentTime) {
167
+ try {
168
+ // Extract text from screenshot if available
169
+ let textOnScreen = [];
170
+ let uiState = "UI state unknown";
171
+ if (step.screenshot && (await fs.pathExists(step.screenshot))) {
172
+ const extracted = await this.extractTextFromScreenshot(step.screenshot);
173
+ textOnScreen = extracted.textElements;
174
+ uiState = extracted.summary;
175
+ }
176
+ else if (step.visualAnalysis) {
177
+ textOnScreen = step.visualAnalysis.textContent || [];
178
+ uiState = step.visualAnalysis.interactiveElements.join(", ");
179
+ }
180
+ // Generate narration based on style
181
+ const narration = await this.generateNarration(step, textOnScreen, uiState, narrationStyle);
182
+ // Estimate duration (roughly 150 words per minute)
183
+ const wordCount = narration.split(/\s+/).length;
184
+ const duration = (wordCount / 150) * 60; // seconds
185
+ const segment = {
186
+ id: (0, uuid_1.v4)(),
187
+ timestamp: currentTime,
188
+ duration: Math.max(duration, 3), // Minimum 3 seconds per segment
189
+ narration,
190
+ action: step.action,
191
+ screenshot: step.screenshot,
192
+ metadata: {
193
+ step: index + 1,
194
+ uiState,
195
+ textOnScreen,
196
+ },
197
+ };
198
+ return segment;
199
+ }
200
+ catch (error) {
201
+ // Fallback segment
202
+ return {
203
+ id: (0, uuid_1.v4)(),
204
+ timestamp: currentTime,
205
+ duration: 3,
206
+ narration: `Step ${index + 1}: ${step.action}`,
207
+ action: step.action,
208
+ screenshot: step.screenshot,
209
+ metadata: {
210
+ step: index + 1,
211
+ uiState: "unknown",
212
+ textOnScreen: [],
213
+ },
214
+ };
215
+ }
216
+ }
217
+ /**
218
+ * Extract text from screenshot using vision AI
219
+ */
220
+ async extractTextFromScreenshot(screenshotPath) {
221
+ try {
222
+ // Use Gemini Vision to extract text
223
+ const imageData = await fs.readFile(screenshotPath);
224
+ const base64Image = imageData.toString("base64");
225
+ const { GoogleGenerativeAI } = require("@google/generative-ai");
226
+ const apiKey = process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY;
227
+ if (!apiKey) {
228
+ throw new Error("Gemini API key not found");
229
+ }
230
+ const genAI = new GoogleGenerativeAI(apiKey);
231
+ const model = genAI.getGenerativeModel({ model: "gemini-2.0-flash" });
232
+ const prompt = `Extract all visible text from this screenshot.
233
+
234
+ Return JSON:
235
+ {
236
+ "textElements": ["text 1", "text 2", ...],
237
+ "summary": "Brief description of what's on screen"
238
+ }`;
239
+ const result = await model.generateContent([
240
+ prompt,
241
+ {
242
+ inlineData: {
243
+ data: base64Image,
244
+ mimeType: "image/png",
245
+ },
246
+ },
247
+ ]);
248
+ const response = await result.response;
249
+ const text = response.text();
250
+ // Extract JSON
251
+ const jsonMatch = text.match(/```json\n([\s\S]*?)\n```/);
252
+ const jsonText = jsonMatch ? jsonMatch[1] : text;
253
+ const parsed = JSON.parse(jsonText);
254
+ return parsed;
255
+ }
256
+ catch (error) {
257
+ console.warn("Text extraction failed:", error);
258
+ return {
259
+ textElements: [],
260
+ summary: "Could not extract text",
261
+ };
262
+ }
263
+ }
264
+ /**
265
+ * Generate narration for a step
266
+ */
267
+ async generateNarration(step, textOnScreen, uiState, style) {
268
+ const styleGuidance = {
269
+ technical: "Use precise, technical language. Focus on actions and outcomes.",
270
+ casual: "Use conversational, friendly language. Make it relatable and easy to follow.",
271
+ marketing: "Use persuasive, benefit-focused language. Highlight features and value.",
272
+ educational: "Use clear, instructive language. Explain why and how things work.",
273
+ };
274
+ const prompt = `You are writing voice-over narration for a demo video.
275
+
276
+ **Style**: ${style} - ${styleGuidance[style] || styleGuidance.technical}
277
+
278
+ **Current Step**: ${step.action}
279
+ **Intent**: ${step.intent}
280
+ ${step.visionDecision ? `**Reasoning**: ${step.visionDecision.reasoning}` : ""}
281
+
282
+ **UI State**: ${uiState}
283
+ **Text visible on screen**: ${textOnScreen.join(", ") || "None"}
284
+
285
+ Write 1-2 sentences of narration for this step. Be natural and engaging.
286
+ Do NOT include any formatting, just the plain narration text.`;
287
+ try {
288
+ const aiCore = AICore_1.AICore.getInstance();
289
+ const narration = await aiCore.generateText(prompt, {
290
+ temperature: 0.7,
291
+ });
292
+ return narration.trim();
293
+ }
294
+ catch (error) {
295
+ console.warn("Narration generation failed, using fallback:", error);
296
+ return this.generateFallbackNarration(step, style);
297
+ }
298
+ }
299
+ /**
300
+ * Generate fallback narration
301
+ */
302
+ generateFallbackNarration(step, style) {
303
+ const templates = {
304
+ technical: `Now we ${step.action.toLowerCase()}. ${step.intent}`,
305
+ casual: `Let's ${step.action.toLowerCase()}. ${step.intent}`,
306
+ marketing: `Watch as we ${step.action.toLowerCase()}, ${step.intent}`,
307
+ educational: `In this step, we'll ${step.action.toLowerCase()}. ${step.intent}`,
308
+ };
309
+ return (templates[style] || templates.technical);
310
+ }
311
+ /**
312
+ * Build full script markdown
313
+ */
314
+ buildFullScript(segments, title, description, style) {
315
+ const lines = [
316
+ `# ${title} - Voice-Over Script`,
317
+ "",
318
+ description || "",
319
+ "",
320
+ `**Style**: ${style || "technical"}`,
321
+ `**Duration**: ${this.formatDuration(segments.reduce((sum, s) => sum + s.duration, 0))}`,
322
+ `**Segments**: ${segments.length}`,
323
+ "",
324
+ "---",
325
+ "",
326
+ "## Narration",
327
+ "",
328
+ ];
329
+ segments.forEach((segment, index) => {
330
+ lines.push(`### Segment ${index + 1} [${this.formatTimestamp(segment.timestamp)}]`);
331
+ lines.push("");
332
+ lines.push(`**Action**: ${segment.action}`);
333
+ lines.push(`**Duration**: ${segment.duration.toFixed(1)}s`);
334
+ lines.push("");
335
+ lines.push(segment.narration);
336
+ lines.push("");
337
+ lines.push("---");
338
+ lines.push("");
339
+ });
340
+ return lines.join("\n");
341
+ }
342
+ /**
343
+ * Generate audio using TTS
344
+ */
345
+ async generateAudio(script, ttsConfig, outputPath) {
346
+ // Placeholder for TTS integration
347
+ // In production, would integrate with:
348
+ // - Google Cloud Text-to-Speech
349
+ // - ElevenLabs API
350
+ // - Local TTS engines
351
+ const audioPath = outputPath.replace(".json", ".mp3");
352
+ console.log(chalk_1.default.yellow(` ℹ️ Audio generation not yet implemented. Would use ${ttsConfig?.provider}`));
353
+ // For now, just create an empty placeholder file
354
+ await fs.writeFile(audioPath, "Audio generation placeholder", "utf-8");
355
+ return audioPath;
356
+ }
357
+ /**
358
+ * Format timestamp as MM:SS
359
+ */
360
+ formatTimestamp(seconds) {
361
+ const mins = Math.floor(seconds / 60);
362
+ const secs = Math.floor(seconds % 60);
363
+ return `${mins.toString().padStart(2, "0")}:${secs.toString().padStart(2, "0")}`;
364
+ }
365
+ /**
366
+ * Format duration as human-readable string
367
+ */
368
+ formatDuration(seconds) {
369
+ const mins = Math.floor(seconds / 60);
370
+ const secs = Math.floor(seconds % 60);
371
+ if (mins > 0) {
372
+ return `${mins}m ${secs}s`;
373
+ }
374
+ return `${secs}s`;
375
+ }
376
+ /**
377
+ * Write update to Living Brain
378
+ */
379
+ async writeLivingBrain(update) {
380
+ try {
381
+ // Read current context using AICore
382
+ const aiCore = AICore_1.AICore.getInstance();
383
+ const currentContext = (await aiCore.getLivingContext()) || {};
384
+ if (!currentContext.testExecutionHistory) {
385
+ currentContext.testExecutionHistory = [];
386
+ }
387
+ currentContext.testExecutionHistory.push(update);
388
+ if (currentContext.testExecutionHistory.length > 100) {
389
+ currentContext.testExecutionHistory = currentContext.testExecutionHistory.slice(-100);
390
+ }
391
+ // Save back to context.json using AICore
392
+ await aiCore.saveLivingContext(currentContext);
393
+ }
394
+ catch (error) {
395
+ console.warn("Failed to write to Living Brain:", error);
396
+ }
397
+ }
398
+ async validate(input) {
399
+ return !!(input.steps &&
400
+ input.missionName &&
401
+ input.projectPath &&
402
+ input.outputPath);
403
+ }
404
+ async getStatus() {
405
+ return {
406
+ name: this.name,
407
+ status: "idle",
408
+ errorCount: 0,
409
+ successCount: 0,
410
+ };
411
+ }
412
+ }
413
+ exports.VOScriptExtractorAgent = VOScriptExtractorAgent;
414
+ //# sourceMappingURL=VOScriptExtractorAgent.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"VOScriptExtractorAgent.js","sourceRoot":"","sources":["../../../src/agents/implementations/VOScriptExtractorAgent.ts"],"names":[],"mappings":";AAAA;;;;;;;GAOG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAKH,gEAAmE;AASnE,kEAA+D;AAC/D,iDAA8C;AAC9C,+BAAoC;AACpC,6CAA+B;AAC/B,2CAA6B;AAC7B,kDAA0B;AAqB1B,MAAa,sBAAsB;IAajC;QAVA,SAAI,GAAG,wBAAwB,CAAC;QAChC,gBAAW,GACT,6EAA6E,CAAC;QAS9E,IAAI,CAAC,WAAW,GAAG;;;uBAGA,CAAC;QAEpB,IAAI,CAAC,WAAW,GAAG,eAAe,CAAC;QACnC,IAAI,CAAC,SAAS,GAAG;YACf,gBAAgB;YAChB,iBAAiB;YACjB,sBAAsB;YACtB,qBAAqB;YACrB,iBAAiB;SAClB,CAAC;QAEF,IAAI,CAAC,aAAa,GAAG,IAAI,mCAAmB,EAAE,CAAC;IACjD,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,KAA6B;QACrC,MAAM,EACJ,KAAK,EACL,WAAW,EACX,kBAAkB,EAClB,cAAc,GAAG,WAAW,EAC5B,WAAW,EACX,UAAU,EACV,aAAa,EACb,SAAS,GACV,GAAG,KAAK,CAAC;QAEV,6BAA6B;QAC7B,IAAI,CAAC,cAAc,GAAG,IAAI,+BAAc,CAAC,WAAW,CAAC,CAAC;QACtD,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,EAAE,CAAC;QAEvC,OAAO,CAAC,GAAG,CAAC,eAAK,CAAC,IAAI,CAAC,UAAU,IAAI,CAAC,IAAI,0BAA0B,CAAC,CAAC,CAAC;QACvE,OAAO,CAAC,GAAG,CAAC,eAAK,CAAC,IAAI,CAAC,UAAU,cAAc,EAAE,CAAC,CAAC,CAAC;QACpD,OAAO,CAAC,GAAG,CAAC,eAAK,CAAC,IAAI,CAAC,UAAU,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAElD,IAAI,CAAC;YACH,qCAAqC;YACrC,MAAM,QAAQ,GAAgB,EAAE,CAAC;YACjC,IAAI,aAAa,GAAG,CAAC,CAAC;YAEtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBACtB,IAAI,CAAC,IAAI;oBAAE,SAAS;gBAEpB,OAAO,CAAC,GAAG,CAAC,eAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;gBAEnE,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,eAAe,CACxC,IAAI,EACJ,CAAC,EACD,cAAc,EACd,aAAa,CACd,CAAC;gBAEF,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACvB,aAAa,IAAI,OAAO,CAAC,QAAQ,CAAC;gBAElC,OAAO,CAAC,GAAG,CAAC,eAAK,CAAC,IAAI,CAAC,SAAS,OAAO,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC;YACzD,CAAC;YAED,oBAAoB;YACpB,MAAM,UAAU,GAAG,IAAI,CAAC,eAAe,CACrC,QAAQ,EACR,WAAW,EACX,kBAAkB,EAClB,cAAc,CACf,CAAC;YAEF,yBAAyB;YACzB,MAAM,QAAQ,GAAa;gBACzB,KAAK,EAAE,WAAW;gBAClB,aAAa;gBACb,QAAQ;gBACR,UAAU;aACX,CAAC;YAEF,sBAAsB;YACtB,MAAM,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;YAC7C,MAAM,EAAE,CAAC,SAAS,CAChB,UAAU,EACV,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,EACjC,OAAO,CACR,CAAC;YAEF,uCAAuC;YACvC,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;YAClD,MAAM,EAAE,CAAC,SAAS,CAAC,MAAM,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;YAEhD,OAAO,CAAC,GAAG,CAAC,eAAK,CAAC,KAAK,CAAC,yBAAyB,UAAU,EAAE,CAAC,CAAC,CAAC;YAEhE,8BAA8B;YAC9B,IAAI,SAA6B,CAAC;YAClC,IAAI,aAAa,IAAI,SAAS,EAAE,CAAC;gBAC/B,OAAO,CAAC,GAAG,CAAC,eAAK,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC,CAAC;gBACtD,SAAS,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC;gBACtE,OAAO,CAAC,GAAG,CAAC,eAAK,CAAC,KAAK,CAAC,sBAAsB,SAAS,EAAE,CAAC,CAAC,CAAC;YAC9D,CAAC;YAED,wBAAwB;YACxB,MAAM,IAAI,CAAC,gBAAgB,CAAC;gBAC1B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,SAAS,EAAE,IAAI,CAAC,IAAI;gBACpB,UAAU,EAAE,gBAAgB;gBAC5B,IAAI,EAAE;oBACJ,UAAU,EAAE,UAAU;oBACtB,SAAS;oBACT,YAAY,EAAE,QAAQ,CAAC,MAAM;oBAC7B,aAAa;iBACd;aACF,CAAC,CAAC;YAEH,OAAO,CAAC,GAAG,CAAC,eAAK,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC,CAAC;YAE3D,OAAO;gBACL,MAAM,EAAE,QAAQ;gBAChB,UAAU,EAAE,UAAU;gBACtB,SAAS;gBACT,OAAO,EAAE,IAAI;aACd,CAAC;QACJ,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,OAAO,CAAC,KAAK,CAAC,eAAK,CAAC,GAAG,CAAC,gCAAgC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YAE1E,8BAA8B;YAC9B,MAAM,IAAI,CAAC,gBAAgB,CAAC;gBAC1B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,SAAS,EAAE,IAAI,CAAC,IAAI;gBACpB,UAAU,EAAE,OAAO;gBACnB,IAAI,EAAE;oBACJ,KAAK,EAAE,KAAK,CAAC,OAAO;oBACpB,KAAK,EAAE,KAAK,CAAC,KAAK;iBACnB;aACF,CAAC,CAAC;YAEH,OAAO;gBACL,MAAM,EAAE;oBACN,KAAK,EAAE,WAAW;oBAClB,aAAa,EAAE,CAAC;oBAChB,QAAQ,EAAE,EAAE;oBACZ,UAAU,EAAE,EAAE;iBACf;gBACD,UAAU,EAAE,UAAU;gBACtB,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,KAAK,CAAC,OAAO;aACrB,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,eAAe,CAC3B,IAAoB,EACpB,KAAa,EACb,cAAoE,EACpE,WAAmB;QAEnB,IAAI,CAAC;YACH,4CAA4C;YAC5C,IAAI,YAAY,GAAa,EAAE,CAAC;YAChC,IAAI,OAAO,GAAG,kBAAkB,CAAC;YAEjC,IAAI,IAAI,CAAC,UAAU,IAAI,CAAC,MAAM,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC;gBAC9D,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,yBAAyB,CACpD,IAAI,CAAC,UAAU,CAChB,CAAC;gBACF,YAAY,GAAG,SAAS,CAAC,YAAY,CAAC;gBACtC,OAAO,GAAG,SAAS,CAAC,OAAO,CAAC;YAC9B,CAAC;iBAAM,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;gBAC/B,YAAY,GAAG,IAAI,CAAC,cAAc,CAAC,WAAW,IAAI,EAAE,CAAC;gBACrD,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC/D,CAAC;YAED,oCAAoC;YACpC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAC5C,IAAI,EACJ,YAAY,EACZ,OAAO,EACP,cAAc,CACf,CAAC;YAEF,mDAAmD;YACnD,MAAM,SAAS,GAAG,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC;YAChD,MAAM,QAAQ,GAAG,CAAC,SAAS,GAAG,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,UAAU;YAEnD,MAAM,OAAO,GAAc;gBACzB,EAAE,EAAE,IAAA,SAAM,GAAE;gBACZ,SAAS,EAAE,WAAW;gBACtB,QAAQ,EAAE,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,gCAAgC;gBACjE,SAAS;gBACT,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,UAAU,EAAE,IAAI,CAAC,UAAU;gBAC3B,QAAQ,EAAE;oBACR,IAAI,EAAE,KAAK,GAAG,CAAC;oBACf,OAAO;oBACP,YAAY;iBACb;aACF,CAAC;YAEF,OAAO,OAAO,CAAC;QACjB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,mBAAmB;YACnB,OAAO;gBACL,EAAE,EAAE,IAAA,SAAM,GAAE;gBACZ,SAAS,EAAE,WAAW;gBACtB,QAAQ,EAAE,CAAC;gBACX,SAAS,EAAE,QAAQ,KAAK,GAAG,CAAC,KAAK,IAAI,CAAC,MAAM,EAAE;gBAC9C,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,UAAU,EAAE,IAAI,CAAC,UAAU;gBAC3B,QAAQ,EAAE;oBACR,IAAI,EAAE,KAAK,GAAG,CAAC;oBACf,OAAO,EAAE,SAAS;oBAClB,YAAY,EAAE,EAAE;iBACjB;aACF,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,yBAAyB,CACrC,cAAsB;QAEtB,IAAI,CAAC;YACH,oCAAoC;YACpC,MAAM,SAAS,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC;YACpD,MAAM,WAAW,GAAG,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YAEjD,MAAM,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC,uBAAuB,CAAC,CAAC;YAChE,MAAM,MAAM,GACV,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;YAC3D,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;YAC9C,CAAC;YAED,MAAM,KAAK,GAAG,IAAI,kBAAkB,CAAC,MAAM,CAAC,CAAC;YAC7C,MAAM,KAAK,GAAG,KAAK,CAAC,kBAAkB,CAAC,EAAE,KAAK,EAAE,kBAAkB,EAAE,CAAC,CAAC;YAEtE,MAAM,MAAM,GAAG;;;;;;EAMnB,CAAC;YAEG,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,eAAe,CAAC;gBACzC,MAAM;gBACN;oBACE,UAAU,EAAE;wBACV,IAAI,EAAE,WAAW;wBACjB,QAAQ,EAAE,WAAW;qBACtB;iBACF;aACF,CAAC,CAAC;YAEH,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC;YACvC,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;YAE7B,eAAe;YACf,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;YACzD,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YAEjD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;YACpC,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,yBAAyB,EAAE,KAAK,CAAC,CAAC;YAC/C,OAAO;gBACL,YAAY,EAAE,EAAE;gBAChB,OAAO,EAAE,wBAAwB;aAClC,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,iBAAiB,CAC7B,IAAoB,EACpB,YAAsB,EACtB,OAAe,EACf,KAAa;QAEb,MAAM,aAAa,GAAG;YACpB,SAAS,EACP,iEAAiE;YACnE,MAAM,EACJ,8EAA8E;YAChF,SAAS,EACP,yEAAyE;YAC3E,WAAW,EACT,mEAAmE;SACtE,CAAC;QAEF,MAAM,MAAM,GAAG;;aAEN,KAAK,MAAM,aAAa,CAAC,KAAmC,CAAC,IAAI,aAAa,CAAC,SAAS;;oBAEjF,IAAI,CAAC,MAAM;cACjB,IAAI,CAAC,MAAM;EACvB,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,kBAAkB,IAAI,CAAC,cAAc,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE;;gBAE9D,OAAO;8BACO,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,MAAM;;;8DAGD,CAAC;QAE3D,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,eAAM,CAAC,WAAW,EAAE,CAAC;YAEpC,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,YAAY,CAAC,MAAM,EAAE;gBAClD,WAAW,EAAE,GAAG;aACjB,CAAC,CAAC;YAEH,OAAO,SAAS,CAAC,IAAI,EAAE,CAAC;QAC1B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,8CAA8C,EAAE,KAAK,CAAC,CAAC;YACpE,OAAO,IAAI,CAAC,yBAAyB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QACrD,CAAC;IACH,CAAC;IAED;;OAEG;IACK,yBAAyB,CAC/B,IAAoB,EACpB,KAAa;QAEb,MAAM,SAAS,GAAG;YAChB,SAAS,EAAE,UAAU,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,KAAK,IAAI,CAAC,MAAM,EAAE;YAChE,MAAM,EAAE,SAAS,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,KAAK,IAAI,CAAC,MAAM,EAAE;YAC5D,SAAS,EAAE,eAAe,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,KAAK,IAAI,CAAC,MAAM,EAAE;YACrE,WAAW,EAAE,uBAAuB,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,KAAK,IAAI,CAAC,MAAM,EAAE;SAChF,CAAC;QAEF,OAAO,CACL,SAAS,CAAC,KAA+B,CAAC,IAAI,SAAS,CAAC,SAAS,CAClE,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,eAAe,CACrB,QAAqB,EACrB,KAAa,EACb,WAAoB,EACpB,KAAc;QAEd,MAAM,KAAK,GAAa;YACtB,KAAK,KAAK,sBAAsB;YAChC,EAAE;YACF,WAAW,IAAI,EAAE;YACjB,EAAE;YACF,cAAc,KAAK,IAAI,WAAW,EAAE;YACpC,iBAAiB,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE;YACxF,iBAAiB,QAAQ,CAAC,MAAM,EAAE;YAClC,EAAE;YACF,KAAK;YACL,EAAE;YACF,cAAc;YACd,EAAE;SACH,CAAC;QAEF,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE;YAClC,KAAK,CAAC,IAAI,CACR,eAAe,KAAK,GAAG,CAAC,KAAK,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,CACxE,CAAC;YACF,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,KAAK,CAAC,IAAI,CAAC,eAAe,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;YAC5C,KAAK,CAAC,IAAI,CAAC,iBAAiB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YAC5D,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;YAC9B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAClB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACjB,CAAC,CAAC,CAAC;QAEH,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,aAAa,CACzB,MAAgB,EAChB,SAAkC,EAClC,UAAkB;QAElB,kCAAkC;QAClC,uCAAuC;QACvC,gCAAgC;QAChC,mBAAmB;QACnB,sBAAsB;QAEtB,MAAM,SAAS,GAAG,UAAU,CAAC,OAAO,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAEtD,OAAO,CAAC,GAAG,CACT,eAAK,CAAC,MAAM,CACV,2DAA2D,SAAS,EAAE,QAAQ,EAAE,CACjF,CACF,CAAC;QAEF,iDAAiD;QACjD,MAAM,EAAE,CAAC,SAAS,CAChB,SAAS,EACT,8BAA8B,EAC9B,OAAO,CACR,CAAC;QAEF,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;OAEG;IACK,eAAe,CAAC,OAAe;QACrC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC;QACtC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC;QACtC,OAAO,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;IACnF,CAAC;IAED;;OAEG;IACK,cAAc,CAAC,OAAe;QACpC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC;QACtC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC;QAEtC,IAAI,IAAI,GAAG,CAAC,EAAE,CAAC;YACb,OAAO,GAAG,IAAI,KAAK,IAAI,GAAG,CAAC;QAC7B,CAAC;QACD,OAAO,GAAG,IAAI,GAAG,CAAC;IACpB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,gBAAgB,CAAC,MAAyB;QACtD,IAAI,CAAC;YACH,oCAAoC;YACpC,MAAM,MAAM,GAAG,eAAM,CAAC,WAAW,EAAE,CAAC;YACpC,MAAM,cAAc,GAAQ,CAAC,MAAM,MAAM,CAAC,gBAAgB,EAAE,CAAC,IAAI,EAAE,CAAC;YAEpE,IAAI,CAAC,cAAc,CAAC,oBAAoB,EAAE,CAAC;gBACzC,cAAc,CAAC,oBAAoB,GAAG,EAAE,CAAC;YAC3C,CAAC;YAED,cAAc,CAAC,oBAAoB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAEjD,IAAI,cAAc,CAAC,oBAAoB,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBACrD,cAAc,CAAC,oBAAoB,GAAG,cAAc,CAAC,oBAAoB,CAAC,KAAK,CAC7E,CAAC,GAAG,CACL,CAAC;YACJ,CAAC;YAED,yCAAyC;YACzC,MAAM,MAAM,CAAC,iBAAiB,CAAC,cAAc,CAAC,CAAC;QACjD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,kCAAkC,EAAE,KAAK,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,KAA6B;QAC1C,OAAO,CAAC,CAAC,CACP,KAAK,CAAC,KAAK;YACX,KAAK,CAAC,WAAW;YACjB,KAAK,CAAC,WAAW;YACjB,KAAK,CAAC,UAAU,CACjB,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,SAAS;QAQb,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,MAAM,EAAE,MAAM;YACd,UAAU,EAAE,CAAC;YACb,YAAY,EAAE,CAAC;SAChB,CAAC;IACJ,CAAC;CACF;AAvfD,wDAufC"}
@@ -0,0 +1,52 @@
1
+ /**
2
+ * Vision Navigator Agent
3
+ *
4
+ * Autonomous agent that navigates UIs using vision-based understanding.
5
+ * Observes -> Thinks -> Acts loop powered by visual AI.
6
+ *
7
+ * Communicates via Living Brain (context.json) following the blackboard pattern.
8
+ */
9
+ import { Page } from "playwright";
10
+ import { SubAgent } from "../interfaces/SubAgent";
11
+ import { VisionTestMission, VisionTestStep } from "../../types/vision-testing";
12
+ export interface VisionNavigatorInput {
13
+ page: Page;
14
+ mission: VisionTestMission;
15
+ projectPath: string;
16
+ maxSteps?: number;
17
+ }
18
+ export interface VisionNavigatorOutput {
19
+ steps: VisionTestStep[];
20
+ completed: boolean;
21
+ finalUrl: string;
22
+ reason?: string;
23
+ }
24
+ export declare class VisionNavigatorAgent implements SubAgent<VisionNavigatorInput, VisionNavigatorOutput> {
25
+ name: string;
26
+ description: string;
27
+ personality: string;
28
+ llmProvider: string;
29
+ expertise: string[];
30
+ private visionFinder?;
31
+ private contextService?;
32
+ constructor();
33
+ run(input: VisionNavigatorInput): Promise<VisionNavigatorOutput>;
34
+ /**
35
+ * Execute an action decided by the AI
36
+ */
37
+ private executeAction;
38
+ /**
39
+ * Write update to Living Brain (context.json)
40
+ */
41
+ private writeLivingBrain;
42
+ validate(input: VisionNavigatorInput): Promise<boolean>;
43
+ getStatus(): Promise<{
44
+ name: string;
45
+ status: "idle" | "running" | "completed" | "error";
46
+ lastRun?: Date;
47
+ executionTime?: number;
48
+ errorCount: number;
49
+ successCount: number;
50
+ }>;
51
+ }
52
+ //# sourceMappingURL=VisionNavigatorAgent.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"VisionNavigatorAgent.d.ts","sourceRoot":"","sources":["../../../src/agents/implementations/VisionNavigatorAgent.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EACL,QAAQ,EAGT,MAAM,wBAAwB,CAAC;AAEhC,OAAO,EACL,iBAAiB,EAEjB,cAAc,EAEf,MAAM,4BAA4B,CAAC;AAMpC,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,IAAI,CAAC;IACX,OAAO,EAAE,iBAAiB,CAAC;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,qBAAqB;IACpC,KAAK,EAAE,cAAc,EAAE,CAAC;IACxB,SAAS,EAAE,OAAO,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,qBAAa,oBACX,YAAW,QAAQ,CAAC,oBAAoB,EAAE,qBAAqB,CAAC;IAEhE,IAAI,SAA0B;IAC9B,WAAW,SAC6E;IACxF,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,EAAE,CAAC;IAEpB,OAAO,CAAC,YAAY,CAAC,CAAsB;IAC3C,OAAO,CAAC,cAAc,CAAC,CAAiB;;IAkBlC,GAAG,CAAC,KAAK,EAAE,oBAAoB,GAAG,OAAO,CAAC,qBAAqB,CAAC;IA6NtE;;OAEG;YACW,aAAa;IAyG3B;;OAEG;YACW,gBAAgB;IA4BxB,QAAQ,CAAC,KAAK,EAAE,oBAAoB,GAAG,OAAO,CAAC,OAAO,CAAC;IAIvD,SAAS,IAAI,OAAO,CAAC;QACzB,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,EAAE,MAAM,GAAG,SAAS,GAAG,WAAW,GAAG,OAAO,CAAC;QACnD,OAAO,CAAC,EAAE,IAAI,CAAC;QACf,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,UAAU,EAAE,MAAM,CAAC;QACnB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;CAQH"}