mycontext-cli 4.2.8 → 4.2.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -4
- package/dist/README.md +13 -4
- package/dist/agents/implementations/ArchitectAgent.js.map +1 -1
- package/dist/agents/implementations/BuildStrategyAgent.js.map +1 -1
- package/dist/agents/implementations/CodeGenSubAgent.js.map +1 -1
- package/dist/agents/implementations/DemoRecorderAgent.d.ts +73 -0
- package/dist/agents/implementations/DemoRecorderAgent.d.ts.map +1 -0
- package/dist/agents/implementations/DemoRecorderAgent.js +485 -0
- package/dist/agents/implementations/DemoRecorderAgent.js.map +1 -0
- package/dist/agents/implementations/DesignPipelineAgent.d.ts +4 -0
- package/dist/agents/implementations/DesignPipelineAgent.d.ts.map +1 -1
- package/dist/agents/implementations/DesignPipelineAgent.js +82 -10
- package/dist/agents/implementations/DesignPipelineAgent.js.map +1 -1
- package/dist/agents/implementations/DocsSubAgent.js.map +1 -1
- package/dist/agents/implementations/FeaturePlannerAgent.d.ts +10 -0
- package/dist/agents/implementations/FeaturePlannerAgent.d.ts.map +1 -0
- package/dist/agents/implementations/FeaturePlannerAgent.js +35 -0
- package/dist/agents/implementations/FeaturePlannerAgent.js.map +1 -0
- package/dist/agents/implementations/QASubAgent.js.map +1 -1
- package/dist/agents/implementations/SecurityAgent.js.map +1 -1
- package/dist/agents/implementations/VOScriptExtractorAgent.d.ts +84 -0
- package/dist/agents/implementations/VOScriptExtractorAgent.d.ts.map +1 -0
- package/dist/agents/implementations/VOScriptExtractorAgent.js +414 -0
- package/dist/agents/implementations/VOScriptExtractorAgent.js.map +1 -0
- package/dist/agents/implementations/VisionNavigatorAgent.d.ts +52 -0
- package/dist/agents/implementations/VisionNavigatorAgent.d.ts.map +1 -0
- package/dist/agents/implementations/VisionNavigatorAgent.js +313 -0
- package/dist/agents/implementations/VisionNavigatorAgent.js.map +1 -0
- package/dist/agents/implementations/VisionTestCoordinator.d.ts +51 -0
- package/dist/agents/implementations/VisionTestCoordinator.d.ts.map +1 -0
- package/dist/agents/implementations/VisionTestCoordinator.js +348 -0
- package/dist/agents/implementations/VisionTestCoordinator.js.map +1 -0
- package/dist/agents/implementations/VisualValidatorAgent.d.ts +83 -0
- package/dist/agents/implementations/VisualValidatorAgent.d.ts.map +1 -0
- package/dist/agents/implementations/VisualValidatorAgent.js +472 -0
- package/dist/agents/implementations/VisualValidatorAgent.js.map +1 -0
- package/dist/agents/orchestrator/SubAgentOrchestrator.js.map +1 -1
- package/dist/cli.js +3 -0
- package/dist/cli.js.map +1 -1
- package/dist/clients/ClaudeSDKClient.d.ts +4 -0
- package/dist/clients/ClaudeSDKClient.d.ts.map +1 -1
- package/dist/clients/ClaudeSDKClient.js +8 -0
- package/dist/clients/ClaudeSDKClient.js.map +1 -1
- package/dist/clients/MyContextAIClient.d.ts +4 -0
- package/dist/clients/MyContextAIClient.d.ts.map +1 -1
- package/dist/clients/MyContextAIClient.js +69 -0
- package/dist/clients/MyContextAIClient.js.map +1 -1
- package/dist/clients/ProviderChain.d.ts +1 -0
- package/dist/clients/ProviderChain.d.ts.map +1 -1
- package/dist/clients/ProviderChain.js +3 -0
- package/dist/clients/ProviderChain.js.map +1 -1
- package/dist/clients/XAIClient.d.ts +4 -0
- package/dist/clients/XAIClient.d.ts.map +1 -1
- package/dist/clients/XAIClient.js +8 -0
- package/dist/clients/XAIClient.js.map +1 -1
- package/dist/commands/agent.d.ts.map +1 -1
- package/dist/commands/agent.js +10 -1
- package/dist/commands/agent.js.map +1 -1
- package/dist/commands/init.d.ts.map +1 -1
- package/dist/commands/init.js +110 -204
- package/dist/commands/init.js.map +1 -1
- package/dist/commands/plan-feature.d.ts +3 -0
- package/dist/commands/plan-feature.d.ts.map +1 -0
- package/dist/commands/plan-feature.js +57 -0
- package/dist/commands/plan-feature.js.map +1 -0
- package/dist/commands/sanitize.d.ts.map +1 -1
- package/dist/commands/sanitize.js +95 -7
- package/dist/commands/sanitize.js.map +1 -1
- package/dist/commands/vision-test.d.ts +67 -0
- package/dist/commands/vision-test.d.ts.map +1 -0
- package/dist/commands/vision-test.js +347 -0
- package/dist/commands/vision-test.js.map +1 -0
- package/dist/core/ai/AICore.d.ts +13 -0
- package/dist/core/ai/AICore.d.ts.map +1 -1
- package/dist/core/ai/AICore.js +65 -0
- package/dist/core/ai/AICore.js.map +1 -1
- package/dist/doctor/rules/node-rules.d.ts.map +1 -1
- package/dist/doctor/rules/node-rules.js +41 -0
- package/dist/doctor/rules/node-rules.js.map +1 -1
- package/dist/interfaces/AIClient.d.ts +1 -0
- package/dist/interfaces/AIClient.d.ts.map +1 -1
- package/dist/interfaces/AIClient.js.map +1 -1
- package/dist/mcp/vision-test-runner.d.ts +57 -0
- package/dist/mcp/vision-test-runner.d.ts.map +1 -0
- package/dist/mcp/vision-test-runner.js +337 -0
- package/dist/mcp/vision-test-runner.js.map +1 -0
- package/dist/mcp/vision-testing-server.d.ts +38 -0
- package/dist/mcp/vision-testing-server.d.ts.map +1 -0
- package/dist/mcp/vision-testing-server.js +637 -0
- package/dist/mcp/vision-testing-server.js.map +1 -0
- package/dist/package.json +1 -1
- package/dist/services/ProjectScanner.d.ts.map +1 -1
- package/dist/services/ProjectScanner.js +8 -5
- package/dist/services/ProjectScanner.js.map +1 -1
- package/dist/services/gemini-vision.d.ts +3 -11
- package/dist/services/gemini-vision.d.ts.map +1 -1
- package/dist/services/gemini-vision.js +13 -132
- package/dist/services/gemini-vision.js.map +1 -1
- package/dist/services/llmClarifier.d.ts +9 -0
- package/dist/services/llmClarifier.d.ts.map +1 -0
- package/dist/services/llmClarifier.js +22 -0
- package/dist/services/llmClarifier.js.map +1 -0
- package/dist/services/vision-element-finder.d.ts +64 -0
- package/dist/services/vision-element-finder.d.ts.map +1 -0
- package/dist/services/vision-element-finder.js +446 -0
- package/dist/services/vision-element-finder.js.map +1 -0
- package/dist/types/flow-testing.d.ts +1 -1
- package/dist/types/flow-testing.d.ts.map +1 -1
- package/dist/types/vision-testing.d.ts +382 -0
- package/dist/types/vision-testing.d.ts.map +1 -0
- package/dist/types/vision-testing.js +9 -0
- package/dist/types/vision-testing.js.map +1 -0
- package/dist/utils/claudeAgentClient.d.ts +4 -0
- package/dist/utils/claudeAgentClient.d.ts.map +1 -1
- package/dist/utils/claudeAgentClient.js +8 -0
- package/dist/utils/claudeAgentClient.js.map +1 -1
- package/dist/utils/geminiClient.d.ts +41 -6
- package/dist/utils/geminiClient.d.ts.map +1 -1
- package/dist/utils/geminiClient.js +124 -18
- package/dist/utils/geminiClient.js.map +1 -1
- package/dist/utils/githubModelsClient.d.ts +19 -6
- package/dist/utils/githubModelsClient.d.ts.map +1 -1
- package/dist/utils/githubModelsClient.js +94 -7
- package/dist/utils/githubModelsClient.js.map +1 -1
- package/dist/utils/openRouterClient.d.ts +12 -3
- package/dist/utils/openRouterClient.d.ts.map +1 -1
- package/dist/utils/openRouterClient.js +91 -2
- package/dist/utils/openRouterClient.js.map +1 -1
- package/dist/utils/unifiedDesignContextLoader.d.ts.map +1 -1
- package/dist/utils/unifiedDesignContextLoader.js +15 -0
- package/dist/utils/unifiedDesignContextLoader.js.map +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Voice-Over Script Extractor Agent
|
|
3
|
+
*
|
|
4
|
+
* Extracts text from screenshots and generates professional narration scripts
|
|
5
|
+
* with timestamps. Supports multiple narration styles and optional TTS integration.
|
|
6
|
+
*
|
|
7
|
+
* Communicates via Living Brain (context.json).
|
|
8
|
+
*/
|
|
9
|
+
import { SubAgent } from "../interfaces/SubAgent";
|
|
10
|
+
import { VisionTestStep, VOScript, DemoConfig } from "../../types/vision-testing";
|
|
11
|
+
export interface VOScriptExtractorInput {
|
|
12
|
+
steps: VisionTestStep[];
|
|
13
|
+
missionName: string;
|
|
14
|
+
missionDescription?: string;
|
|
15
|
+
narrationStyle?: "technical" | "casual" | "marketing" | "educational";
|
|
16
|
+
projectPath: string;
|
|
17
|
+
outputPath: string;
|
|
18
|
+
generateAudio?: boolean;
|
|
19
|
+
ttsConfig?: DemoConfig["ttsConfig"];
|
|
20
|
+
}
|
|
21
|
+
export interface VOScriptExtractorOutput {
|
|
22
|
+
script: VOScript;
|
|
23
|
+
scriptPath: string;
|
|
24
|
+
audioPath?: string;
|
|
25
|
+
success: boolean;
|
|
26
|
+
error?: string;
|
|
27
|
+
}
|
|
28
|
+
export declare class VOScriptExtractorAgent implements SubAgent<VOScriptExtractorInput, VOScriptExtractorOutput> {
|
|
29
|
+
name: string;
|
|
30
|
+
description: string;
|
|
31
|
+
personality: string;
|
|
32
|
+
llmProvider: string;
|
|
33
|
+
expertise: string[];
|
|
34
|
+
private visionService;
|
|
35
|
+
private contextService?;
|
|
36
|
+
constructor();
|
|
37
|
+
run(input: VOScriptExtractorInput): Promise<VOScriptExtractorOutput>;
|
|
38
|
+
/**
|
|
39
|
+
* Generate a single narration segment
|
|
40
|
+
*/
|
|
41
|
+
private generateSegment;
|
|
42
|
+
/**
|
|
43
|
+
* Extract text from screenshot using vision AI
|
|
44
|
+
*/
|
|
45
|
+
private extractTextFromScreenshot;
|
|
46
|
+
/**
|
|
47
|
+
* Generate narration for a step
|
|
48
|
+
*/
|
|
49
|
+
private generateNarration;
|
|
50
|
+
/**
|
|
51
|
+
* Generate fallback narration
|
|
52
|
+
*/
|
|
53
|
+
private generateFallbackNarration;
|
|
54
|
+
/**
|
|
55
|
+
* Build full script markdown
|
|
56
|
+
*/
|
|
57
|
+
private buildFullScript;
|
|
58
|
+
/**
|
|
59
|
+
* Generate audio using TTS
|
|
60
|
+
*/
|
|
61
|
+
private generateAudio;
|
|
62
|
+
/**
|
|
63
|
+
* Format timestamp as MM:SS
|
|
64
|
+
*/
|
|
65
|
+
private formatTimestamp;
|
|
66
|
+
/**
|
|
67
|
+
* Format duration as human-readable string
|
|
68
|
+
*/
|
|
69
|
+
private formatDuration;
|
|
70
|
+
/**
|
|
71
|
+
* Write update to Living Brain
|
|
72
|
+
*/
|
|
73
|
+
private writeLivingBrain;
|
|
74
|
+
validate(input: VOScriptExtractorInput): Promise<boolean>;
|
|
75
|
+
getStatus(): Promise<{
|
|
76
|
+
name: string;
|
|
77
|
+
status: "idle" | "running" | "completed" | "error";
|
|
78
|
+
lastRun?: Date;
|
|
79
|
+
executionTime?: number;
|
|
80
|
+
errorCount: number;
|
|
81
|
+
successCount: number;
|
|
82
|
+
}>;
|
|
83
|
+
}
|
|
84
|
+
//# sourceMappingURL=VOScriptExtractorAgent.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"VOScriptExtractorAgent.d.ts","sourceRoot":"","sources":["../../../src/agents/implementations/VOScriptExtractorAgent.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EACL,QAAQ,EACT,MAAM,wBAAwB,CAAC;AAGhC,OAAO,EACL,cAAc,EACd,QAAQ,EAER,UAAU,EAEX,MAAM,4BAA4B,CAAC;AAQpC,MAAM,WAAW,sBAAsB;IACrC,KAAK,EAAE,cAAc,EAAE,CAAC;IACxB,WAAW,EAAE,MAAM,CAAC;IACpB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,cAAc,CAAC,EAAE,WAAW,GAAG,QAAQ,GAAG,WAAW,GAAG,aAAa,CAAC;IACtE,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,SAAS,CAAC,EAAE,UAAU,CAAC,WAAW,CAAC,CAAC;CACrC;AAED,MAAM,WAAW,uBAAuB;IACtC,MAAM,EAAE,QAAQ,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,qBAAa,sBACX,YAAW,QAAQ,CAAC,sBAAsB,EAAE,uBAAuB,CAAC;IAEpE,IAAI,SAA4B;IAChC,WAAW,SACqE;IAChF,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,EAAE,CAAC;IAEpB,OAAO,CAAC,aAAa,CAAsB;IAC3C,OAAO,CAAC,cAAc,CAAC,CAAiB;;IAoBlC,GAAG,CAAC,KAAK,EAAE,sBAAsB,GAAG,OAAO,CAAC,uBAAuB,CAAC;IAmI1E;;OAEG;YACW,eAAe;IAmE7B;;OAEG;YACW,yBAAyB;IAsDvC;;OAEG;YACW,iBAAiB;IA6C/B;;OAEG;IACH,OAAO,CAAC,yBAAyB;IAgBjC;;OAEG;IACH,OAAO,CAAC,eAAe;IAsCvB;;OAEG;YACW,aAAa;IA6B3B;;OAEG;IACH,OAAO,CAAC,eAAe;IAMvB;;OAEG;IACH,OAAO,CAAC,cAAc;IAUtB;;OAEG;YACW,gBAAgB;IAyBxB,QAAQ,CAAC,KAAK,EAAE,sBAAsB,GAAG,OAAO,CAAC,OAAO,CAAC;IASzD,SAAS,IAAI,OAAO,CAAC;QACzB,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,EAAE,MAAM,GAAG,SAAS,GAAG,WAAW,GAAG,OAAO,CAAC;QACnD,OAAO,CAAC,EAAE,IAAI,CAAC;QACf,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,UAAU,EAAE,MAAM,CAAC;QACnB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;CAQH"}
|
|
@@ -0,0 +1,414 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Voice-Over Script Extractor Agent
|
|
4
|
+
*
|
|
5
|
+
* Extracts text from screenshots and generates professional narration scripts
|
|
6
|
+
* with timestamps. Supports multiple narration styles and optional TTS integration.
|
|
7
|
+
*
|
|
8
|
+
* Communicates via Living Brain (context.json).
|
|
9
|
+
*/
|
|
10
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
11
|
+
if (k2 === undefined) k2 = k;
|
|
12
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
13
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
14
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
15
|
+
}
|
|
16
|
+
Object.defineProperty(o, k2, desc);
|
|
17
|
+
}) : (function(o, m, k, k2) {
|
|
18
|
+
if (k2 === undefined) k2 = k;
|
|
19
|
+
o[k2] = m[k];
|
|
20
|
+
}));
|
|
21
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
22
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
23
|
+
}) : function(o, v) {
|
|
24
|
+
o["default"] = v;
|
|
25
|
+
});
|
|
26
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
27
|
+
var ownKeys = function(o) {
|
|
28
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
29
|
+
var ar = [];
|
|
30
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
31
|
+
return ar;
|
|
32
|
+
};
|
|
33
|
+
return ownKeys(o);
|
|
34
|
+
};
|
|
35
|
+
return function (mod) {
|
|
36
|
+
if (mod && mod.__esModule) return mod;
|
|
37
|
+
var result = {};
|
|
38
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
39
|
+
__setModuleDefault(result, mod);
|
|
40
|
+
return result;
|
|
41
|
+
};
|
|
42
|
+
})();
|
|
43
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
44
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
45
|
+
};
|
|
46
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
47
|
+
exports.VOScriptExtractorAgent = void 0;
|
|
48
|
+
const gemini_vision_1 = require("../../services/gemini-vision");
|
|
49
|
+
const ContextService_1 = require("../../services/ContextService");
|
|
50
|
+
const AICore_1 = require("../../core/ai/AICore");
|
|
51
|
+
const uuid_1 = require("uuid");
|
|
52
|
+
const fs = __importStar(require("fs-extra"));
|
|
53
|
+
const path = __importStar(require("path"));
|
|
54
|
+
const chalk_1 = __importDefault(require("chalk"));
|
|
55
|
+
class VOScriptExtractorAgent {
|
|
56
|
+
constructor() {
|
|
57
|
+
this.name = "VOScriptExtractorAgent";
|
|
58
|
+
this.description = "Extracts text from screenshots and generates professional narration scripts";
|
|
59
|
+
this.personality = `You are a professional voice-over artist and scriptwriter.
|
|
60
|
+
You write engaging, clear narration that guides viewers through demos smoothly. Your scripts
|
|
61
|
+
are natural, informative, and adapt to different audiences - from technical developers to
|
|
62
|
+
business stakeholders.`;
|
|
63
|
+
this.llmProvider = "gemini-vision";
|
|
64
|
+
this.expertise = [
|
|
65
|
+
"Script writing",
|
|
66
|
+
"Text extraction",
|
|
67
|
+
"Narration generation",
|
|
68
|
+
"Timing coordination",
|
|
69
|
+
"TTS integration",
|
|
70
|
+
];
|
|
71
|
+
this.visionService = new gemini_vision_1.GeminiVisionService();
|
|
72
|
+
}
|
|
73
|
+
async run(input) {
|
|
74
|
+
const { steps, missionName, missionDescription, narrationStyle = "technical", projectPath, outputPath, generateAudio, ttsConfig, } = input;
|
|
75
|
+
// Initialize context service
|
|
76
|
+
this.contextService = new ContextService_1.ContextService(projectPath);
|
|
77
|
+
await this.contextService.initialize();
|
|
78
|
+
console.log(chalk_1.default.blue(`\n🎙️ ${this.name} generating narration...`));
|
|
79
|
+
console.log(chalk_1.default.gray(`Style: ${narrationStyle}`));
|
|
80
|
+
console.log(chalk_1.default.gray(`Steps: ${steps.length}`));
|
|
81
|
+
try {
|
|
82
|
+
// Generate VO segments for each step
|
|
83
|
+
const segments = [];
|
|
84
|
+
let totalDuration = 0;
|
|
85
|
+
for (let i = 0; i < steps.length; i++) {
|
|
86
|
+
const step = steps[i];
|
|
87
|
+
if (!step)
|
|
88
|
+
continue;
|
|
89
|
+
console.log(chalk_1.default.cyan(`\n 📝 Segment ${i + 1}/${steps.length}`));
|
|
90
|
+
const segment = await this.generateSegment(step, i, narrationStyle, totalDuration);
|
|
91
|
+
segments.push(segment);
|
|
92
|
+
totalDuration += segment.duration;
|
|
93
|
+
console.log(chalk_1.default.gray(` "${segment.narration}"`));
|
|
94
|
+
}
|
|
95
|
+
// Build full script
|
|
96
|
+
const fullScript = this.buildFullScript(segments, missionName, missionDescription, narrationStyle);
|
|
97
|
+
// Create VOScript object
|
|
98
|
+
const voScript = {
|
|
99
|
+
title: missionName,
|
|
100
|
+
totalDuration,
|
|
101
|
+
segments,
|
|
102
|
+
fullScript,
|
|
103
|
+
};
|
|
104
|
+
// Save script to file
|
|
105
|
+
await fs.ensureDir(path.dirname(outputPath));
|
|
106
|
+
await fs.writeFile(outputPath, JSON.stringify(voScript, null, 2), "utf-8");
|
|
107
|
+
// Also save as human-readable markdown
|
|
108
|
+
const mdPath = outputPath.replace(".json", ".md");
|
|
109
|
+
await fs.writeFile(mdPath, fullScript, "utf-8");
|
|
110
|
+
console.log(chalk_1.default.green(`\n ✓ Script saved to ${outputPath}`));
|
|
111
|
+
// Generate audio if requested
|
|
112
|
+
let audioPath;
|
|
113
|
+
if (generateAudio && ttsConfig) {
|
|
114
|
+
console.log(chalk_1.default.cyan(`\n 🔊 Generating audio...`));
|
|
115
|
+
audioPath = await this.generateAudio(voScript, ttsConfig, outputPath);
|
|
116
|
+
console.log(chalk_1.default.green(` ✓ Audio saved to ${audioPath}`));
|
|
117
|
+
}
|
|
118
|
+
// Write to Living Brain
|
|
119
|
+
await this.writeLivingBrain({
|
|
120
|
+
timestamp: new Date().toISOString(),
|
|
121
|
+
agentName: this.name,
|
|
122
|
+
updateType: "demo-generated",
|
|
123
|
+
data: {
|
|
124
|
+
scriptPath: outputPath,
|
|
125
|
+
audioPath,
|
|
126
|
+
segmentCount: segments.length,
|
|
127
|
+
totalDuration,
|
|
128
|
+
},
|
|
129
|
+
});
|
|
130
|
+
console.log(chalk_1.default.blue(`\n🎉 Narration script complete!`));
|
|
131
|
+
return {
|
|
132
|
+
script: voScript,
|
|
133
|
+
scriptPath: outputPath,
|
|
134
|
+
audioPath,
|
|
135
|
+
success: true,
|
|
136
|
+
};
|
|
137
|
+
}
|
|
138
|
+
catch (error) {
|
|
139
|
+
console.error(chalk_1.default.red(`\n❌ Script generation error: ${error.message}`));
|
|
140
|
+
// Write error to Living Brain
|
|
141
|
+
await this.writeLivingBrain({
|
|
142
|
+
timestamp: new Date().toISOString(),
|
|
143
|
+
agentName: this.name,
|
|
144
|
+
updateType: "error",
|
|
145
|
+
data: {
|
|
146
|
+
error: error.message,
|
|
147
|
+
stack: error.stack,
|
|
148
|
+
},
|
|
149
|
+
});
|
|
150
|
+
return {
|
|
151
|
+
script: {
|
|
152
|
+
title: missionName,
|
|
153
|
+
totalDuration: 0,
|
|
154
|
+
segments: [],
|
|
155
|
+
fullScript: "",
|
|
156
|
+
},
|
|
157
|
+
scriptPath: outputPath,
|
|
158
|
+
success: false,
|
|
159
|
+
error: error.message,
|
|
160
|
+
};
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
/**
|
|
164
|
+
* Generate a single narration segment
|
|
165
|
+
*/
|
|
166
|
+
async generateSegment(step, index, narrationStyle, currentTime) {
|
|
167
|
+
try {
|
|
168
|
+
// Extract text from screenshot if available
|
|
169
|
+
let textOnScreen = [];
|
|
170
|
+
let uiState = "UI state unknown";
|
|
171
|
+
if (step.screenshot && (await fs.pathExists(step.screenshot))) {
|
|
172
|
+
const extracted = await this.extractTextFromScreenshot(step.screenshot);
|
|
173
|
+
textOnScreen = extracted.textElements;
|
|
174
|
+
uiState = extracted.summary;
|
|
175
|
+
}
|
|
176
|
+
else if (step.visualAnalysis) {
|
|
177
|
+
textOnScreen = step.visualAnalysis.textContent || [];
|
|
178
|
+
uiState = step.visualAnalysis.interactiveElements.join(", ");
|
|
179
|
+
}
|
|
180
|
+
// Generate narration based on style
|
|
181
|
+
const narration = await this.generateNarration(step, textOnScreen, uiState, narrationStyle);
|
|
182
|
+
// Estimate duration (roughly 150 words per minute)
|
|
183
|
+
const wordCount = narration.split(/\s+/).length;
|
|
184
|
+
const duration = (wordCount / 150) * 60; // seconds
|
|
185
|
+
const segment = {
|
|
186
|
+
id: (0, uuid_1.v4)(),
|
|
187
|
+
timestamp: currentTime,
|
|
188
|
+
duration: Math.max(duration, 3), // Minimum 3 seconds per segment
|
|
189
|
+
narration,
|
|
190
|
+
action: step.action,
|
|
191
|
+
screenshot: step.screenshot,
|
|
192
|
+
metadata: {
|
|
193
|
+
step: index + 1,
|
|
194
|
+
uiState,
|
|
195
|
+
textOnScreen,
|
|
196
|
+
},
|
|
197
|
+
};
|
|
198
|
+
return segment;
|
|
199
|
+
}
|
|
200
|
+
catch (error) {
|
|
201
|
+
// Fallback segment
|
|
202
|
+
return {
|
|
203
|
+
id: (0, uuid_1.v4)(),
|
|
204
|
+
timestamp: currentTime,
|
|
205
|
+
duration: 3,
|
|
206
|
+
narration: `Step ${index + 1}: ${step.action}`,
|
|
207
|
+
action: step.action,
|
|
208
|
+
screenshot: step.screenshot,
|
|
209
|
+
metadata: {
|
|
210
|
+
step: index + 1,
|
|
211
|
+
uiState: "unknown",
|
|
212
|
+
textOnScreen: [],
|
|
213
|
+
},
|
|
214
|
+
};
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
/**
|
|
218
|
+
* Extract text from screenshot using vision AI
|
|
219
|
+
*/
|
|
220
|
+
async extractTextFromScreenshot(screenshotPath) {
|
|
221
|
+
try {
|
|
222
|
+
// Use Gemini Vision to extract text
|
|
223
|
+
const imageData = await fs.readFile(screenshotPath);
|
|
224
|
+
const base64Image = imageData.toString("base64");
|
|
225
|
+
const { GoogleGenerativeAI } = require("@google/generative-ai");
|
|
226
|
+
const apiKey = process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY;
|
|
227
|
+
if (!apiKey) {
|
|
228
|
+
throw new Error("Gemini API key not found");
|
|
229
|
+
}
|
|
230
|
+
const genAI = new GoogleGenerativeAI(apiKey);
|
|
231
|
+
const model = genAI.getGenerativeModel({ model: "gemini-2.0-flash" });
|
|
232
|
+
const prompt = `Extract all visible text from this screenshot.
|
|
233
|
+
|
|
234
|
+
Return JSON:
|
|
235
|
+
{
|
|
236
|
+
"textElements": ["text 1", "text 2", ...],
|
|
237
|
+
"summary": "Brief description of what's on screen"
|
|
238
|
+
}`;
|
|
239
|
+
const result = await model.generateContent([
|
|
240
|
+
prompt,
|
|
241
|
+
{
|
|
242
|
+
inlineData: {
|
|
243
|
+
data: base64Image,
|
|
244
|
+
mimeType: "image/png",
|
|
245
|
+
},
|
|
246
|
+
},
|
|
247
|
+
]);
|
|
248
|
+
const response = await result.response;
|
|
249
|
+
const text = response.text();
|
|
250
|
+
// Extract JSON
|
|
251
|
+
const jsonMatch = text.match(/```json\n([\s\S]*?)\n```/);
|
|
252
|
+
const jsonText = jsonMatch ? jsonMatch[1] : text;
|
|
253
|
+
const parsed = JSON.parse(jsonText);
|
|
254
|
+
return parsed;
|
|
255
|
+
}
|
|
256
|
+
catch (error) {
|
|
257
|
+
console.warn("Text extraction failed:", error);
|
|
258
|
+
return {
|
|
259
|
+
textElements: [],
|
|
260
|
+
summary: "Could not extract text",
|
|
261
|
+
};
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
/**
|
|
265
|
+
* Generate narration for a step
|
|
266
|
+
*/
|
|
267
|
+
async generateNarration(step, textOnScreen, uiState, style) {
|
|
268
|
+
const styleGuidance = {
|
|
269
|
+
technical: "Use precise, technical language. Focus on actions and outcomes.",
|
|
270
|
+
casual: "Use conversational, friendly language. Make it relatable and easy to follow.",
|
|
271
|
+
marketing: "Use persuasive, benefit-focused language. Highlight features and value.",
|
|
272
|
+
educational: "Use clear, instructive language. Explain why and how things work.",
|
|
273
|
+
};
|
|
274
|
+
const prompt = `You are writing voice-over narration for a demo video.
|
|
275
|
+
|
|
276
|
+
**Style**: ${style} - ${styleGuidance[style] || styleGuidance.technical}
|
|
277
|
+
|
|
278
|
+
**Current Step**: ${step.action}
|
|
279
|
+
**Intent**: ${step.intent}
|
|
280
|
+
${step.visionDecision ? `**Reasoning**: ${step.visionDecision.reasoning}` : ""}
|
|
281
|
+
|
|
282
|
+
**UI State**: ${uiState}
|
|
283
|
+
**Text visible on screen**: ${textOnScreen.join(", ") || "None"}
|
|
284
|
+
|
|
285
|
+
Write 1-2 sentences of narration for this step. Be natural and engaging.
|
|
286
|
+
Do NOT include any formatting, just the plain narration text.`;
|
|
287
|
+
try {
|
|
288
|
+
const aiCore = AICore_1.AICore.getInstance();
|
|
289
|
+
const narration = await aiCore.generateText(prompt, {
|
|
290
|
+
temperature: 0.7,
|
|
291
|
+
});
|
|
292
|
+
return narration.trim();
|
|
293
|
+
}
|
|
294
|
+
catch (error) {
|
|
295
|
+
console.warn("Narration generation failed, using fallback:", error);
|
|
296
|
+
return this.generateFallbackNarration(step, style);
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
/**
|
|
300
|
+
* Generate fallback narration
|
|
301
|
+
*/
|
|
302
|
+
generateFallbackNarration(step, style) {
|
|
303
|
+
const templates = {
|
|
304
|
+
technical: `Now we ${step.action.toLowerCase()}. ${step.intent}`,
|
|
305
|
+
casual: `Let's ${step.action.toLowerCase()}. ${step.intent}`,
|
|
306
|
+
marketing: `Watch as we ${step.action.toLowerCase()}, ${step.intent}`,
|
|
307
|
+
educational: `In this step, we'll ${step.action.toLowerCase()}. ${step.intent}`,
|
|
308
|
+
};
|
|
309
|
+
return (templates[style] || templates.technical);
|
|
310
|
+
}
|
|
311
|
+
/**
|
|
312
|
+
* Build full script markdown
|
|
313
|
+
*/
|
|
314
|
+
buildFullScript(segments, title, description, style) {
|
|
315
|
+
const lines = [
|
|
316
|
+
`# ${title} - Voice-Over Script`,
|
|
317
|
+
"",
|
|
318
|
+
description || "",
|
|
319
|
+
"",
|
|
320
|
+
`**Style**: ${style || "technical"}`,
|
|
321
|
+
`**Duration**: ${this.formatDuration(segments.reduce((sum, s) => sum + s.duration, 0))}`,
|
|
322
|
+
`**Segments**: ${segments.length}`,
|
|
323
|
+
"",
|
|
324
|
+
"---",
|
|
325
|
+
"",
|
|
326
|
+
"## Narration",
|
|
327
|
+
"",
|
|
328
|
+
];
|
|
329
|
+
segments.forEach((segment, index) => {
|
|
330
|
+
lines.push(`### Segment ${index + 1} [${this.formatTimestamp(segment.timestamp)}]`);
|
|
331
|
+
lines.push("");
|
|
332
|
+
lines.push(`**Action**: ${segment.action}`);
|
|
333
|
+
lines.push(`**Duration**: ${segment.duration.toFixed(1)}s`);
|
|
334
|
+
lines.push("");
|
|
335
|
+
lines.push(segment.narration);
|
|
336
|
+
lines.push("");
|
|
337
|
+
lines.push("---");
|
|
338
|
+
lines.push("");
|
|
339
|
+
});
|
|
340
|
+
return lines.join("\n");
|
|
341
|
+
}
|
|
342
|
+
/**
|
|
343
|
+
* Generate audio using TTS
|
|
344
|
+
*/
|
|
345
|
+
async generateAudio(script, ttsConfig, outputPath) {
|
|
346
|
+
// Placeholder for TTS integration
|
|
347
|
+
// In production, would integrate with:
|
|
348
|
+
// - Google Cloud Text-to-Speech
|
|
349
|
+
// - ElevenLabs API
|
|
350
|
+
// - Local TTS engines
|
|
351
|
+
const audioPath = outputPath.replace(".json", ".mp3");
|
|
352
|
+
console.log(chalk_1.default.yellow(` ℹ️ Audio generation not yet implemented. Would use ${ttsConfig?.provider}`));
|
|
353
|
+
// For now, just create an empty placeholder file
|
|
354
|
+
await fs.writeFile(audioPath, "Audio generation placeholder", "utf-8");
|
|
355
|
+
return audioPath;
|
|
356
|
+
}
|
|
357
|
+
/**
|
|
358
|
+
* Format timestamp as MM:SS
|
|
359
|
+
*/
|
|
360
|
+
formatTimestamp(seconds) {
|
|
361
|
+
const mins = Math.floor(seconds / 60);
|
|
362
|
+
const secs = Math.floor(seconds % 60);
|
|
363
|
+
return `${mins.toString().padStart(2, "0")}:${secs.toString().padStart(2, "0")}`;
|
|
364
|
+
}
|
|
365
|
+
/**
|
|
366
|
+
* Format duration as human-readable string
|
|
367
|
+
*/
|
|
368
|
+
formatDuration(seconds) {
|
|
369
|
+
const mins = Math.floor(seconds / 60);
|
|
370
|
+
const secs = Math.floor(seconds % 60);
|
|
371
|
+
if (mins > 0) {
|
|
372
|
+
return `${mins}m ${secs}s`;
|
|
373
|
+
}
|
|
374
|
+
return `${secs}s`;
|
|
375
|
+
}
|
|
376
|
+
/**
|
|
377
|
+
* Write update to Living Brain
|
|
378
|
+
*/
|
|
379
|
+
async writeLivingBrain(update) {
|
|
380
|
+
try {
|
|
381
|
+
// Read current context using AICore
|
|
382
|
+
const aiCore = AICore_1.AICore.getInstance();
|
|
383
|
+
const currentContext = (await aiCore.getLivingContext()) || {};
|
|
384
|
+
if (!currentContext.testExecutionHistory) {
|
|
385
|
+
currentContext.testExecutionHistory = [];
|
|
386
|
+
}
|
|
387
|
+
currentContext.testExecutionHistory.push(update);
|
|
388
|
+
if (currentContext.testExecutionHistory.length > 100) {
|
|
389
|
+
currentContext.testExecutionHistory = currentContext.testExecutionHistory.slice(-100);
|
|
390
|
+
}
|
|
391
|
+
// Save back to context.json using AICore
|
|
392
|
+
await aiCore.saveLivingContext(currentContext);
|
|
393
|
+
}
|
|
394
|
+
catch (error) {
|
|
395
|
+
console.warn("Failed to write to Living Brain:", error);
|
|
396
|
+
}
|
|
397
|
+
}
|
|
398
|
+
async validate(input) {
|
|
399
|
+
return !!(input.steps &&
|
|
400
|
+
input.missionName &&
|
|
401
|
+
input.projectPath &&
|
|
402
|
+
input.outputPath);
|
|
403
|
+
}
|
|
404
|
+
async getStatus() {
|
|
405
|
+
return {
|
|
406
|
+
name: this.name,
|
|
407
|
+
status: "idle",
|
|
408
|
+
errorCount: 0,
|
|
409
|
+
successCount: 0,
|
|
410
|
+
};
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
exports.VOScriptExtractorAgent = VOScriptExtractorAgent;
|
|
414
|
+
//# sourceMappingURL=VOScriptExtractorAgent.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"VOScriptExtractorAgent.js","sourceRoot":"","sources":["../../../src/agents/implementations/VOScriptExtractorAgent.ts"],"names":[],"mappings":";AAAA;;;;;;;GAOG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAKH,gEAAmE;AASnE,kEAA+D;AAC/D,iDAA8C;AAC9C,+BAAoC;AACpC,6CAA+B;AAC/B,2CAA6B;AAC7B,kDAA0B;AAqB1B,MAAa,sBAAsB;IAajC;QAVA,SAAI,GAAG,wBAAwB,CAAC;QAChC,gBAAW,GACT,6EAA6E,CAAC;QAS9E,IAAI,CAAC,WAAW,GAAG;;;uBAGA,CAAC;QAEpB,IAAI,CAAC,WAAW,GAAG,eAAe,CAAC;QACnC,IAAI,CAAC,SAAS,GAAG;YACf,gBAAgB;YAChB,iBAAiB;YACjB,sBAAsB;YACtB,qBAAqB;YACrB,iBAAiB;SAClB,CAAC;QAEF,IAAI,CAAC,aAAa,GAAG,IAAI,mCAAmB,EAAE,CAAC;IACjD,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,KAA6B;QACrC,MAAM,EACJ,KAAK,EACL,WAAW,EACX,kBAAkB,EAClB,cAAc,GAAG,WAAW,EAC5B,WAAW,EACX,UAAU,EACV,aAAa,EACb,SAAS,GACV,GAAG,KAAK,CAAC;QAEV,6BAA6B;QAC7B,IAAI,CAAC,cAAc,GAAG,IAAI,+BAAc,CAAC,WAAW,CAAC,CAAC;QACtD,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,EAAE,CAAC;QAEvC,OAAO,CAAC,GAAG,CAAC,eAAK,CAAC,IAAI,CAAC,UAAU,IAAI,CAAC,IAAI,0BAA0B,CAAC,CAAC,CAAC;QACvE,OAAO,CAAC,GAAG,CAAC,eAAK,CAAC,IAAI,CAAC,UAAU,cAAc,EAAE,CAAC,CAAC,CAAC;QACpD,OAAO,CAAC,GAAG,CAAC,eAAK,CAAC,IAAI,CAAC,UAAU,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAElD,IAAI,CAAC;YACH,qCAAqC;YACrC,MAAM,QAAQ,GAAgB,EAAE,CAAC;YACjC,IAAI,aAAa,GAAG,CAAC,CAAC;YAEtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBACtB,IAAI,CAAC,IAAI;oBAAE,SAAS;gBAEpB,OAAO,CAAC,GAAG,CAAC,eAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;gBAEnE,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,eAAe,CACxC,IAAI,EACJ,CAAC,EACD,cAAc,EACd,aAAa,CACd,CAAC;gBAEF,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACvB,aAAa,IAAI,OAAO,CAAC,QAAQ,CAAC;gBAElC,OAAO,CAAC,GAAG,CAAC,eAAK,CAAC,IAAI,CAAC,SAAS,OAAO,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC;YACzD,CAAC;YAED,oBAAoB;YACpB,MAAM,UAAU,GAAG,IAAI,CAAC,eAAe,CACrC,QAAQ,EACR,WAAW,EACX,kBAAkB,EAClB,cAAc,CACf,CAAC;YAEF,yBAAyB;YACzB,MAAM,QAAQ,GAAa;gBACzB,KAAK,EAAE,WAAW;gBAClB,aAAa;gBACb,QAAQ;gBACR,UAAU;aACX,CAAC;YAEF,sBAAsB;YACtB,MAAM,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;YAC7C,MAAM,EAAE,CAAC,SAAS,CAChB,UAAU,EACV,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,EACjC,OAAO,CACR,CAAC;YAEF,uCAAuC;YACvC,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;YAClD,MAAM,EAAE,CAAC,SAAS,CAAC,MAAM,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;YAEhD,OAAO,CAAC,GAAG,CAAC,eAAK,CAAC,KAAK,CAAC,yBAAyB,UAAU,EAAE,CAAC,CAAC,CAAC;YAEhE,8BAA8B;YAC9B,IAAI,SAA6B,CAAC;YAClC,IAAI,aAAa,IAAI,SAAS,EAAE,CAAC;gBAC/B,OAAO,CAAC,GAAG,CAAC,eAAK,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC,CAAC;gBACtD,SAAS,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC;gBACtE,OAAO,CAAC,GAAG,CAAC,eAAK,CAAC,KAAK,CAAC,sBAAsB,SAAS,EAAE,CAAC,CAAC,CAAC;YAC9D,CAAC;YAED,wBAAwB;YACxB,MAAM,IAAI,CAAC,gBAAgB,CAAC;gBAC1B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,SAAS,EAAE,IAAI,CAAC,IAAI;gBACpB,UAAU,EAAE,gBAAgB;gBAC5B,IAAI,EAAE;oBACJ,UAAU,EAAE,UAAU;oBACtB,SAAS;oBACT,YAAY,EAAE,QAAQ,CAAC,MAAM;oBAC7B,aAAa;iBACd;aACF,CAAC,CAAC;YAEH,OAAO,CAAC,GAAG,CAAC,eAAK,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC,CAAC;YAE3D,OAAO;gBACL,MAAM,EAAE,QAAQ;gBAChB,UAAU,EAAE,UAAU;gBACtB,SAAS;gBACT,OAAO,EAAE,IAAI;aACd,CAAC;QACJ,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,OAAO,CAAC,KAAK,CAAC,eAAK,CAAC,GAAG,CAAC,gCAAgC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YAE1E,8BAA8B;YAC9B,MAAM,IAAI,CAAC,gBAAgB,CAAC;gBAC1B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,SAAS,EAAE,IAAI,CAAC,IAAI;gBACpB,UAAU,EAAE,OAAO;gBACnB,IAAI,EAAE;oBACJ,KAAK,EAAE,KAAK,CAAC,OAAO;oBACpB,KAAK,EAAE,KAAK,CAAC,KAAK;iBACnB;aACF,CAAC,CAAC;YAEH,OAAO;gBACL,MAAM,EAAE;oBACN,KAAK,EAAE,WAAW;oBAClB,aAAa,EAAE,CAAC;oBAChB,QAAQ,EAAE,EAAE;oBACZ,UAAU,EAAE,EAAE;iBACf;gBACD,UAAU,EAAE,UAAU;gBACtB,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,KAAK,CAAC,OAAO;aACrB,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,eAAe,CAC3B,IAAoB,EACpB,KAAa,EACb,cAAoE,EACpE,WAAmB;QAEnB,IAAI,CAAC;YACH,4CAA4C;YAC5C,IAAI,YAAY,GAAa,EAAE,CAAC;YAChC,IAAI,OAAO,GAAG,kBAAkB,CAAC;YAEjC,IAAI,IAAI,CAAC,UAAU,IAAI,CAAC,MAAM,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC;gBAC9D,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,yBAAyB,CACpD,IAAI,CAAC,UAAU,CAChB,CAAC;gBACF,YAAY,GAAG,SAAS,CAAC,YAAY,CAAC;gBACtC,OAAO,GAAG,SAAS,CAAC,OAAO,CAAC;YAC9B,CAAC;iBAAM,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;gBAC/B,YAAY,GAAG,IAAI,CAAC,cAAc,CAAC,WAAW,IAAI,EAAE,CAAC;gBACrD,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC/D,CAAC;YAED,oCAAoC;YACpC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAC5C,IAAI,EACJ,YAAY,EACZ,OAAO,EACP,cAAc,CACf,CAAC;YAEF,mDAAmD;YACnD,MAAM,SAAS,GAAG,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC;YAChD,MAAM,QAAQ,GAAG,CAAC,SAAS,GAAG,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,UAAU;YAEnD,MAAM,OAAO,GAAc;gBACzB,EAAE,EAAE,IAAA,SAAM,GAAE;gBACZ,SAAS,EAAE,WAAW;gBACtB,QAAQ,EAAE,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,gCAAgC;gBACjE,SAAS;gBACT,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,UAAU,EAAE,IAAI,CAAC,UAAU;gBAC3B,QAAQ,EAAE;oBACR,IAAI,EAAE,KAAK,GAAG,CAAC;oBACf,OAAO;oBACP,YAAY;iBACb;aACF,CAAC;YAEF,OAAO,OAAO,CAAC;QACjB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,mBAAmB;YACnB,OAAO;gBACL,EAAE,EAAE,IAAA,SAAM,GAAE;gBACZ,SAAS,EAAE,WAAW;gBACtB,QAAQ,EAAE,CAAC;gBACX,SAAS,EAAE,QAAQ,KAAK,GAAG,CAAC,KAAK,IAAI,CAAC,MAAM,EAAE;gBAC9C,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,UAAU,EAAE,IAAI,CAAC,UAAU;gBAC3B,QAAQ,EAAE;oBACR,IAAI,EAAE,KAAK,GAAG,CAAC;oBACf,OAAO,EAAE,SAAS;oBAClB,YAAY,EAAE,EAAE;iBACjB;aACF,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,yBAAyB,CACrC,cAAsB;QAEtB,IAAI,CAAC;YACH,oCAAoC;YACpC,MAAM,SAAS,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC;YACpD,MAAM,WAAW,GAAG,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YAEjD,MAAM,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC,uBAAuB,CAAC,CAAC;YAChE,MAAM,MAAM,GACV,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;YAC3D,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;YAC9C,CAAC;YAED,MAAM,KAAK,GAAG,IAAI,kBAAkB,CAAC,MAAM,CAAC,CAAC;YAC7C,MAAM,KAAK,GAAG,KAAK,CAAC,kBAAkB,CAAC,EAAE,KAAK,EAAE,kBAAkB,EAAE,CAAC,CAAC;YAEtE,MAAM,MAAM,GAAG;;;;;;EAMnB,CAAC;YAEG,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,eAAe,CAAC;gBACzC,MAAM;gBACN;oBACE,UAAU,EAAE;wBACV,IAAI,EAAE,WAAW;wBACjB,QAAQ,EAAE,WAAW;qBACtB;iBACF;aACF,CAAC,CAAC;YAEH,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC;YACvC,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;YAE7B,eAAe;YACf,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;YACzD,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YAEjD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;YACpC,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,yBAAyB,EAAE,KAAK,CAAC,CAAC;YAC/C,OAAO;gBACL,YAAY,EAAE,EAAE;gBAChB,OAAO,EAAE,wBAAwB;aAClC,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,iBAAiB,CAC7B,IAAoB,EACpB,YAAsB,EACtB,OAAe,EACf,KAAa;QAEb,MAAM,aAAa,GAAG;YACpB,SAAS,EACP,iEAAiE;YACnE,MAAM,EACJ,8EAA8E;YAChF,SAAS,EACP,yEAAyE;YAC3E,WAAW,EACT,mEAAmE;SACtE,CAAC;QAEF,MAAM,MAAM,GAAG;;aAEN,KAAK,MAAM,aAAa,CAAC,KAAmC,CAAC,IAAI,aAAa,CAAC,SAAS;;oBAEjF,IAAI,CAAC,MAAM;cACjB,IAAI,CAAC,MAAM;EACvB,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,kBAAkB,IAAI,CAAC,cAAc,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE;;gBAE9D,OAAO;8BACO,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,MAAM;;;8DAGD,CAAC;QAE3D,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,eAAM,CAAC,WAAW,EAAE,CAAC;YAEpC,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,YAAY,CAAC,MAAM,EAAE;gBAClD,WAAW,EAAE,GAAG;aACjB,CAAC,CAAC;YAEH,OAAO,SAAS,CAAC,IAAI,EAAE,CAAC;QAC1B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,8CAA8C,EAAE,KAAK,CAAC,CAAC;YACpE,OAAO,IAAI,CAAC,yBAAyB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QACrD,CAAC;IACH,CAAC;IAED;;OAEG;IACK,yBAAyB,CAC/B,IAAoB,EACpB,KAAa;QAEb,MAAM,SAAS,GAAG;YAChB,SAAS,EAAE,UAAU,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,KAAK,IAAI,CAAC,MAAM,EAAE;YAChE,MAAM,EAAE,SAAS,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,KAAK,IAAI,CAAC,MAAM,EAAE;YAC5D,SAAS,EAAE,eAAe,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,KAAK,IAAI,CAAC,MAAM,EAAE;YACrE,WAAW,EAAE,uBAAuB,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,KAAK,IAAI,CAAC,MAAM,EAAE;SAChF,CAAC;QAEF,OAAO,CACL,SAAS,CAAC,KAA+B,CAAC,IAAI,SAAS,CAAC,SAAS,CAClE,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,eAAe,CACrB,QAAqB,EACrB,KAAa,EACb,WAAoB,EACpB,KAAc;QAEd,MAAM,KAAK,GAAa;YACtB,KAAK,KAAK,sBAAsB;YAChC,EAAE;YACF,WAAW,IAAI,EAAE;YACjB,EAAE;YACF,cAAc,KAAK,IAAI,WAAW,EAAE;YACpC,iBAAiB,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE;YACxF,iBAAiB,QAAQ,CAAC,MAAM,EAAE;YAClC,EAAE;YACF,KAAK;YACL,EAAE;YACF,cAAc;YACd,EAAE;SACH,CAAC;QAEF,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE;YAClC,KAAK,CAAC,IAAI,CACR,eAAe,KAAK,GAAG,CAAC,KAAK,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,CACxE,CAAC;YACF,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,KAAK,CAAC,IAAI,CAAC,eAAe,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;YAC5C,KAAK,CAAC,IAAI,CAAC,iBAAiB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YAC5D,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;YAC9B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAClB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACjB,CAAC,CAAC,CAAC;QAEH,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,aAAa,CACzB,MAAgB,EAChB,SAAkC,EAClC,UAAkB;QAElB,kCAAkC;QAClC,uCAAuC;QACvC,gCAAgC;QAChC,mBAAmB;QACnB,sBAAsB;QAEtB,MAAM,SAAS,GAAG,UAAU,CAAC,OAAO,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAEtD,OAAO,CAAC,GAAG,CACT,eAAK,CAAC,MAAM,CACV,2DAA2D,SAAS,EAAE,QAAQ,EAAE,CACjF,CACF,CAAC;QAEF,iDAAiD;QACjD,MAAM,EAAE,CAAC,SAAS,CAChB,SAAS,EACT,8BAA8B,EAC9B,OAAO,CACR,CAAC;QAEF,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;OAEG;IACK,eAAe,CAAC,OAAe;QACrC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC;QACtC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC;QACtC,OAAO,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;IACnF,CAAC;IAED;;OAEG;IACK,cAAc,CAAC,OAAe;QACpC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC;QACtC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC;QAEtC,IAAI,IAAI,GAAG,CAAC,EAAE,CAAC;YACb,OAAO,GAAG,IAAI,KAAK,IAAI,GAAG,CAAC;QAC7B,CAAC;QACD,OAAO,GAAG,IAAI,GAAG,CAAC;IACpB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,gBAAgB,CAAC,MAAyB;QACtD,IAAI,CAAC;YACH,oCAAoC;YACpC,MAAM,MAAM,GAAG,eAAM,CAAC,WAAW,EAAE,CAAC;YACpC,MAAM,cAAc,GAAQ,CAAC,MAAM,MAAM,CAAC,gBAAgB,EAAE,CAAC,IAAI,EAAE,CAAC;YAEpE,IAAI,CAAC,cAAc,CAAC,oBAAoB,EAAE,CAAC;gBACzC,cAAc,CAAC,oBAAoB,GAAG,EAAE,CAAC;YAC3C,CAAC;YAED,cAAc,CAAC,oBAAoB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAEjD,IAAI,cAAc,CAAC,oBAAoB,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBACrD,cAAc,CAAC,oBAAoB,GAAG,cAAc,CAAC,oBAAoB,CAAC,KAAK,CAC7E,CAAC,GAAG,CACL,CAAC;YACJ,CAAC;YAED,yCAAyC;YACzC,MAAM,MAAM,CAAC,iBAAiB,CAAC,cAAc,CAAC,CAAC;QACjD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,kCAAkC,EAAE,KAAK,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,KAA6B;QAC1C,OAAO,CAAC,CAAC,CACP,KAAK,CAAC,KAAK;YACX,KAAK,CAAC,WAAW;YACjB,KAAK,CAAC,WAAW;YACjB,KAAK,CAAC,UAAU,CACjB,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,SAAS;QAQb,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,MAAM,EAAE,MAAM;YACd,UAAU,EAAE,CAAC;YACb,YAAY,EAAE,CAAC;SAChB,CAAC;IACJ,CAAC;CACF;AAvfD,wDAufC"}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Vision Navigator Agent
|
|
3
|
+
*
|
|
4
|
+
* Autonomous agent that navigates UIs using vision-based understanding.
|
|
5
|
+
* Observes -> Thinks -> Acts loop powered by visual AI.
|
|
6
|
+
*
|
|
7
|
+
* Communicates via Living Brain (context.json) following the blackboard pattern.
|
|
8
|
+
*/
|
|
9
|
+
import { Page } from "playwright";
|
|
10
|
+
import { SubAgent } from "../interfaces/SubAgent";
|
|
11
|
+
import { VisionTestMission, VisionTestStep } from "../../types/vision-testing";
|
|
12
|
+
export interface VisionNavigatorInput {
|
|
13
|
+
page: Page;
|
|
14
|
+
mission: VisionTestMission;
|
|
15
|
+
projectPath: string;
|
|
16
|
+
maxSteps?: number;
|
|
17
|
+
}
|
|
18
|
+
export interface VisionNavigatorOutput {
|
|
19
|
+
steps: VisionTestStep[];
|
|
20
|
+
completed: boolean;
|
|
21
|
+
finalUrl: string;
|
|
22
|
+
reason?: string;
|
|
23
|
+
}
|
|
24
|
+
export declare class VisionNavigatorAgent implements SubAgent<VisionNavigatorInput, VisionNavigatorOutput> {
|
|
25
|
+
name: string;
|
|
26
|
+
description: string;
|
|
27
|
+
personality: string;
|
|
28
|
+
llmProvider: string;
|
|
29
|
+
expertise: string[];
|
|
30
|
+
private visionFinder?;
|
|
31
|
+
private contextService?;
|
|
32
|
+
constructor();
|
|
33
|
+
run(input: VisionNavigatorInput): Promise<VisionNavigatorOutput>;
|
|
34
|
+
/**
|
|
35
|
+
* Execute an action decided by the AI
|
|
36
|
+
*/
|
|
37
|
+
private executeAction;
|
|
38
|
+
/**
|
|
39
|
+
* Write update to Living Brain (context.json)
|
|
40
|
+
*/
|
|
41
|
+
private writeLivingBrain;
|
|
42
|
+
validate(input: VisionNavigatorInput): Promise<boolean>;
|
|
43
|
+
getStatus(): Promise<{
|
|
44
|
+
name: string;
|
|
45
|
+
status: "idle" | "running" | "completed" | "error";
|
|
46
|
+
lastRun?: Date;
|
|
47
|
+
executionTime?: number;
|
|
48
|
+
errorCount: number;
|
|
49
|
+
successCount: number;
|
|
50
|
+
}>;
|
|
51
|
+
}
|
|
52
|
+
//# sourceMappingURL=VisionNavigatorAgent.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"VisionNavigatorAgent.d.ts","sourceRoot":"","sources":["../../../src/agents/implementations/VisionNavigatorAgent.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EACL,QAAQ,EAGT,MAAM,wBAAwB,CAAC;AAEhC,OAAO,EACL,iBAAiB,EAEjB,cAAc,EAEf,MAAM,4BAA4B,CAAC;AAMpC,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,IAAI,CAAC;IACX,OAAO,EAAE,iBAAiB,CAAC;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,qBAAqB;IACpC,KAAK,EAAE,cAAc,EAAE,CAAC;IACxB,SAAS,EAAE,OAAO,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,qBAAa,oBACX,YAAW,QAAQ,CAAC,oBAAoB,EAAE,qBAAqB,CAAC;IAEhE,IAAI,SAA0B;IAC9B,WAAW,SAC6E;IACxF,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,EAAE,CAAC;IAEpB,OAAO,CAAC,YAAY,CAAC,CAAsB;IAC3C,OAAO,CAAC,cAAc,CAAC,CAAiB;;IAkBlC,GAAG,CAAC,KAAK,EAAE,oBAAoB,GAAG,OAAO,CAAC,qBAAqB,CAAC;IA6NtE;;OAEG;YACW,aAAa;IAyG3B;;OAEG;YACW,gBAAgB;IA4BxB,QAAQ,CAAC,KAAK,EAAE,oBAAoB,GAAG,OAAO,CAAC,OAAO,CAAC;IAIvD,SAAS,IAAI,OAAO,CAAC;QACzB,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,EAAE,MAAM,GAAG,SAAS,GAAG,WAAW,GAAG,OAAO,CAAC;QACnD,OAAO,CAAC,EAAE,IAAI,CAAC;QACf,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,UAAU,EAAE,MAAM,CAAC;QACnB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;CAQH"}
|