browser-debugging-daemon 1.0.0

package/subagent/BrowserSubagent.js

@@ -0,0 +1,689 @@
+import path from "path";
+import { OpenAIPlanner } from "./OpenAIPlanner.js";
+
+function compactHistory(history, limit = 8) {
+    return history.slice(-limit);
+}
+
+function compactElements(elements, limit = 50) {
+    return elements.slice(0, limit).map((element) => ({
+        id: element.id,
+        tag: element.tag,
+        text: element.text,
+        placeholder: element.placeholder,
+        ariaLabel: element.ariaLabel,
+        role: element.role,
+        type: element.type,
+        href: element.href,
+    }));
+}
+
+function compactDebugState(debugState, limit = 8) {
+    return {
+        recentConsole: (debugState?.recentConsole || []).slice(-limit),
+        recentNetwork: (debugState?.recentNetwork || []).slice(-limit),
+        recentErrors: (debugState?.recentErrors || []).slice(-limit),
+        capabilities: debugState?.capabilities || null,
+        counts: debugState?.counts || {
+            console: 0,
+            network: 0,
+            errors: 0,
+            observedElements: 0,
+        },
+    };
+}
+
+function compactOperatorMessages(messages, limit = 8) {
+    return (messages || []).slice(-limit).map((message) => ({
+        role: message.role,
+        content: message.content,
+        timestamp: message.timestamp || null,
+    }));
+}
+
+function defaultVerification() {
+    return {
+        goal_status: "incomplete",
+        confidence: "low",
+        summary: "Verification was skipped because no verifier was configured.",
+        evidence: [],
+        next_hint: "",
+    };
+}
+
+function formatAction(action) {
+    if (!action) return "no-op";
+    const detail = action.url || action.text || action.key || action.direction || action.id || "";
+    return detail ? `${action.type} (${detail})` : action.type;
+}
+
+function toIsoString(timestampMs) {
+    if (!Number.isFinite(timestampMs)) return null;
+    return new Date(timestampMs).toISOString();
+}
+
+function formatDurationMs(durationMs) {
+    if (!Number.isFinite(durationMs) || durationMs < 0) return "unknown";
+    if (durationMs < 1000) return `${Math.round(durationMs)}ms`;
+    return `${(durationMs / 1000).toFixed(2)}s`;
+}
+
+function formatVideoTimestamp(seconds) {
+    if (!Number.isFinite(seconds) || seconds < 0) return "00:00";
+    const total = Math.floor(seconds);
+    const mins = Math.floor(total / 60);
+    const secs = total % 60;
+    return `${String(mins).padStart(2, "0")}:${String(secs).padStart(2, "0")}`;
+}
+
+function toArtifactRelativePath(absolutePath, sessionDir) {
+    if (!absolutePath || !sessionDir) {
+        return absolutePath || null;
+    }
+
+    try {
+        const relative = path.relative(sessionDir, absolutePath);
+        if (!relative || relative.startsWith("..")) {
+            return absolutePath;
+        }
+        return relative.split(path.sep).join("/");
+    } catch (error) {
+        return absolutePath;
+    }
+}
+
+function buildStructuredTimeline(result) {
+    const sessionDir = result.artifacts?.sessionDir || null;
+    const primaryVideoPath = result.artifacts?.videoFiles?.[0] || null;
+    const primaryVideoRelative = toArtifactRelativePath(primaryVideoPath, sessionDir);
+    const history = Array.isArray(result.history) ? result.history : [];
+
+    const steps = history.map((entry, index) => {
+        const screenshotRelative = toArtifactRelativePath(entry.actionResult?.screenshotPath, sessionDir);
+        const videoOffsetSeconds = Number.isFinite(entry.videoOffsetSeconds)
+            ? Math.max(0, entry.videoOffsetSeconds)
+            : null;
+        const videoAnchor = videoOffsetSeconds !== null
+            ? Math.max(0, Math.floor(videoOffsetSeconds))
+            : null;
+        return {
+            index,
+            step: entry.step,
+            actionType: entry.action?.type || "unknown",
+            action: entry.action || null,
+            plannerSummary: entry.summary || "",
+            plannerThinking: entry.thinking || "",
+            verification: entry.verification || null,
+            timestamps: {
+                startedAt: entry.stepStartedAt || null,
+                finishedAt: entry.stepFinishedAt || null,
+                actionDurationMs: Number.isFinite(entry.actionDurationMs) ? entry.actionDurationMs : null,
+                elapsedMs: Number.isFinite(entry.elapsedMs) ? entry.elapsedMs : null,
+                videoOffsetSeconds,
+                videoTimestamp: videoOffsetSeconds !== null ? formatVideoTimestamp(videoOffsetSeconds) : null,
+            },
+            page: entry.page || null,
+            artifacts: {
+                screenshotPath: screenshotRelative,
+                videoPath: primaryVideoRelative,
+                videoJump: primaryVideoRelative && videoAnchor !== null ? `${primaryVideoRelative}#t=${videoAnchor}` : null,
+                eventsPath: toArtifactRelativePath(result.artifacts?.eventsPath, sessionDir),
+                tracePath: toArtifactRelativePath(result.artifacts?.tracePath, sessionDir),
+            },
+        };
+    });
+
+    return {
+        schemaVersion: 1,
+        generatedAt: new Date().toISOString(),
+        status: result.status,
+        summary: result.summary,
+        finalPage: result.page || null,
+        session: {
+            sessionDir,
+            primaryVideoPath: primaryVideoRelative,
+            eventsPath: toArtifactRelativePath(result.artifacts?.eventsPath, sessionDir),
+            tracePath: toArtifactRelativePath(result.artifacts?.tracePath, sessionDir),
+        },
+        steps,
+    };
+}
+
+function includesAny(text, patterns) {
+    const normalized = (text || "").toLowerCase();
+    return patterns.some((pattern) => normalized.includes(pattern));
+}
+
+function collectElementText(elements) {
+    return (elements || []).map((element) => [
+        element.text,
+        element.placeholder,
+        element.ariaLabel,
+        element.href,
+        element.role,
+        element.type,
+    ].filter(Boolean).join(" ")).join(" ").toLowerCase();
+}
+
+function detectGuidanceRequest({ taskInstruction, page, elements, debugState, raisedSignals }) {
+    const pageText = `${page?.title || ""} ${page?.url || ""} ${page?.textPreview || ""}`.toLowerCase();
+    const elementText = collectElementText(elements);
+    const combinedText = `${pageText} ${elementText}`;
+    const taskText = (taskInstruction || "").toLowerCase();
+
+    if (!raisedSignals.has("page_errors") && (debugState?.recentErrors || []).length > 0) {
+        const errorSummary = debugState.recentErrors
+            .slice(-2)
+            .map((entry) => entry.message || "Unknown page error")
+            .join(" | ");
+        return {
+            signal: "page_errors",
+            question: "The page is throwing runtime errors. Should I keep going, refresh, or stop for inspection?",
+            details: errorSummary,
+            suggestedReply: "Stop and inspect the page errors before continuing.",
+        };
+    }
+
+    const loginPatterns = ["sign in", "log in", "login", "sign up", "password", "continue with google", "continue with github", "continue with apple", "enter your email"];
+    const taskExpectsAuth = includesAny(taskText, ["login", "log in", "sign in", "authenticate", "auth", "credentials"]);
+    if (!raisedSignals.has("login_wall") && !taskExpectsAuth && includesAny(combinedText, loginPatterns)) {
+        return {
+            signal: "login_wall",
+            question: "I hit a login or authentication wall. Do you want to provide guidance or take over manually?",
+            details: "The page content looks like a sign-in flow rather than the intended task surface.",
+            suggestedReply: "Pause here and let me handle login manually.",
+        };
+    }
+
+    const permissionPatterns = ["allow notifications", "show notifications", "allow location", "use your location", "allow camera", "allow microphone", "permission", "notifications", "microphone", "camera", "location access"];
+    const taskExpectsPermission = includesAny(taskText, ["notification", "permission", "camera", "microphone", "location"]);
+    if (!raisedSignals.has("permission_prompt") && !taskExpectsPermission && includesAny(combinedText, permissionPatterns)) {
+        return {
+            signal: "permission_prompt",
+            question: "A browser permission prompt may need a human decision. Should I continue, deny it, or stop here?",
+            details: "The page appears to be asking for browser permissions such as notifications, camera, microphone, or location.",
+            suggestedReply: "Deny the permission and continue with the task.",
+        };
+    }
+
+    return null;
+}
+
+export class BrowserSubagent {
+    constructor(runtime, options = {}) {
+        this.runtime = runtime;
+        this.planner = options.planner || new OpenAIPlanner(options);
+        this.defaultMaxSteps = options.defaultMaxSteps || 12;
+    }
+
+    async delegateTask(taskInstruction, options = {}) {
+        const maxSteps = options.maxSteps || this.defaultMaxSteps;
+        const onProgress = typeof options.onProgress === "function" ? options.onProgress : null;
+        const onNeedsInput = typeof options.onNeedsInput === "function" ? options.onNeedsInput : null;
+        const shouldAbort = typeof options.shouldAbort === "function" ? options.shouldAbort : null;
+        const getAbortReason = typeof options.getAbortReason === "function" ? options.getAbortReason : null;
+        const pullHandoffRequest = typeof options.pullHandoffRequest === "function" ? options.pullHandoffRequest : null;
+        const startOptions = options.startOptions || {};
+        const history = [];
+        const operatorMessages = [];
+        const raisedSignals = new Set();
+        const taskStartedAtMs = Date.now();
+
+        await this.runtime.ensureStarted(startOptions);
+        this.runtime.recordEvent("subagent_task_started", { taskInstruction, maxSteps });
+
+        for (let step = 1; step <= maxSteps; step += 1) {
+            if (shouldAbort?.()) {
+                const result = await this.buildResult({
+                    status: "aborted",
+                    step: Math.max(0, step - 1),
+                    summary: getAbortReason?.() || "Browser task aborted.",
+                    history,
+                    page: await this.runtime.getPageMetadata().catch(() => null),
+                    verification: null,
+                    operatorMessages,
+                });
+                this.runtime.recordEvent("subagent_task_aborted", result);
+                return result;
+            }
+
+            const observation = await this.runtime.observe();
+            const page = await this.runtime.getPageMetadata();
+            const debugState = this.runtime.getDebugState(10);
+            const externalHandoffRequest = pullHandoffRequest?.() || null;
+            if (externalHandoffRequest) {
+                const externalOutcome = await this.handlePendingInput({
+                    step,
+                    pendingInput: {
+                        ...externalHandoffRequest,
+                        mode: externalHandoffRequest.mode || "manual_control",
+                    },
+                    page,
+                    history,
+                    operatorMessages,
+                    onProgress,
+                    onNeedsInput,
+                });
+
+                if (externalOutcome.result) {
+                    return externalOutcome.result;
+                }
+
+                continue;
+            }
+
+            const guidanceRequest = detectGuidanceRequest({
+                taskInstruction,
+                page,
+                elements: observation.elements,
+                debugState,
+                raisedSignals,
+            });
+
+            if (guidanceRequest) {
+                raisedSignals.add(guidanceRequest.signal);
+                const guidanceOutcome = await this.handlePendingInput({
+                    step,
+                    pendingInput: {
+                        step,
+                        question: guidanceRequest.question,
+                        details: guidanceRequest.details,
+                        suggestedReply: guidanceRequest.suggestedReply,
+                    },
+                    page,
+                    history,
+                    operatorMessages,
+                    onProgress,
+                    onNeedsInput,
+                });
+
+                if (guidanceOutcome.result) {
+                    return guidanceOutcome.result;
+                }
+                continue;
+            }
+
+            const decision = await this.planner.decide({
+                taskInstruction,
+                page,
+                elements: compactElements(observation.elements),
+                history: compactHistory(history),
+                operatorMessages: compactOperatorMessages(operatorMessages),
+                debugState,
+            });
+
+            this.runtime.recordEvent("subagent_decision", {
+                step,
+                decision,
+            });
+
+            const nextAction = decision.next_action || {};
+            const status = decision.status || "continue";
+
+            if (status === "completed" || nextAction.type === "finish") {
+                const result = await this.buildResult({
+                    status: "completed",
+                    step,
+                    summary: decision.summary || nextAction.result || "Task completed.",
+                    history,
+                    page,
+                    verification: null,
+                    operatorMessages,
+                });
+                this.runtime.recordEvent("subagent_task_completed", result);
+                return result;
+            }
+
+            if (status === "failed" || nextAction.type === "fail") {
+                const result = await this.buildResult({
+                    status: "failed",
+                    step,
+                    summary: decision.summary || nextAction.reason || "Task failed.",
+                    history,
+                    page,
+                    verification: null,
+                    operatorMessages,
+                });
+                this.runtime.recordEvent("subagent_task_failed", result);
+                return result;
+            }
+
+            if (status === "needs_input" || nextAction.type === "ask_main_agent") {
+                const pendingInput = {
+                    step,
+                    question: nextAction.question || decision.summary || "Need guidance before continuing.",
+                    details: nextAction.details || decision.thinking || "",
+                    suggestedReply: nextAction.suggested_reply || "",
+                };
+                const guidanceOutcome = await this.handlePendingInput({
+                    step,
+                    pendingInput,
+                    page,
+                    history,
+                    operatorMessages,
+                    onProgress,
+                    onNeedsInput,
+                });
+
+                if (guidanceOutcome.result) {
+                    return guidanceOutcome.result;
+                }
+                continue;
+            }
+
+            const stepStartedAtMs = Date.now();
+            const actionResult = await this.executeAction(nextAction);
+            const postActionObservation = await this.runtime.observe();
+            const postActionPage = await this.runtime.getPageMetadata();
+            const postActionDebugState = this.runtime.getDebugState(10);
+            const verification = await this.verifyProgress({
+                taskInstruction,
+                page: postActionPage,
+                elements: compactElements(postActionObservation.elements),
+                history,
+                lastAction: nextAction,
+                lastActionResult: actionResult,
+                debugState: postActionDebugState,
+            });
+
+            this.runtime.recordEvent("subagent_verification", {
+                step,
+                verification,
+            });
+            const stepFinishedAtMs = Date.now();
+
+            history.push({
+                step,
+                stepStartedAt: toIsoString(stepStartedAtMs),
+                stepFinishedAt: toIsoString(stepFinishedAtMs),
+                actionDurationMs: Math.max(0, stepFinishedAtMs - stepStartedAtMs),
+                elapsedMs: Math.max(0, stepFinishedAtMs - taskStartedAtMs),
+                videoOffsetSeconds: Math.max(0, (stepStartedAtMs - taskStartedAtMs) / 1000),
+                thinking: decision.thinking || "",
+                summary: decision.summary || "",
+                action: nextAction,
+                actionResult,
+                verification,
+                page: postActionPage,
+                debug: compactDebugState(postActionDebugState),
+            });
+
+            if (onProgress) {
+                await onProgress(this.buildProgressSnapshot({
+                    status: "running",
+                    step,
+                    summary: verification.summary || decision.summary || `Completed step ${step}.`,
+                    history,
+                    page: postActionPage,
+                    verification,
+                    operatorMessages,
+                    pendingInput: null,
+                }));
+            }
+
+            if (verification.goal_status === "completed") {
+                const result = await this.buildResult({
+                    status: "completed",
+                    step,
+                    summary: verification.summary || decision.summary || "Task completed.",
+                    history,
+                    page: postActionPage,
+                    verification,
+                    operatorMessages,
+                });
+                this.runtime.recordEvent("subagent_task_completed", result);
+                return result;
+            }
+
+            if (verification.goal_status === "blocked") {
+                const result = await this.buildResult({
+                    status: "failed",
+                    step,
+                    summary: verification.summary || "Task is blocked.",
+                    history,
+                    page: postActionPage,
+                    verification,
+                    operatorMessages,
+                });
+                this.runtime.recordEvent("subagent_task_failed", result);
+                return result;
+            }
+        }
+
+        const timeoutResult = await this.buildResult({
+            status: "failed",
+            step: maxSteps,
+            summary: `Stopped after ${maxSteps} steps without completing the task.`,
+            history,
+            page: await this.runtime.getPageMetadata().catch(() => null),
+            verification: null,
+            operatorMessages,
+        });
+        this.runtime.recordEvent("subagent_task_failed", timeoutResult);
+        return timeoutResult;
+    }
+
+    async verifyProgress(input) {
+        if (typeof this.planner.verify !== "function") {
+            return defaultVerification();
+        }
+
+        try {
+            return await this.planner.verify(input);
+        } catch (error) {
+            return {
+                goal_status: "incomplete",
+                confidence: "low",
+                summary: `Verification failed: ${error.message}`,
+                evidence: [],
+                next_hint: "Continue with caution.",
+            };
+        }
+    }
+
+    async handlePendingInput({ step, pendingInput, page, history, operatorMessages, onProgress, onNeedsInput }) {
+        const mode = pendingInput.mode || "guidance";
+        const waitingStatus = mode === "manual_control" ? "manual_control" : "waiting_for_instruction";
+        const requestMessage = {
+            role: mode === "manual_control" ? "system" : "subagent",
+            content: pendingInput.question,
+            timestamp: new Date().toISOString(),
+        };
+
+        operatorMessages.push(requestMessage);
+        this.runtime.recordEvent("subagent_input_requested", pendingInput);
+
+        if (onProgress) {
+            await onProgress(this.buildProgressSnapshot({
+                status: waitingStatus,
+                step,
+                summary: pendingInput.question,
+                history,
+                page,
+                verification: null,
+                operatorMessages,
+                pendingInput,
+            }));
+        }
+
+        if (!onNeedsInput) {
+            const result = await this.buildResult({
+                status: "failed",
+                step,
+                summary: pendingInput.question,
+                history,
+                page,
+                verification: null,
+                operatorMessages,
+                pendingInput,
+            });
+            this.runtime.recordEvent("subagent_task_failed", result);
+            return { result };
+        }
+
+        const reply = await onNeedsInput(pendingInput);
+        if (reply?.abort) {
+            const result = await this.buildResult({
+                status: "aborted",
+                step,
+                summary: reply.reason || "Browser task aborted.",
+                history,
+                page,
+                verification: null,
+                operatorMessages,
+            });
+            this.runtime.recordEvent("subagent_task_aborted", result);
+            return { result };
+        }
+
+        const operatorMessage = {
+            role: "main_agent",
+            content: typeof reply === "string" ? reply : reply?.instruction || "",
+            timestamp: new Date().toISOString(),
+        };
+
+        operatorMessages.push(operatorMessage);
+        this.runtime.recordEvent("subagent_input_received", operatorMessage);
+
+        if (onProgress) {
+            await onProgress(this.buildProgressSnapshot({
+                status: "running",
+                step,
+                summary: mode === "manual_control" ? "Manual control complete. Resuming browser task." : "Guidance received. Continuing browser task.",
+                history,
+                page,
+                verification: null,
+                operatorMessages,
+                pendingInput: null,
+            }));
+        }
+
+        return { result: null };
+    }
+
+    buildProgressSnapshot({ status, step, summary, history, page, verification, operatorMessages, pendingInput }) {
+        const debugState = this.runtime.getDebugState(20);
+        return {
+            status: status || "running",
+            step,
+            summary,
+            history: [...history],
+            artifacts: debugState.artifacts,
+            page,
+            verification,
+            operatorMessages: compactOperatorMessages(operatorMessages, 12),
+            pendingInput,
+            debug: compactDebugState(debugState, 12),
+        };
+    }
+
+    async buildResult({ status, step, summary, history, page, verification, operatorMessages, pendingInput = null }) {
+        const artifacts = this.runtime.getDebugState(10).artifacts;
+        const result = {
+            status,
+            step,
+            summary,
+            history,
+            artifacts,
+            page,
+            verification,
+            operatorMessages: compactOperatorMessages(operatorMessages, 12),
+            pendingInput,
+            debug: compactDebugState(this.runtime.getDebugState(20), 12),
+        };
+        result.capabilities = result.debug?.capabilities || null;
+
+        const reportPaths = this.writeReports(result);
+        return {
+            ...result,
+            reports: reportPaths,
+        };
+    }
+
+    writeReports(result) {
+        const reportJsonPath = this.runtime.writeArtifactJson("task-report.json", result);
+        const walkthroughPath = this.runtime.writeArtifactText("walkthrough.md", this.renderWalkthrough(result));
+        const timelineJsonPath = this.runtime.writeArtifactJson("task-timeline.json", buildStructuredTimeline(result));
+        return {
+            reportJsonPath,
+            walkthroughPath,
+            timelineJsonPath,
+        };
+    }
+
+    renderWalkthrough(result) {
+        const sessionDir = result.artifacts?.sessionDir || null;
+        const primaryVideoPath = result.artifacts?.videoFiles?.[0] || null;
+        const primaryVideoRelative = toArtifactRelativePath(primaryVideoPath, sessionDir);
+        const lines = [
+            "# Browser Task Report",
+            "",
+            `Status: ${result.status}`,
+            `Summary: ${result.summary}`,
+            `Final URL: ${result.page?.url || "unknown"}`,
+            `Final Title: ${result.page?.title || "unknown"}`,
+            "",
+            "## Steps",
+        ];
+
+        if (!result.history.length) {
+            lines.push("", "No browser actions were executed.");
+        }
+
+        for (const entry of result.history) {
+            lines.push("");
+            lines.push(`### Step ${entry.step}`);
+            lines.push(`Action: ${formatAction(entry.action)}`);
+            if (entry.summary) lines.push(`Plan Summary: ${entry.summary}`);
+            if (entry.verification?.summary) lines.push(`Verification: ${entry.verification.summary}`);
+            if (entry.stepStartedAt || entry.stepFinishedAt) {
+                lines.push(`Timing: ${entry.stepStartedAt || "unknown"} -> ${entry.stepFinishedAt || "unknown"} (${formatDurationMs(entry.actionDurationMs)})`);
+            }
+            if (entry.page?.url) lines.push(`Page URL: ${entry.page.url}`);
+            if (entry.page?.title) lines.push(`Page Title: ${entry.page.title}`);
+
+            const screenshotPath = toArtifactRelativePath(entry.actionResult?.screenshotPath, sessionDir);
+            if (screenshotPath) {
+                lines.push(`Screenshot: ${screenshotPath}`);
+            }
+
+            if (primaryVideoRelative && Number.isFinite(entry.videoOffsetSeconds)) {
+                const secondAnchor = Math.max(0, Math.floor(entry.videoOffsetSeconds));
+                lines.push(`Video Timestamp: ${formatVideoTimestamp(entry.videoOffsetSeconds)} (+${secondAnchor}s)`);
+                lines.push(`Video Jump: ${primaryVideoRelative}#t=${secondAnchor}`);
+            }
+        }
+
+        lines.push("");
+        lines.push("## Artifacts");
+        lines.push(`Session Dir: ${result.artifacts?.sessionDir || "unknown"}`);
+        lines.push(`Trace: ${result.artifacts?.tracePath || "pending"}`);
+        lines.push(`Events: ${result.artifacts?.eventsPath || "unknown"}`);
+        if (primaryVideoRelative) {
+            lines.push(`Primary Video: ${primaryVideoRelative}`);
+        } else {
+            lines.push("Primary Video: pending");
+        }
+
+        return `${lines.join("\n")}\n`;
+    }
+
+    async executeAction(action) {
+        switch (action.type) {
+            case "goto":
+                return this.runtime.goto(action.url);
+            case "click":
+                return this.runtime.click(Number(action.id));
+            case "type":
+                return this.runtime.type(Number(action.id), action.text || "");
+            case "hover":
+                return this.runtime.hover(Number(action.id));
+            case "keypress":
+                return this.runtime.keypress(action.key);
+            case "scroll":
+                return this.runtime.scroll(action.direction || "down");
+            default:
+                throw new Error(`Unsupported subagent action: ${action.type}`);
+        }
+    }
+}