@ddwang/magnitude-core 0.3.1-ddwang.1

package/dist/connectors/browserConnector.js

@@ -0,0 +1,127 @@
+//import { Observation, BamlRenderable } from "@/memory";
+import { WebHarness } from "@/web/harness";
+import { webActions } from '@/actions/webActions';
+import { BrowserProvider } from "@/web/browserProvider";
+import logger from "@/logger";
+import { Observation } from "@/memory/observation";
+// export type BrowserOptions = ({ instance: Browser } | { launchOptions?: LaunchOptions }) & {
+//     contextOptions?: BrowserContextOptions;
+// };
+// const foo: BrowserOptions = {
+//     launchOptions: {},
+//     instance: {},
+// }
+// Changed back to 3 - too many situations where the amnesia of having only 1 is very problematic and makes agent act stupidly
+// With caching, using 3 is relatively ok tradeoff
+// Maybe try 2 for now, or could do 3 when prompt caching available else 2
+const DEFAULT_MIN_RETAINED_SCREENSHOTS = 2;
+export class BrowserConnector {
+    id = "web";
+    harness;
+    options;
+    browser;
+    context;
+    logger;
+    constructor(options = {}) {
+        // console.log("options", options)
+        // console.log("options.screenshotMemoryLimit", options.screenshotMemoryLimit)
+        this.options = options;
+        this.logger = logger.child({
+            name: `connectors.${this.id}`
+        });
+    }
+    async onStart() {
+        this.logger.info("Starting...");
+        this.logger.info("Creating new browser context.");
+        this.context = await BrowserProvider.getInstance().newContext(this.options.browser);
+        //const contextOptions = this.options.browser && 'contextOptions' in this.options.browser ? this.options.browser.contextOptions : {};
+        this.harness = new WebHarness(this.context, {
+            //fallbackViewportDimensions: contextOptions?.viewport ?? { width: 1024, height: 768 },
+            virtualScreenDimensions: this.options.virtualScreenDimensions,
+            visuals: this.options.visuals
+        });
+        await this.harness.start();
+        this.logger.info("WebHarness started.");
+        if (this.options.url) {
+            this.logger.info(`Navigating to initial URL: ${this.options.url}`);
+            await this.harness.navigate(this.options.url);
+            //await this.harness.waitForStability();
+        }
+        this.logger.info("Started successfully.");
+    }
+    async onStop() {
+        this.logger.info("Stopping...");
+        if (this.harness) {
+            await this.harness.stop();
+            this.logger.info("WebHarness cleaned up.");
+        }
+        if (this.context) {
+            await this.context.close();
+            this.logger.info("Browser context closed.");
+        }
+        // Note: We don't close this.browser here if obtained from BrowserProvider,
+        // as BrowserProvider manages the singleton browser lifecycle.
+        // If this.options.browser was provided, its lifecycle is managed externally.
+        this.logger.info("Stopped successfully.");
+    }
+    getActionSpace() {
+        return [...webActions];
+    }
+    // public get page(): Page {
+    //     if (!this.harness || !this.harness.page) {
+    //         throw new Error("WebInteractionConnector: Harness or Page is not available. Ensure onStart has completed.");
+    //     }
+    //     return this.harness.page;
+    // }
+    getHarness() {
+        if (!this.harness) {
+            throw new Error("WebInteractionConnector: Harness is not available. Ensure onStart has completed.");
+        }
+        return this.harness;
+    }
+    async captureCurrentState() {
+        if (!this.harness || !this.harness.page) {
+            throw new Error("WebInteractionConnector: Harness or Page is not available for capturing state.");
+        }
+        const [screenshot, tabs] = await Promise.all([
+            this.harness.screenshot(),
+            this.harness.retrieveTabState()
+        ]);
+        //const resizedScreenshot = await screenshot.resize()
+        // if (this.options.autoResize) {
+        //     return { screenshot: await screenshot.resize(this.options.autoResize.width, this.options.autoResize.height), tabs: tabs };
+        // }
+        return { screenshot: await this.transformScreenshot(screenshot), tabs: tabs };
+    }
+    async transformScreenshot(screenshot) {
+        if (this.options.virtualScreenDimensions) {
+            return await screenshot.resize(this.options.virtualScreenDimensions.width, this.options.virtualScreenDimensions.height);
+        }
+        else {
+            return screenshot;
+        }
+    }
+    async getLastScreenshot() {
+        //return { image: "", dimensions: { width: 0, height: 0 } };
+        // TODO: better to use last
+        return (await this.captureCurrentState()).screenshot;
+    }
+    async collectObservations() {
+        const currentState = await this.captureCurrentState();
+        const observations = [];
+        const currentTabs = currentState.tabs;
+        let tabInfo = "Open Tabs:\n";
+        currentTabs.tabs.forEach((tab, index) => {
+            tabInfo += `${index === currentTabs.activeTab ? '[ACTIVE] ' : ''}${tab.title} (${tab.url})`;
+        });
+        //console.log("this.options.screenshotMemoryLimit", this.options.screenshotMemoryLimit);
+        const screenshotLimit = this.options.minScreenshots ?? DEFAULT_MIN_RETAINED_SCREENSHOTS;
+        //console.log("screenshotLimit:", screenshotLimit);
+        observations.push(Observation.fromConnector(this.id, await this.transformScreenshot(currentState.screenshot), { type: 'screenshot', limit: screenshotLimit, dedupe: true }));
+        observations.push(Observation.fromConnector(this.id, tabInfo, { type: 'tabinfo', limit: 1 }));
+        return observations;
+    }
+    async getInstructions() {
+        return;
+    }
+}

package/dist/connectors/desktopConnector.d.ts

@@ -0,0 +1,58 @@
+import { AgentConnector } from ".";
+import { ActionDefinition } from '@/actions';
+import { Observation } from "@/memory/observation";
+/**
+ * Generic desktop automation interface.
+ * Implementations can use any desktop automation technology
+ * (Lume, PyAutoGUI, Windows UI Automation, etc.)
+ */
+export interface DesktopInterface {
+    click(x: number, y: number): Promise<void>;
+    rightClick(x: number, y: number): Promise<void>;
+    doubleClick(x: number, y: number): Promise<void>;
+    moveCursor(x: number, y: number): Promise<void>;
+    drag(fromX: number, fromY: number, toX: number, toY: number): Promise<void>;
+    scroll(x: number, y: number, deltaX: number, deltaY: number): Promise<void>;
+    type(text: string): Promise<void>;
+    key(key: string): Promise<void>;
+    hotkey(keys: string[]): Promise<void>;
+    screenshot(): Promise<Buffer>;
+    getScreenSize(): Promise<{
+        width: number;
+        height: number;
+    }>;
+    navigate?(url: string): Promise<void>;
+    getActiveWindow?(): Promise<{
+        title: string;
+        app: string;
+    }>;
+    getOpenWindows?(): Promise<Array<{
+        title: string;
+        app: string;
+        isActive: boolean;
+    }>>;
+    focusWindow?(title: string): Promise<void>;
+    openApplication?(name: string): Promise<void>;
+    closeApplication?(name: string): Promise<void>;
+}
+export interface DesktopConnectorOptions {
+    desktopInterface: DesktopInterface;
+    virtualScreenDimensions?: {
+        width: number;
+        height: number;
+    };
+    minScreenshots?: number;
+}
+export declare class DesktopConnector implements AgentConnector {
+    readonly id: string;
+    private desktopInterface;
+    private options;
+    private logger;
+    constructor(options: DesktopConnectorOptions);
+    onStart(): Promise<void>;
+    onStop(): Promise<void>;
+    getActionSpace(): ActionDefinition<any>[];
+    collectObservations(): Promise<Observation[]>;
+    private formatWindowInfo;
+    getInterface(): DesktopInterface;
+}

package/dist/connectors/desktopConnector.js

@@ -0,0 +1,68 @@
+import { desktopActions } from '@/actions/desktopActions';
+import { Observation } from "@/memory/observation";
+import { Image } from "@/memory/image";
+import logger from "@/logger";
+import sharp from 'sharp';
+export class DesktopConnector {
+    id = "desktop";
+    desktopInterface;
+    options;
+    logger;
+    constructor(options) {
+        this.options = options;
+        this.desktopInterface = options.desktopInterface;
+        this.logger = logger.child({
+            name: `connectors.${this.id}`
+        });
+    }
+    async onStart() {
+        // Desktop interface should already be initialized by the service
+        this.logger.info("Desktop connector started");
+    }
+    async onStop() {
+        // Cleanup handled by the interface provider
+        this.logger.info("Desktop connector stopped");
+    }
+    getActionSpace() {
+        return desktopActions;
+    }
+    async collectObservations() {
+        const observations = [];
+        // Always collect screenshot
+        const screenshot = await this.desktopInterface.screenshot();
+        const sharpImage = sharp(screenshot);
+        const image = new Image(sharpImage);
+        // Apply virtual screen dimensions if configured
+        const transformedImage = this.options.virtualScreenDimensions
+            ? await image.resize(this.options.virtualScreenDimensions.width, this.options.virtualScreenDimensions.height)
+            : image;
+        observations.push(Observation.fromConnector(this.id, transformedImage, {
+            type: 'screenshot',
+            limit: this.options.minScreenshots ?? 2,
+            dedupe: true
+        }));
+        // Optional: Window information
+        if (this.desktopInterface.getOpenWindows) {
+            try {
+                const windows = await this.desktopInterface.getOpenWindows();
+                const windowInfo = this.formatWindowInfo(windows);
+                observations.push(Observation.fromConnector(this.id, windowInfo, { type: 'window-info', limit: 1 }));
+            }
+            catch (error) {
+                this.logger.warn('Failed to get window information', error);
+            }
+        }
+        return observations;
+    }
+    formatWindowInfo(windows) {
+        let info = "Open Windows:\n";
+        windows.forEach(window => {
+            info += `${window.isActive ? '[ACTIVE] ' : ''}${window.title} (${window.app})\n`;
+        });
+        return info;
+    }
+    // Expose interface for actions to use
+    getInterface() {
+        return this.desktopInterface;
+    }
+}

package/dist/connectors/index.d.ts

@@ -0,0 +1,10 @@
+import { ActionDefinition } from '@/actions';
+import { Observation } from '@/memory/observation';
+export interface AgentConnector {
+    id: string;
+    onStart?(): Promise<void>;
+    onStop?(): Promise<void>;
+    getActionSpace?(): ActionDefinition<any>[];
+    collectObservations?(): Promise<Observation[]>;
+    getInstructions?(): Promise<void | string>;
+}

package/dist/connectors/index.js

@@ -0,0 +1,2 @@
+export {};
+//export { BrowserConnector, BrowserConnectorOptions } from './browserConnector';