@ddwang/magnitude-core 0.3.1-ddwang.1

package/dist/memory/serde.js

@@ -0,0 +1,81 @@
+import { Image } from './image';
+export async function observableDataToJson(data) {
+    if (data instanceof Image) {
+        return await data.toJson(); //return imageToJson(data);
+    }
+    if (typeof data === 'string' || typeof data === 'number' || typeof data === 'boolean') {
+        return { type: "primitive", content: data };
+    }
+    if (data === undefined) {
+        return undefined; // JSON.stringify will omit keys with this value or handle standalone undefined
+    }
+    if (data === null) {
+        return null;
+    }
+    if (Array.isArray(data)) {
+        // Filter out undefined items first, then map and process the rest.
+        return Promise.all(data
+            .filter(item => item !== undefined)
+            .map(item => observableDataToJson(item)));
+    }
+    if (typeof data === 'object') { // Known not to be null or array here
+        const processedObject = {};
+        for (const key in data) {
+            if (Object.prototype.hasOwnProperty.call(data, key)) {
+                const value = data[key];
+                const processedValue = observableDataToJson(value);
+                // Only add the key to the new object if its processed value is not undefined
+                if (processedValue !== undefined) {
+                    processedObject[key] = processedValue;
+                }
+            }
+        }
+        return processedObject;
+    }
+    // Fallback for any unexpected data types not covered by ObservableData.
+    // Returning undefined is consistent with how unrepresentable values are handled.
+    return undefined;
+}
+// TODO: Implement deserialization, and actually leverage serde for logging stuff
+export async function jsonToObservableData(data) {
+    // Handle null and undefined primitives
+    if (data === null || data === undefined) {
+        return data;
+    }
+    // Handle arrays by recursively converting each item
+    if (Array.isArray(data)) {
+        return Promise.all(data.map(item => jsonToObservableData(item)));
+    }
+    // Handle objects, which can be StoredMedia, StoredPrimitive, or a generic object
+    if (typeof data === 'object') {
+        // Check for our special typed objects
+        if ('type' in data && typeof data.type === 'string') {
+            switch (data.type) {
+                case 'media':
+                    // Cast the object to StoredMedia and use the static `fromBase64` method.
+                    const media = data;
+                    if (media.storage === 'base64') {
+                        return Image.fromBase64(media.base64);
+                    }
+                    throw new Error(`Unsupported media storage type: ${media.storage}`);
+                case 'primitive':
+                    // Unwrap the primitive value from its container object.
+                    return data.content;
+            }
+        }
+        // Handle generic observable data object by recursively converting each property's value
+        const result = {};
+        const keys = Object.keys(data);
+        // Process all values concurrently for better performance
+        const values = await Promise.all(keys.map(key => jsonToObservableData(data[key])));
+        // Reconstruct the object from the processed keys and values
+        keys.forEach((key, index) => {
+            result[key] = values[index];
+        });
+        return result;
+    }
+    // According to the MultiMediaJson type definition, raw primitives (string, number, boolean)
+    // are not valid at the top level and should be wrapped in a StoredPrimitive object.
+    // Throw an error if we encounter them, as the input is malformed.
+    throw new Error(`Invalid MultiMediaJson format: Unexpected primitive value '${data}'.`);
+}

package/dist/memory/util.d.ts

@@ -0,0 +1,2 @@
+import { MultiMediaMessage } from '@/ai/baml_client';
+export declare function mergeMessages(messages: MultiMediaMessage[]): MultiMediaMessage[];

package/dist/memory/util.js

@@ -0,0 +1,17 @@
+export function mergeMessages(messages) {
+    if (messages.length === 0)
+        return [];
+    const merged = [];
+    let current = messages[0];
+    for (let i = 1; i < messages.length; i++) {
+        if (messages[i].role === current.role) {
+            current.content.push(...messages[i].content);
+        }
+        else {
+            merged.push(current);
+            current = messages[i];
+        }
+    }
+    merged.push(current);
+    return merged;
+}

package/dist/telemetry/events.d.ts

@@ -0,0 +1,9 @@
+import type { Agent } from "@/agent";
+import { LLMClientIdentifier } from "@/ai/types";
+export type UsageReport = {
+    llm: LLMClientIdentifier;
+    inputTokens: number;
+    outputTokens: number;
+    numCalls: number;
+}[];
+export declare function telemetrifyAgent(agent: Agent): void;

package/dist/telemetry/events.js

@@ -0,0 +1,80 @@
+import { createId, sendTelemetry } from "@/telemetry";
+// kinda jank
+function addUsageToReport(report, usage) {
+    const modelHash = JSON.stringify(usage.llm);
+    let exists = false;
+    for (const entry of report) {
+        const compare = JSON.stringify(usage.llm);
+        if (modelHash === compare) {
+            // merge with existing usage
+            exists = true;
+            entry.inputTokens += usage.inputTokens;
+            entry.outputTokens += usage.outputTokens;
+            entry.numCalls += 1;
+        }
+    }
+    if (!exists) {
+        report.push({ ...usage, numCalls: 1 });
+    }
+}
+export function telemetrifyAgent(agent) {
+    // Generate ID here that will be used to represent this agent in all telemetry events
+    const agentId = createId();
+    agent.events.on('start', async () => {
+        await sendTelemetry('agent-start', { agentId });
+    });
+    agent.events.on('stop', async () => {
+        await sendTelemetry('agent-stop', { agentId });
+    });
+    agent.events.on('actStarted', async () => {
+        const startedAt = Date.now();
+        const report = [];
+        let actionCount = 0;
+        const tokenListener = (usage) => {
+            addUsageToReport(report, usage);
+        };
+        agent.events.on('tokensUsed', tokenListener);
+        const actionListener = () => {
+            actionCount++;
+        };
+        agent.events.on('actionDone', actionListener);
+        agent.events.once('actDone', async () => {
+            agent.events.removeListener('tokensUsed', tokenListener);
+            agent.events.removeListener('actionDone', actionListener);
+            const doneAt = Date.now();
+            await sendTelemetry('agent-act', {
+                agentId,
+                startedAt,
+                doneAt,
+                actionCount,
+                modelUsage: report,
+            });
+        });
+    });
+    //if (agent instanceof BrowserAgent) {
+    if (agent.hasOwnProperty('browserAgentEvents')) {
+        // Report nav and extract telemetry if browser agent
+        const browserAgent = agent;
+        browserAgent.browserAgentEvents.on('nav', async () => {
+            await sendTelemetry('agent-nav', { agentId });
+        });
+        browserAgent.browserAgentEvents.on('extractStarted', async () => {
+            const startedAt = Date.now();
+            const report = [];
+            const tokenListener = (usage) => {
+                addUsageToReport(report, usage);
+            };
+            browserAgent.events.on('tokensUsed', tokenListener);
+            browserAgent.browserAgentEvents.once('extractDone', async () => {
+                browserAgent.events.removeListener('tokensUsed', tokenListener);
+                const doneAt = Date.now();
+                await sendTelemetry('agent-extract', {
+                    agentId,
+                    startedAt,
+                    doneAt,
+                    modelUsage: report,
+                });
+            });
+        });
+    }
+}

package/dist/telemetry/index.d.ts

@@ -0,0 +1,6 @@
+import { PostHog } from 'posthog-node';
+export declare const createId: () => string;
+export declare const posthog: PostHog;
+export declare function getMachineId(): string;
+export declare function getCodebaseId(): string | undefined;
+export declare function sendTelemetry(eventName: string, properties: Record<string, any>): Promise<void>;

package/dist/telemetry/index.js

@@ -0,0 +1,91 @@
+import logger from '@/logger';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import * as os from 'node:os';
+import { execSync } from 'node:child_process';
+import cuid2 from '@paralleldrive/cuid2';
+import { PostHog } from 'posthog-node';
+import { VERSION } from '@/version';
+import { createHash } from 'crypto';
+export const createId = cuid2.init({ length: 12 });
+export const posthog = new PostHog('phc_BTdnTtG68V5QG6sqUNGqGfmjXk8g0ePBRu9FIr9upNu', {
+    host: 'https://us.i.posthog.com'
+});
+export function getMachineId() {
+    // Define storage location
+    const dir = path.join(os.homedir(), '.magnitude');
+    const filePath = path.join(dir, 'user.json');
+    try {
+        // Read existing ID if available
+        if (fs.existsSync(filePath)) {
+            const data = JSON.parse(fs.readFileSync(filePath, 'utf8'));
+            if (data.id)
+                return data.id;
+        }
+        // Generate new ID if needed
+        //console.log(`generating new ID in ${filePath}`)
+        fs.mkdirSync(dir, { recursive: true });
+        const id = createId();
+        fs.writeFileSync(filePath, JSON.stringify({ id }));
+        return id;
+    }
+    catch {
+        // Fallback to temporary ID if storage fails
+        return createId();
+    }
+}
+export function getCodebaseId() {
+    try {
+        const command = 'git rev-list --max-parents=0 HEAD';
+        const firstCommitHash = execSync(command, {
+            encoding: 'utf8',
+            stdio: 'pipe',
+        }).trim();
+        //return firstCommitHash;
+        return createHash('sha256').update(firstCommitHash).digest('hex').substring(0, 12);
+    }
+    catch (error) {
+        // ID representing no git repo detected
+        return undefined; //'000000000000';
+    }
+}
+export async function sendTelemetry(eventName, properties) {
+    /**
+     * Send telemetry to Posthog with given event name and properites
+     * - automatically grouped by codebase derived from SHA256 hash of first git commit if available
+     * - telemetryVersion, packageVersion, and codebase all added to properties automatically
+     */
+    const userId = getMachineId();
+    const codebaseId = getCodebaseId();
+    if (codebaseId) {
+        try {
+            posthog.groupIdentify({
+                groupType: 'codebase',
+                groupKey: codebaseId,
+                //properties: {}
+            });
+        }
+        catch (error) {
+            logger.warn(`Failed to identify group: ${error.message}`);
+        }
+    }
+    try {
+        const props = {
+            source: "magnitude-core",
+            packageVersion: VERSION,
+            //telemetryVersion: "0.1",
+            codebase: codebaseId,
+            ...properties,
+        };
+        posthog.capture({
+            distinctId: userId,
+            event: eventName,
+            properties: props,
+            ...(codebaseId ? { groups: { codebase: codebaseId } } : {})
+        });
+        await posthog.shutdown();
+    }
+    catch (error) {
+        logger.warn(`Failed to send telemetry (may have timed out): ${error.message}`);
+    }
+}

package/dist/types.d.ts

@@ -0,0 +1,30 @@
+import { FailureDescriptor } from "./common";
+import { Intent } from "./actions/types";
+export interface TestDataEntry {
+    key: string;
+    value: string;
+    sensitive: boolean;
+}
+export interface TestData {
+    data?: TestDataEntry[];
+    other?: string;
+}
+export interface TestStepDefinition {
+    description: string;
+    checks: string[];
+    testData: TestData;
+}
+export interface TestCaseDefinition {
+    url: string;
+    steps: TestStepDefinition[];
+    recipe?: Intent[];
+}
+export type TestCaseResult = SuccessfulTestCaseResult | FailedTestCaseResult;
+export interface SuccessfulTestCaseResult {
+    passed: true;
+    recipe: Intent[];
+}
+export interface FailedTestCaseResult {
+    passed: false;
+    failure: FailureDescriptor;
+}

package/dist/types.js

@@ -0,0 +1 @@
+export {};

package/dist/util.d.ts

@@ -0,0 +1 @@
+export declare function fnv1a32Hex(str: string, seed?: number): string;

package/dist/util.js

@@ -0,0 +1,12 @@
+export function fnv1a32Hex(str, seed = 0x811c9dc5) {
+    const FNV_PRIME_32 = 0x01000193;
+    let hash = seed;
+    const encoder = new TextEncoder();
+    const bytes = encoder.encode(str);
+    for (let i = 0; i < bytes.length; i++) {
+        hash ^= bytes[i];
+        hash = (hash * FNV_PRIME_32) | 0;
+    }
+    const hexString = (hash >>> 0).toString(16);
+    return hexString.padStart(8, '0');
+}

package/dist/version.d.ts

@@ -0,0 +1 @@
+export declare const VERSION: any;

package/dist/version.js

@@ -0,0 +1,3 @@
+import { createRequire } from 'node:module';
+const require = createRequire(import.meta.url);
+export const VERSION = require('../package.json').version;

package/dist/web/browserProvider.d.ts

@@ -0,0 +1,23 @@
+import { Browser, BrowserContext, BrowserContextOptions, LaunchOptions } from "playwright";
+export type BrowserOptions = {
+    instance: Browser;
+    contextOptions?: BrowserContextOptions;
+} | {
+    cdp: string;
+    contextOptions?: BrowserContextOptions;
+} | {
+    launchOptions?: LaunchOptions;
+    contextOptions?: BrowserContextOptions;
+} | {
+    context: BrowserContext;
+};
+export declare class BrowserProvider {
+    private activeBrowsers;
+    private logger;
+    private constructor();
+    static getInstance(): BrowserProvider;
+    private _launchOrReuseBrowser;
+    _createAndTrackContext(options: BrowserOptions): Promise<BrowserContext>;
+    newContext(options?: BrowserOptions): Promise<BrowserContext>;
+    private _applyEmulationSettings;
+}

package/dist/web/browserProvider.js

@@ -0,0 +1,153 @@
+import { chromium } from "playwright";
+import objectHash from 'object-hash';
+import crypto from 'node:crypto';
+import logger from "@/logger";
+const DEFAULT_BROWSER_OPTIONS = {
+    headless: false,
+    args: ["--disable-gpu", "--disable-blink-features=AutomationControlled"],
+};
+const DEFAULT_BROWSER_CONTEXT_OPTIONS = {
+    viewport: { width: 1024, height: 768 },
+};
+export class BrowserProvider {
+    activeBrowsers = {};
+    logger;
+    constructor() {
+        this.logger = logger.child({ name: 'browser_provider' });
+    }
+    static getInstance() {
+        if (!globalThis.__magnitude__) {
+            globalThis.__magnitude__ = {};
+        }
+        if (!globalThis.__magnitude__.browserProvider) {
+            globalThis.__magnitude__.browserProvider = new BrowserProvider();
+        }
+        return globalThis.__magnitude__.browserProvider;
+    }
+    async _launchOrReuseBrowser(options) {
+        // hash options
+        const hash = objectHash({
+            ...options,
+            logger: options.logger ? crypto.randomUUID() : '' // replace unserializable logger - use UUID to force re-instance in case different loggers provided
+        });
+        let activeBrowser;
+        if (!(hash in this.activeBrowsers)) {
+            this.logger.trace("Launching new browser");
+            // Launch new browser, get the PROMISE
+            const launchPromise = chromium.launch({ ...DEFAULT_BROWSER_OPTIONS, ...options });
+            activeBrowser = {
+                browserPromise: launchPromise,
+                activeContextsCount: 0
+            };
+            // add immediately in case others need to await the same one as well
+            this.activeBrowsers[hash] = activeBrowser;
+            // Wait for browser to fully start
+            const browser = await launchPromise;
+            browser.on('disconnected', () => {
+                delete this.activeBrowsers[hash];
+            });
+            return activeBrowser;
+        }
+        else {
+            this.logger.trace("Browser with same launch options exists, reusing");
+            return this.activeBrowsers[hash];
+        }
+    }
+    async _createAndTrackContext(options) {
+        const activeBrowserEntry = await this._launchOrReuseBrowser('launchOptions' in options ? options.launchOptions : {});
+        const browser = await activeBrowserEntry.browserPromise;
+        const contextOptions = 'contextOptions' in options ? options.contextOptions : undefined;
+        const context = await browser.newContext(contextOptions);
+        // Get viewport dimensions from context options or use defaults
+        const viewport = contextOptions?.viewport || { width: 1024, height: 768 };
+        const deviceScaleFactor = contextOptions?.deviceScaleFactor || 1;
+        // Apply emulation settings to any new pages created
+        context.on('page', async (page) => {
+            const cdpSession = await page.context().newCDPSession(page);
+            await this._applyEmulationSettings(cdpSession, viewport.width, viewport.height, deviceScaleFactor);
+        });
+        activeBrowserEntry.activeContextsCount++;
+        context.on('close', async () => {
+            activeBrowserEntry.activeContextsCount--;
+            if (activeBrowserEntry.activeContextsCount <= 0 && browser.isConnected()) {
+                await browser.close();
+            }
+        });
+        return context;
+    }
+    async newContext(options) {
+        if (options && 'context' in options) {
+            // Context directly provided, we don't need to manage it
+            return options.context;
+        }
+        const dpr = process.env.DEVICE_PIXEL_RATIO ?
+            parseInt(process.env.DEVICE_PIXEL_RATIO) :
+            process.platform === 'darwin' ? 2 : 1;
+        const contextOptions = {
+            ...DEFAULT_BROWSER_CONTEXT_OPTIONS,
+            deviceScaleFactor: dpr,
+            ...(options && 'contextOptions' in options && options.contextOptions ? options.contextOptions : {}) //options.browser?.contextOptions
+        };
+        options = { ...options, contextOptions };
+        if (process.env.MAGNTIUDE_PLAYGROUND) {
+            // this.logger.trace("MAGNITUDE_PLAYGROUND environment detected, connecting to browser via CDP");
+            // Playground environment - force use CDP on 9222
+            //const browser = await chromium.connectOverCDP('http://localhost:9222');
+            //return browser.newContext(options?.contextOptions);
+            this.logger.trace("MAGNITUDE_PLAYGROUND environment detected, applying playground launch options");
+            const playgroundLaunchOptions = {
+                args: [
+                    '--remote-debugging-port=9222',
+                    '--no-sandbox',
+                    '--disable-dev-shm-usage',
+                    '--disable-gpu'
+                ]
+            };
+            // Overwrite any launch options, instance, or cdp configuration with playground launch options
+            // Ignore context options (?)
+            options = {
+                launchOptions: playgroundLaunchOptions
+            };
+        }
+        if ('cdp' in options) {
+            const browser = await chromium.connectOverCDP(options.cdp);
+            if (browser.contexts().length > 0) {
+                return browser.contexts()[0];
+            }
+            else {
+                return browser.newContext(options.contextOptions);
+            }
+        }
+        else if ('instance' in options) {
+            const browser = options.instance;
+            if (browser.contexts().length > 0) {
+                return browser.contexts()[0];
+            }
+            else {
+                return browser.newContext(options.contextOptions);
+            }
+        }
+        else if ('launchOptions' in options) {
+            this.logger.trace('Creating context with custom launch options');
+            return await this._createAndTrackContext(options);
+        }
+        else {
+            // contextOptions might be passed but no instance | cdp | launchOptions
+            this.logger.trace('Creating context for default browser options');
+            return await this._createAndTrackContext(options);
+        }
+    }
+    async _applyEmulationSettings(cdpSession, width, height, deviceScaleFactor) {
+        await cdpSession.send('Emulation.setDeviceMetricsOverride', {
+            width: width,
+            height: height,
+            deviceScaleFactor: deviceScaleFactor,
+            mobile: false,
+            screenWidth: width,
+            screenHeight: height,
+            positionX: 0,
+            positionY: 0,
+            screenOrientation: { angle: 0, type: 'portraitPrimary' }
+        });
+    }
+}

package/dist/web/harness.d.ts

@@ -0,0 +1,102 @@
+import { Page, BrowserContext, PageScreenshotOptions } from "playwright";
+import { WebAction } from '@/web/types';
+import { ActionVisualizer, ActionVisualizerOptions } from "./visualizer";
+import { TabState } from "./tabs";
+import { Image } from '@/memory/image';
+import EventEmitter from "eventemitter3";
+export interface WebHarnessOptions {
+    virtualScreenDimensions?: {
+        width: number;
+        height: number;
+    };
+    visuals?: ActionVisualizerOptions;
+    switchTabsOnActivity?: boolean;
+}
+export interface WebHarnessEvents {
+    'activePageChanged': (page: Page) => Promise<void>;
+}
+export declare class WebHarness {
+    /**
+     * Executes web actions on a page
+     * Not responsible for browser lifecycle
+     */
+    readonly context: BrowserContext;
+    private options;
+    private stability;
+    readonly visualizer: ActionVisualizer;
+    private transformer;
+    private tabs;
+    readonly events: EventEmitter<WebHarnessEvents>;
+    constructor(context: BrowserContext, options?: WebHarnessOptions);
+    setActivePage(page: Page): Promise<void>;
+    retrieveTabState(): Promise<TabState>;
+    start(): Promise<void>;
+    stop(): Promise<void>;
+    get page(): Page;
+    screenshot(options?: PageScreenshotOptions): Promise<Image>;
+    _type(content: string): Promise<void>;
+    transformCoordinates({ x, y }: {
+        x: number;
+        y: number;
+    }): Promise<{
+        x: number;
+        y: number;
+    }>;
+    click({ x, y }: {
+        x: number;
+        y: number;
+    }, options?: {
+        transform: boolean;
+    }): Promise<void>;
+    private _click;
+    rightClick({ x, y }: {
+        x: number;
+        y: number;
+    }, options?: {
+        transform: boolean;
+    }): Promise<void>;
+    doubleClick({ x, y }: {
+        x: number;
+        y: number;
+    }, options?: {
+        transform: boolean;
+    }): Promise<void>;
+    drag({ x1, y1, x2, y2 }: {
+        x1: number;
+        y1: number;
+        x2: number;
+        y2: number;
+    }, options?: {
+        transform: boolean;
+    }): Promise<void>;
+    type({ content }: {
+        content: string;
+    }): Promise<void>;
+    clickAndType({ x, y, content }: {
+        x: number;
+        y: number;
+        content: string;
+    }, options?: {
+        transform: boolean;
+    }): Promise<void>;
+    scroll({ x, y, deltaX, deltaY }: {
+        x: number;
+        y: number;
+        deltaX: number;
+        deltaY: number;
+    }, options?: {
+        transform: boolean;
+    }): Promise<void>;
+    switchTab({ index }: {
+        index: number;
+    }): Promise<void>;
+    newTab(): Promise<void>;
+    navigate(url: string): Promise<void>;
+    selectAll(): Promise<void>;
+    enter(): Promise<void>;
+    backspace(): Promise<void>;
+    tab(): Promise<void>;
+    goBack(): Promise<void>;
+    executeAction(action: WebAction): Promise<void>;
+    waitForStability(timeout?: number): Promise<void>;
+}