npm - @oagi/oagi - Versions diffs - 0.1.4 → 0.1.5 - Mend

@oagi/oagi 0.1.4 → 0.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (19) hide show

package/dist/agent_observer.ts +123 -0
package/dist/{chunk-JVNTVY6W.js → chunk-SRTB44IH.js} +1021 -76
package/dist/chunk-SRTB44IH.js.map +1 -0
package/dist/cli.cjs +952 -14
package/dist/cli.cjs.map +1 -1
package/dist/cli.js +1 -1
package/dist/events.ts +21 -0
package/dist/exporters.ts +432 -0
package/dist/index.cjs +1334 -6
package/dist/index.cjs.map +1 -1
package/dist/index.d.cts +165 -18
package/dist/index.d.ts +165 -18
package/dist/index.js +7 -1
package/dist/index.ts +23 -0
package/dist/protocol.ts +14 -0
package/dist/report_template.html +474 -0
package/package.json +4 -2
package/README.md +0 -154
package/dist/chunk-JVNTVY6W.js.map +0 -1

package/dist/report_template.html ADDED Viewed

@@ -0,0 +1,474 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Agent Execution Report</title>
+    <style>
+        body {
+            font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
+            max-width: 1200px;
+            margin: 0 auto;
+            padding: 20px;
+            background: #f5f5f5;
+        }
+        h1 {
+            color: #333;
+            border-bottom: 2px solid #007bff;
+            padding-bottom: 10px;
+        }
+        .step, .plan {
+            background: white;
+            border-radius: 8px;
+            padding: 20px;
+            margin: 20px 0;
+            box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1);
+        }
+        .step h2 {
+            margin-top: 0;
+            color: #007bff;
+        }
+        .plan {
+            background: #e7f3ff;
+        }
+        .plan h3 {
+            margin-top: 0;
+            color: #0056b3;
+        }
+        .timestamp {
+            color: #666;
+            font-size: 0.9em;
+        }
+        .task-id, .request-id {
+            color: #666;
+            font-size: 0.9em;
+            margin-left: 10px;
+        }
+        .task-id code, .request-id code {
+            background: #e9ecef;
+            padding: 2px 6px;
+            border-radius: 3px;
+            font-family: monospace;
+        }
+        .screenshot-container {
+            position: relative;
+            display: inline-block;
+            margin: 10px 0;
+        }
+        .screenshot {
+            max-width: 100%;
+            border: 1px solid #ddd;
+            border-radius: 4px;
+            display: block;
+        }
+        .reasoning {
+            background: #f8f9fa;
+            padding: 10px;
+            border-left: 3px solid #007bff;
+            margin: 10px 0;
+            white-space: pre-wrap;
+        }
+        .actions {
+            margin: 10px 0;
+        }
+        .actions ul {
+            margin: 5px 0;
+            padding-left: 20px;
+        }
+        .actions code {
+            background: #e9ecef;
+            padding: 2px 6px;
+            border-radius: 3px;
+        }
+        .complete {
+            background: #d4edda;
+            color: #155724;
+            padding: 10px;
+            border-radius: 4px;
+            margin-top: 10px;
+        }
+        .action-result {
+            padding: 10px;
+            margin: 5px 0;
+        }
+        .success {
+            color: #155724;
+        }
+        .error {
+            color: #721c24;
+            background: #f8d7da;
+            padding: 10px;
+            border-radius: 4px;
+        }
+        .log {
+            background: #fff3cd;
+            padding: 10px;
+            margin: 10px 0;
+            border-radius: 4px;
+        }
+        .split {
+            text-align: center;
+            margin: 30px 0;
+        }
+        .split h3 {
+            color: #666;
+        }
+        .split-line {
+            border: none;
+            border-top: 2px dashed #ccc;
+            margin: 30px 0;
+        }
+        .url {
+            word-break: break-all;
+        }
+        .plan-result {
+            background: #d1ecf1;
+            color: #0c5460;
+            padding: 10px;
+            border-radius: 4px;
+            margin-top: 10px;
+        }
+        /* Cursor indicators */
+        .click-indicator {
+            position: absolute;
+            width: 20px;
+            height: 20px;
+            border-radius: 50%;
+            background: rgba(255, 0, 0, 0.8);
+            border: 2px solid #fff;
+            box-shadow: 0 0 10px rgba(255, 0, 0, 0.6);
+            transform: translate(-50%, -50%);
+            pointer-events: none;
+            z-index: 10;
+        }
+        .drag-indicator {
+            position: absolute;
+            width: 20px;
+            height: 20px;
+            border-radius: 50%;
+            border: 2px solid #fff;
+            box-shadow: 0 0 10px rgba(0, 0, 0, 0.3);
+            transform: translate(-50%, -50%);
+            pointer-events: none;
+            z-index: 10;
+        }
+        .drag-start {
+            background: rgba(0, 255, 0, 0.8);
+            box-shadow: 0 0 10px rgba(0, 255, 0, 0.6);
+        }
+        .drag-end {
+            background: rgba(255, 0, 0, 0.8);
+            box-shadow: 0 0 10px rgba(255, 0, 0, 0.6);
+        }
+        .drag-line {
+            position: absolute;
+            background: rgba(255, 255, 0, 0.8);
+            height: 3px;
+            border-radius: 2px;
+            box-shadow: 0 0 5px rgba(255, 255, 0, 0.4);
+            pointer-events: none;
+            z-index: 9;
+            transform-origin: left center;
+        }
+        .scroll-indicator {
+            position: absolute;
+            width: 40px;
+            height: 40px;
+            border-radius: 50%;
+            background: rgba(138, 43, 226, 0.9);
+            border: 2px solid #fff;
+            box-shadow: 0 0 15px rgba(138, 43, 226, 0.6);
+            transform: translate(-50%, -50%);
+            pointer-events: none;
+            z-index: 10;
+            display: flex;
+            align-items: center;
+            justify-content: center;
+            font-size: 20px;
+            color: white;
+            font-weight: bold;
+        }
+    </style>
+</head>
+<body>
+<h1>Agent Execution Report</h1>
+<div id="content"></div>
+<script>
+    const eventsData = {EVENTS_DATA};
+    function escapeHtml(text) {
+        const div = document.createElement('div');
+        div.textContent = text;
+        return div.innerHTML;
+    }
+    function removeIndicators(container) {
+        container.querySelectorAll('.click-indicator, .drag-indicator, .drag-line, .scroll-indicator')
+            .forEach(el => el.remove());
+    }
+    function addIndicators(container) {
+        const img = container.querySelector('.screenshot');
+        const actionsAttr = container.getAttribute('data-actions');
+        if (!actionsAttr || !img) return;
+        // Remove existing indicators before adding new ones
+        removeIndicators(container);
+        const actions = JSON.parse(actionsAttr);
+        const imgWidth = img.offsetWidth;
+        const imgHeight = img.offsetHeight;
+        actions.forEach(action => {
+            switch (action.type) {
+                case 'click':
+                    addClickIndicator(container, action, imgWidth, imgHeight);
+                    break;
+                case 'drag':
+                    addDragIndicator(container, action, imgWidth, imgHeight);
+                    break;
+                case 'scroll':
+                    addScrollIndicator(container, action, imgWidth, imgHeight);
+                    break;
+            }
+        });
+    }
+    // Recalculate indicator positions on window resize
+    let resizeTimeout;
+    window.addEventListener('resize', function () {
+        clearTimeout(resizeTimeout);
+        resizeTimeout = setTimeout(function () {
+            document.querySelectorAll('.screenshot-container').forEach(container => {
+                addIndicators(container);
+            });
+        }, 100);
+    });
+    function addClickIndicator(container, action, imgWidth, imgHeight) {
+        const x = (action.x / 1000) * imgWidth;
+        const y = (action.y / 1000) * imgHeight;
+        const indicator = document.createElement('div');
+        indicator.className = 'click-indicator';
+        indicator.style.left = x + 'px';
+        indicator.style.top = y + 'px';
+        container.appendChild(indicator);
+    }
+    function addDragIndicator(container, action, imgWidth, imgHeight) {
+        const x1 = (action.x1 / 1000) * imgWidth;
+        const y1 = (action.y1 / 1000) * imgHeight;
+        const x2 = (action.x2 / 1000) * imgWidth;
+        const y2 = (action.y2 / 1000) * imgHeight;
+        const startIndicator = document.createElement('div');
+        startIndicator.className = 'drag-indicator drag-start';
+        startIndicator.style.left = x1 + 'px';
+        startIndicator.style.top = y1 + 'px';
+        container.appendChild(startIndicator);
+        const endIndicator = document.createElement('div');
+        endIndicator.className = 'drag-indicator drag-end';
+        endIndicator.style.left = x2 + 'px';
+        endIndicator.style.top = y2 + 'px';
+        container.appendChild(endIndicator);
+        const line = document.createElement('div');
+        line.className = 'drag-line';
+        const deltaX = x2 - x1;
+        const deltaY = y2 - y1;
+        const length = Math.sqrt(deltaX * deltaX + deltaY * deltaY);
+        const angle = Math.atan2(deltaY, deltaX) * (180 / Math.PI);
+        line.style.left = x1 + 'px';
+        line.style.top = y1 + 'px';
+        line.style.width = length + 'px';
+        line.style.transform = 'rotate(' + angle + 'deg)';
+        container.appendChild(line);
+    }
+    function addScrollIndicator(container, action, imgWidth, imgHeight) {
+        const x = (action.x / 1000) * imgWidth;
+        const y = (action.y / 1000) * imgHeight;
+        const indicator = document.createElement('div');
+        indicator.className = 'scroll-indicator';
+        indicator.style.left = x + 'px';
+        indicator.style.top = y + 'px';
+        if (action.direction === 'up') {
+            indicator.innerHTML = '&#8593;';
+            indicator.title = 'Scroll Up';
+        } else if (action.direction === 'down') {
+            indicator.innerHTML = '&#8595;';
+            indicator.title = 'Scroll Down';
+        } else {
+            indicator.innerHTML = '&#8597;';
+            indicator.title = 'Scroll';
+        }
+        container.appendChild(indicator);
+    }
+    function renderEvents() {
+        const content = document.getElementById('content');
+        let html = '';
+        eventsData.forEach(event => {
+            const timestamp = event.timestamp;
+            switch (event.event_type) {
+                case 'step':
+                    html += '<div class="step">';
+                    html += `<h2>Step ${event.step_num}</h2>`;
+                    html += `<span class="timestamp">${timestamp}</span>`;
+                    if (event.task_id) {
+                        html += ` <span class="task-id">Task ID: <code>${event.task_id}</code></span>`;
+                    }
+                    if (event.image) {
+                        const actionsJson = JSON.stringify(event.action_coords || []).replace(/"/g, '&quot;');
+                        html += `<div class="screenshot-container" data-actions="${actionsJson}">`;
+                        if (event.image.startsWith('data:') || event.image.startsWith('http')) {
+                            html += `<img src="${event.image}" alt="Step ${event.step_num}" class="screenshot"/>`;
+                        } else {
+                            html += `<img src="data:image/png;base64,${event.image}" alt="Step ${event.step_num}" class="screenshot"/>`;
+                        }
+                        html += '</div>';
+                    }
+                    if (event.reason) {
+                        html += '<div class="reasoning">';
+                        html += `<strong>Reasoning:</strong><p>${escapeHtml(event.reason)}</p>`;
+                        html += '</div>';
+                    }
+                    if (event.actions && event.actions.length > 0) {
+                        html += '<div class="actions"><strong>Planned Actions:</strong><ul>';
+                        event.actions.forEach(action => {
+                            const countStr = action.count > 1 ? ` (x${action.count})` : '';
+                            html += `<li><code>${action.type}</code>: ${escapeHtml(action.argument)}${countStr}</li>`;
+                        });
+                        html += '</ul></div>';
+                    }
+                    if (event.stop) {
+                        html += '<div class="complete">Task Complete</div>';
+                    }
+                    html += '</div>';
+                    break;
+                case 'action':
+                    html += '<div class="action-result">';
+                    html += `<span class="timestamp">${timestamp}</span>`;
+                    if (event.error) {
+                        html += `<div class="error">Error: ${escapeHtml(event.error)}</div>`;
+                    } else {
+                        html += '<div class="success">Actions executed successfully</div>';
+                    }
+                    html += '</div>';
+                    break;
+                case 'log':
+                    html += '<div class="log">';
+                    html += `<span class="timestamp">${timestamp}</span>`;
+                    html += `<p>${escapeHtml(event.message)}</p>`;
+                    html += '</div>';
+                    break;
+                case 'split':
+                    if (event.label) {
+                        html += `<div class="split"><h3>${escapeHtml(event.label)}</h3></div>`;
+                    } else {
+                        html += '<hr class="split-line"/>';
+                    }
+                    break;
+                case 'plan':
+                    const phaseTitles = {
+                        'initial': 'Initial Planning',
+                        'reflection': 'Reflection',
+                        'summary': 'Summary'
+                    };
+                    const phaseTitle = phaseTitles[event.phase] || event.phase;
+                    html += '<div class="plan">';
+                    html += `<h3>${phaseTitle}</h3>`;
+                    html += `<span class="timestamp">${timestamp}</span>`;
+                    if (event.request_id) {
+                        html += ` <span class="request-id">Request ID: <code>${event.request_id}</code></span>`;
+                    }
+                    if (event.image) {
+                        html += '<div class="screenshot-container">';
+                        if (event.image.startsWith('data:') || event.image.startsWith('http')) {
+                            html += `<img src="${event.image}" alt="${phaseTitle}" class="screenshot"/>`;
+                        } else {
+                            html += `<img src="data:image/png;base64,${event.image}" alt="${phaseTitle}" class="screenshot"/>`;
+                        }
+                        html += '</div>';
+                    }
+                    if (event.reasoning) {
+                        html += '<div class="reasoning">';
+                        html += `<strong>Reasoning:</strong><p>${escapeHtml(event.reasoning)}</p>`;
+                        html += '</div>';
+                    }
+                    if (event.result) {
+                        html += `<div class="plan-result"><strong>Result:</strong> ${escapeHtml(event.result)}</div>`;
+                    }
+                    html += '</div>';
+                    break;
+            }
+        });
+        content.innerHTML = html;
+        // Add cursor indicators after images load
+        document.querySelectorAll('.screenshot-container').forEach(container => {
+            const img = container.querySelector('.screenshot');
+            if (img) {
+                if (img.complete) {
+                    addIndicators(container);
+                } else {
+                    img.onload = () => addIndicators(container);
+                }
+            }
+        });
+    }
+    document.addEventListener('DOMContentLoaded', renderEvents);
+</script>
+</body>
+</html>

package/package.json CHANGED Viewed

@@ -1,6 +1,6 @@
 {
   "name": "@oagi/oagi",
-  "version": "0.1.4",
+  "version": "0.1.5",
   "description": "Official API of OpenAGI Foundation",
   "license": "MIT",
   "homepage": "https://agiopen.org/",
@@ -43,10 +43,12 @@
     "husky": "^9.1.7",
     "prettier": "3.7.4",
     "tsup": "^8.5.1",
+    "tsx": "^4.21.0",
     "typescript": "^5.9.3"
   },
   "scripts": {
     "build": "tsup",
-    "start": "node dist/cli.js"
+    "start": "node dist/cli.js",
+    "dev": "tsx src/cli.ts"
   }
 }

package/README.md DELETED Viewed

@@ -1,154 +0,0 @@
-# OAGI TypeScript SDK
-TypeScript SDK for the OAGI API - vision-based task automation.
-## What is OAGI?
-OAGI is the TypeScript SDK for **Lux**, the world's most advanced computer-use model from the OpenAGI Foundation.
-**Computer Use** is AI's ability to operate human-facing software — not just through APIs, but by operating computers natively, just as human users do. It's a paradigm shift in what AI can do: not just generating, reasoning, or researching, but actually operating on your computer.
-Lux comes in three modes, giving you control over depth, speed, and style of execution:
-- **Tasker** — Strictly follows step-by-step instructions with ultra-stable, controllable execution
-- **Actor** — Ideal for immediate tasks, completing actions at near-instant speed
-- **Thinker** — Understands vague, complex goals, performing hour-long executions
-### Use Cases
-With Lux, possibilities are endless. Here are a few examples:
-- **Web Scraping & Data Crawl** — Navigate websites, sort results, and collect product information autonomously
-- **Software QA** — Automate repetitive testing tasks, navigate applications, perform test actions, and validate expected behaviors
-- **Financial Data Extraction** — Navigate to sites like NASDAQ and extract insider activity data
-- **Data Entry** — Enter accurate data across dashboards and forms
-- **Workflow Automation** — Chain together multi-step tasks across different applications
-## Table of Contents
-- [OAGI TypeScript SDK](#oagi-typescript-sdk)
-  - [What is OAGI?](#what-is-oagi)
-    - [Use Cases](#use-cases)
-  - [Table of Contents](#table-of-contents)
-  - [Installation](#installation)
-  - [Quick Start](#quick-start)
-    - [Automated Task Execution](#automated-task-execution)
-    - [Command Line Interface](#command-line-interface)
-    - [Image Processing](#image-processing)
-    - [Manual Control with Actor](#manual-control-with-actor)
-  - [Documentation](#documentation)
-  - [License](#license)
-## Installation
-```bash
-# If you are using Node.js
-npm install @oagi/oagi
-yarn add @oagi/oagi
-pnpm add @oagi/oagi
-# If you are using Deno
-deno add npm:@oagi/oagi
-# If you are using Bun
-bun install @oagi/oagi
-```
-## Quick Start
-Set your API credentials:
-```bash
-export OAGI_API_KEY="your-api-key" # get your API key from https://developer.agiopen.org/
-# export OAGI_BASE_URL="https://api.agiopen.org/", # optional, defaults to production endpoint
-```
-### Automated Task Execution
-Run tasks automatically with screenshot capture and action execution:
-```typescript
-import { DefaultActionHandler, DefaultAgent, ScreenshotMaker } from '@oagi/oagi';
-const agent = new DefaultAgent();
-await agent.execute(
-  'Search weather on Google',
-  new DefaultActionHandler(),
-  new ScreenshotMaker(),
-);
-```
-### Command Line Interface
-Run agents directly from the terminal:
-```bash
-# Run with actor model
-oagi agent run "Go to nasdaq.com, search for AAPL. Under More, go to Insider Activity" --model lux-actor-1
-# Run with thinker mode (uses lux-thinker-1 model with more steps)
-oagi agent run "Look up the store hours for the nearest Apple Store to zip code 23456 using the Apple Store Locator" --model lux-thinker-1
-# Run pre-configured tasker workflows (no instruction needed)
-oagi agent run --mode tasker:software_qa
-# List all available modes
-oagi agent modes
-# Check macOS permissions (screen recording & accessibility)
-oagi agent permission
-# Export execution history
-oagi agent run "Complete the form" --export html --export-file report.html
-```
-CLI options:
-- `--mode`: Agent mode (default: actor). Use `oagi agent modes` to list available modes
-- `--model`: Override the model (default: determined by mode)
-- `--max-steps`: Maximum steps (default: determined by mode)
-- `--temperature`: Sampling temperature (default: determined by mode)
-- `--step-delay`: Delay after each action before next screenshot (default: 0.3s)
-- `--export`: Export format (markdown, html, json)
-- `--export-file`: Output file path for export
-### Image Processing
-Process and optimize images before sending to API:
-```typescript
-import sharp from 'sharp';
-const compressed = await sharp('large_screenshot.png')
-  .resize(1260, 700, { fit: 'fill' })
-  .jpeg({ quality: 85 })
-  .toBuffer();
-```
-### Manual Control with Actor
-For step-by-step control over task execution:
-```typescript
-import { Actor, DefaultActionHandler, ScreenshotMaker } from '@oagi/oagi';
-const actor = new Actor();
-actor.initTask('Complete the form');
-const image_provider = new ScreenshotMaker();
-const action_handler = new DefaultActionHandler();
-for (let i = 0; i < 10; ++i) {
-  const image = await image_provider.provide();
-  const step = await actor.step(image);
-  if (step.stop) break;
-  await action_handler.handle(step.actions);
-}
-```
-## Documentation
-For full Lux documentation and guides, visit the [OAGI Developer Documentation](https://developer.agiopen.org/docs/index).
-## License
-MIT