opensteer 0.6.0 → 0.6.2

package/dist/types-BxiRblC7.d.ts

@@ -0,0 +1,327 @@
+import { BrowserContextOptions } from 'playwright';
+
+type MatchOperator = 'exact' | 'startsWith' | 'contains';
+interface AttributeMatchClause {
+    kind: 'attr';
+    key: string;
+    op?: MatchOperator;
+    value?: string;
+}
+interface PositionMatchClause {
+    kind: 'position';
+    axis: 'nthOfType' | 'nthChild';
+}
+type MatchClause = AttributeMatchClause | PositionMatchClause;
+interface PathNodePosition {
+    nthChild: number;
+    nthOfType: number;
+}
+interface PathNode {
+    tag: string;
+    attrs: Record<string, string>;
+    position: PathNodePosition;
+    match: MatchClause[];
+}
+type DomPath = PathNode[];
+interface ContextHop {
+    kind: 'iframe' | 'shadow';
+    host: DomPath;
+}
+interface ElementPath {
+    context: ContextHop[];
+    nodes: DomPath;
+}
+
+type SnapshotMode = 'action' | 'extraction' | 'clickable' | 'scrollable' | 'full';
+interface SnapshotOptions {
+    mode?: SnapshotMode;
+    withCounters?: boolean;
+    markInteractive?: boolean;
+}
+interface ScreenshotOptions {
+    fullPage?: boolean;
+    type?: 'png' | 'jpeg';
+    /** Ignored for PNG. */
+    quality?: number;
+    omitBackground?: boolean;
+}
+interface AiResolveArgs {
+    html: string;
+    action: string;
+    description: string;
+    url: string | null;
+}
+interface AiResolveResult {
+    element?: number;
+    selector?: string;
+    path?: ElementPath;
+}
+type AiResolveCallbackResult = AiResolveResult | number | string | null | undefined;
+type AiResolveCallback = (args: AiResolveArgs) => Promise<AiResolveCallbackResult>;
+interface AiExtractArgs<TSchema = ExtractSchema> {
+    html: string;
+    schema: TSchema;
+    description?: string;
+    prompt?: string;
+    url: string | null;
+}
+type AiExtractResult<TData = unknown> = TData | ExtractionPlan | string;
+type AiExtractCallback = <TSchema = ExtractSchema, TData = unknown>(args: AiExtractArgs<TSchema>) => Promise<AiExtractResult<TData>>;
+interface GotoOptions {
+    timeout?: number;
+    waitUntil?: 'commit' | 'domcontentloaded' | 'load' | 'networkidle';
+    settleMs?: number;
+}
+interface LaunchOptions {
+    headless?: boolean;
+    executablePath?: string;
+    slowMo?: number;
+    context?: BrowserContextOptions;
+    /** Connect to a running browser. Example: "http://localhost:9222" */
+    connectUrl?: string;
+    /** Browser channel: "chrome", "chrome-beta", or "msedge" */
+    channel?: string;
+    /** Browser profile directory or Chromium user-data dir. Preserves cookies, extensions, and sessions. */
+    profileDir?: string;
+    /** Cloud browser profile preference. Applies only when cloud mode is enabled. */
+    cloudBrowserProfile?: OpensteerCloudBrowserProfileOptions;
+    /** Connection timeout in milliseconds. */
+    timeout?: number;
+}
+interface OpensteerBrowserConfig {
+    headless?: boolean;
+    executablePath?: string;
+    slowMo?: number;
+    /** Connect to a running browser. Example: "http://localhost:9222" */
+    connectUrl?: string;
+    /** Browser channel: "chrome", "chrome-beta", or "msedge" */
+    channel?: string;
+    /** Browser profile directory or Chromium user-data dir. Preserves cookies, extensions, and sessions. */
+    profileDir?: string;
+}
+interface OpensteerStorageConfig {
+    rootDir?: string;
+}
+interface OpensteerCursorColor {
+    r: number;
+    g: number;
+    b: number;
+    a: number;
+}
+interface OpensteerCursorStyle {
+    size?: number;
+    fillColor?: OpensteerCursorColor;
+    outlineColor?: OpensteerCursorColor;
+    haloColor?: OpensteerCursorColor;
+    pulseScale?: number;
+}
+type OpensteerCursorProfile = 'snappy';
+interface OpensteerCursorConfig {
+    enabled?: boolean;
+    profile?: OpensteerCursorProfile;
+    style?: OpensteerCursorStyle;
+}
+type OpensteerAuthScheme = 'api-key' | 'bearer';
+type OpensteerCloudAnnouncePolicy = 'always' | 'off' | 'tty';
+interface OpensteerCloudBrowserProfileOptions {
+    profileId: string;
+    reuseIfActive?: boolean;
+}
+interface OpensteerCloudOptions {
+    apiKey?: string;
+    accessToken?: string;
+    baseUrl?: string;
+    authScheme?: OpensteerAuthScheme;
+    announce?: OpensteerCloudAnnouncePolicy;
+    browserProfile?: OpensteerCloudBrowserProfileOptions;
+}
+type OpensteerCloudConfig = boolean | OpensteerCloudOptions;
+interface OpensteerConfig {
+    name?: string;
+    browser?: OpensteerBrowserConfig;
+    storage?: OpensteerStorageConfig;
+    cursor?: OpensteerCursorConfig;
+    cloud?: OpensteerCloudConfig;
+    model?: string;
+    debug?: boolean;
+}
+interface ActionWaitOptions {
+    enabled?: boolean;
+    timeout?: number;
+    settleMs?: number;
+    networkQuietMs?: number;
+    includeNetwork?: boolean;
+}
+interface BaseActionOptions {
+    description?: string;
+    element?: number;
+    selector?: string;
+    wait?: false | ActionWaitOptions;
+}
+interface ClickOptions extends BaseActionOptions {
+    button?: 'left' | 'right' | 'middle';
+    clickCount?: number;
+    modifiers?: Array<'Alt' | 'Control' | 'Meta' | 'Shift'>;
+}
+interface HoverOptions extends BaseActionOptions {
+    force?: boolean;
+    position?: {
+        x: number;
+        y: number;
+    };
+}
+interface InputOptions extends BaseActionOptions {
+    text: string;
+    clear?: boolean;
+    pressEnter?: boolean;
+}
+interface SelectOptions extends BaseActionOptions {
+    value?: string;
+    label?: string;
+    index?: number;
+}
+interface ScrollOptions extends BaseActionOptions {
+    direction?: 'up' | 'down' | 'left' | 'right';
+    amount?: number;
+}
+interface ExtractSchemaField {
+    element?: number;
+    selector?: string;
+    attribute?: string;
+    source?: 'current_url';
+}
+type ExtractSchemaValue = ExtractSchemaField | string | number | boolean | null | ExtractSchema | ExtractSchema[];
+interface ExtractSchema {
+    [key: string]: ExtractSchemaValue;
+}
+interface ExtractOptions<TSchema = ExtractSchema> extends BaseActionOptions {
+    schema?: TSchema;
+    prompt?: string;
+    snapshot?: SnapshotOptions;
+}
+interface ExtractionFieldPlan {
+    element?: number;
+    selector?: string;
+    attribute?: string;
+    source?: 'current_url';
+}
+interface ExtractionPlan {
+    fields?: Record<string, ExtractionFieldPlan>;
+    paths?: Record<string, ElementPath>;
+    data?: unknown;
+}
+interface ExtractFromPlanOptions<TSchema = ExtractSchema> {
+    description?: string;
+    schema: TSchema;
+    plan: ExtractionPlan;
+}
+interface ActionResult {
+    method: string;
+    namespace: string;
+    persisted: boolean;
+    pathFile: string | null;
+    selectorUsed?: string | null;
+}
+interface OpensteerCursorState {
+    enabled: boolean;
+    active: boolean;
+    reason?: string;
+}
+interface ExtractionRunResult<T = unknown> {
+    namespace: string;
+    persisted: boolean;
+    pathFile: string | null;
+    data: T;
+    paths: Record<string, ElementPath>;
+}
+interface StateResult {
+    url: string;
+    title: string;
+    html: string;
+}
+interface TabInfo {
+    index: number;
+    url: string;
+    title: string;
+    active: boolean;
+}
+interface CookieParam {
+    name: string;
+    value: string;
+    url?: string;
+    domain?: string;
+    path?: string;
+    expires?: number;
+    httpOnly?: boolean;
+    secure?: boolean;
+    sameSite?: 'Strict' | 'Lax' | 'None';
+}
+interface FileUploadOptions extends BaseActionOptions {
+    paths: string[];
+}
+interface BoundingBox {
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+}
+type OpensteerAgentMode = 'cua';
+type OpensteerAgentProvider = 'openai' | 'anthropic' | 'google';
+interface OpensteerAgentModelConfig {
+    modelName: string;
+    apiKey?: string;
+    baseUrl?: string;
+    organization?: string;
+    thinkingBudget?: number;
+    environment?: string;
+}
+interface OpensteerAgentConfig {
+    mode: OpensteerAgentMode;
+    model?: string | OpensteerAgentModelConfig;
+    systemPrompt?: string;
+    waitBetweenActionsMs?: number;
+}
+interface OpensteerAgentExecuteOptions {
+    instruction: string;
+    maxSteps?: number;
+    highlightCursor?: boolean;
+}
+interface OpensteerAgentUsage {
+    inputTokens: number;
+    outputTokens: number;
+    reasoningTokens?: number;
+    inferenceTimeMs: number;
+}
+interface OpensteerAgentAction {
+    type: string;
+    reasoning?: string;
+    button?: string;
+    clickCount?: number;
+    x?: number;
+    y?: number;
+    text?: string;
+    keys?: string[];
+    scrollX?: number;
+    scrollY?: number;
+    timeMs?: number;
+    url?: string;
+    path?: Array<{
+        x: number;
+        y: number;
+    }>;
+    [key: string]: unknown;
+}
+interface OpensteerAgentResult {
+    success: boolean;
+    completed: boolean;
+    message: string;
+    actions: OpensteerAgentAction[];
+    usage?: OpensteerAgentUsage;
+    provider: OpensteerAgentProvider;
+    model: string;
+}
+interface OpensteerAgentInstance {
+    execute(instructionOrOptions: string | OpensteerAgentExecuteOptions): Promise<OpensteerAgentResult>;
+}
+
+export type { OpensteerBrowserConfig as $, ActionResult as A, BaseActionOptions as B, CookieParam as C, AiResolveArgs as D, ExtractOptions as E, FileUploadOptions as F, GotoOptions as G, HoverOptions as H, InputOptions as I, AiResolveCallbackResult as J, AiResolveResult as K, LaunchOptions as L, AttributeMatchClause as M, ContextHop as N, OpensteerAuthScheme as O, DomPath as P, ExtractSchema as Q, ExtractSchemaField as R, SnapshotOptions as S, TabInfo as T, ExtractSchemaValue as U, ExtractionFieldPlan as V, ExtractionPlan as W, MatchClause as X, MatchOperator as Y, OpensteerAgentMode as Z, OpensteerAgentModelConfig as _, OpensteerConfig as a, OpensteerCloudAnnouncePolicy as a0, OpensteerCloudBrowserProfileOptions as a1, OpensteerCloudConfig as a2, OpensteerCloudOptions as a3, OpensteerCursorColor as a4, OpensteerCursorProfile as a5, OpensteerStorageConfig as a6, PathNode as a7, PathNodePosition as a8, PositionMatchClause as a9, StateResult as b, ScreenshotOptions as c, ClickOptions as d, SelectOptions as e, ScrollOptions as f, BoundingBox as g, ExtractFromPlanOptions as h, ExtractionRunResult as i, OpensteerCursorState as j, OpensteerAgentConfig as k, OpensteerAgentInstance as l, ElementPath as m, SnapshotMode as n, AiResolveCallback as o, AiExtractCallback as p, OpensteerAgentProvider as q, OpensteerAgentAction as r, OpensteerAgentResult as s, OpensteerAgentUsage as t, OpensteerCursorStyle as u, OpensteerCursorConfig as v, OpensteerAgentExecuteOptions as w, ActionWaitOptions as x, AiExtractArgs as y, AiExtractResult as z };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "opensteer",
-  "version": "0.6.0",
+  "version": "0.6.2",
   "description": "Open-source browser automation SDK and CLI that lets AI agents build complex scrapers directly in your codebase.",
   "license": "MIT",
   "type": "module",
@@ -36,6 +36,7 @@
     "ai": "^6.0.77",
     "cheerio": "^1.0.0-rc.12",
     "dotenv": "^17.2.4",
+    "open": "^11.0.0",
     "openai": "^6.25.0",
     "playwright": "^1.50.0",
     "skills": "1.4.3",
@@ -70,7 +71,7 @@
     "url": "https://github.com/steerlabs/opensteer/issues"
   },
   "scripts": {
-    "build": "tsup src/index.ts src/cli/server.ts src/cli/skills-installer.ts --dts --format esm,cjs --clean --external ai --external zod --external @ai-sdk/openai --external @ai-sdk/anthropic --external @ai-sdk/google --external @ai-sdk/xai --external @ai-sdk/groq --external openai --external @anthropic-ai/sdk --external @google/genai",
+    "build": "tsup src/index.ts src/cli/server.ts src/cli/skills-installer.ts src/cli/profile.ts src/cli/auth.ts --dts --format esm,cjs --clean --external ai --external zod --external @ai-sdk/openai --external @ai-sdk/anthropic --external @ai-sdk/google --external @ai-sdk/xai --external @ai-sdk/groq --external openai --external @anthropic-ai/sdk --external @google/genai",
     "test": "vitest run",
     "test:live-web": "vitest run --config vitest.live-web.config.ts",
     "test:unit": "vitest run tests/html tests/element-path tests/config.test.ts tests/storage",

package/skills/opensteer/SKILL.md

@@ -51,12 +51,22 @@ opensteer click 3 --description "the products link"
 opensteer input 5 "laptop" --pressEnter --description "the search input"
 ```
 
-For data: take an extraction snapshot, identify counter numbers for each field, then run `extract` with a schema and `--description`. For arrays, include at least 2 items so Opensteer infers the repeating pattern.
+For data, the agent must define the extraction object from the snapshot.
+
+- First run `opensteer snapshot extraction` and inspect the counters.
+- Decide the exact JSON object the task needs.
+- Treat the extraction snapshot as a planning aid only. It is trimmed/filtered, so do not read final values from the snapshot HTML itself.
+- Build the full `extract` schema yourself so every leaf field is explicitly bound with `{ element: N }`, `{ element: N, attribute: "..." }`, or `{ source: "current_url" }`.
+- Always call `extract` to read the actual field values from the live page/runtime DOM.
+- Use `--description` only to cache that extraction for replay. Do not rely on `--description` to tell Opensteer what data to collect.
+- For arrays, include at least 2 representative items so Opensteer infers the repeating pattern.
+- Do not replace `extract` with custom DOM parsing when the desired output can be expressed as a structured object.
 
 ```bash
 opensteer snapshot extraction
-opensteer extract '{"products":[{"name":{"element":11},"price":{"element":12}},{"name":{"element":25},"price":{"element":26}}]}' \
-  --description "product listing"
+# Decide the full output object first, then bind every leaf field explicitly
+opensteer extract '{"images":[{"imageUrl":{"element":11,"attribute":"src"},"alt":{"element":11,"attribute":"alt"},"caption":{"element":14},"credit":{"element":15}},{"imageUrl":{"element":24,"attribute":"src"},"alt":{"element":24,"attribute":"alt"},"caption":{"element":27},"credit":{"element":28}}]}' \
+  --description "article images with captions and credits"
 ```
 
 Repeat Step 3 → Step 4 for every distinct page type the scraper will visit.
@@ -71,7 +81,7 @@ opensteer close
 
 ## Phase 2 — SDK Scraper Script
 
-Use cached `description` strings (exact match to CLI `--description` values). `name` must match `--name` from Phase 1.
+Use cached `description` strings (exact match to CLI `--description` values) only after Phase 1 has already established the exact extraction schema from `snapshot extraction`. `name` must match `--name` from Phase 1.
 
 ```typescript
 import { Opensteer } from "opensteer";
@@ -115,7 +125,26 @@ await opensteer.select({ description: "...", label: "Option A" });
 await opensteer.scroll({ direction: "down", amount: 500 });
 
 await opensteer.extract({ description: "..." });                            // replay from cache
-await opensteer.extract({ schema: { title: { element: 3 } }, description: "..." }); // first cache
+await opensteer.extract({ schema: { title: { element: 3 } }, description: "..." }); // explicit first cache
+await opensteer.extract({
+  description: "article images with captions and credits",
+  schema: {
+    images: [
+      {
+        imageUrl: { element: 11, attribute: "src" },
+        alt: { element: 11, attribute: "alt" },
+        caption: { element: 14 },
+        credit: { element: 15 },
+      },
+      {
+        imageUrl: { element: 24, attribute: "src" },
+        alt: { element: 24, attribute: "alt" },
+        caption: { element: 27 },
+        credit: { element: 28 },
+      },
+    ],
+  },
+}); // first extraction run: agent defines the full object from the snapshot
 
 await opensteer.waitForText("literal text");
 await opensteer.page.waitForSelector("css-selector");                       // SPA content guard

package/skills/opensteer/references/cli-reference.md

@@ -136,19 +136,21 @@ opensteer wait-selector "h1"                        # Wait for selector to appea
 
 ```bash
 opensteer snapshot extraction
-# Read counters from output, then:
+# `schema-json` describes the output shape. It can use explicit bindings or semantic placeholders.
+
+# Explicit field bindings from observed counters/attributes:
 opensteer extract '{"title":{"element":3},"price":{"element":7}}'
 opensteer extract '{"url":{"element":5,"attribute":"href"}}'
 opensteer extract '{"pageUrl":{"source":"current_url"},"title":{"element":3}}'
 
-# Arrays: include multiple items to identify the pattern
+# Explicit array bindings: include multiple items to identify the repeating pattern
 opensteer extract '{"results":[{"title":{"element":11},"url":{"element":10,"attribute":"href"}},{"title":{"element":16},"url":{"element":15,"attribute":"href"}}]}'
-```
 
-### AI-based (limited and requires LLM API keys)
-
-```bash
-opensteer extract '{"title":"","price":""}' --description "product details"
+# Semantic extraction: use the output shape plus description/prompt
+opensteer extract '{"title":"string","price":"string"}' --description "product details"
+opensteer extract '{"images":[{"imageUrl":"string","alt":"string","caption":"string","credit":"string"}]}' \
+  --description "article images with captions and credits" \
+  --prompt "For each image, return the image URL, alt text, caption, and credit. Prefer caption and credit from the same figure. If missing, look at sibling text, then parent/container text, then nearby alt/data-* attributes."
 ```
 
-Always prefer counter-based. AI extraction requires `@ai-sdk/*` packages and does NOT work from workspace root scripts.
+Use explicit bindings when you need deterministic element-to-field mappings. Use semantic extraction when the fields require relationship inference or fallback rules. `--prompt` is the place to describe those rules.

package/skills/opensteer/references/sdk-reference.md

@@ -64,16 +64,27 @@ const data = await opensteer.extract({
   description: "product details",
 });
 
-// Counter-based (during exploration or when no cache exists)
+// Semantic extraction: schema is the output shape
+const images = await opensteer.extract({
+  description: "article images with captions and credits",
+  prompt: "For each image, return the image URL, alt text, caption, and credit. Prefer caption and credit from the same figure. If missing, look at sibling text, then parent/container text, then nearby alt/data-* attributes.",
+  schema: {
+    images: [{ imageUrl: "string", alt: "string", caption: "string", credit: "string" }],
+  },
+});
+
+// Explicit bindings (during exploration or when no cache exists)
 const data = await opensteer.extract({
   schema: { title: { element: 3 }, price: { element: 7 } },
   description: "product details",
 });
 ```
 
-Schema field types: `{ element: N }`, `{ element: N, attribute: "href" }`, `{ selector: ".price" }`, `{ source: "current_url" }`.
+`schema` describes the output shape, not just selector config. It can use semantic placeholders like `"string"` and arrays of objects, or explicit field bindings such as `{ element: N }`, `{ element: N, attribute: "href" }`, `{ selector: ".price" }`, and `{ source: "current_url" }`.
+
+Use `prompt` to describe relationship/fallback rules, such as matching each image to its caption and credit.
 
-For arrays, include multiple items in the schema. Opensteer caches the structural pattern and expands to all matching items on replay.
+For explicit array bindings, include multiple items in the schema so Opensteer can infer the repeating pattern. For semantic extraction, a single representative object shape is enough.
 
 ## Keyboard