@tyvm/knowhow 0.0.144 → 0.0.145

package/src/services/modules/computerUse.ts

@@ -0,0 +1,216 @@
+/**
+ * Shared, driver-agnostic interface for the computer-use service.
+ *
+ * The concrete implementation lives in `@tyvm/knowhow-module-computer-use`, but
+ * we declare the shape here so that `ToolContext` / `ModuleContext` can carry a
+ * `ComputerUse` service without the core knowhow package taking a runtime
+ * dependency on the module.
+ *
+ * An adapter module (e.g. `@tyvm/knowhow-module-computer-use-nutjs`) can
+ * register a `ComputerDriver` against the service in its `register` phase; the
+ * base module then `init()`s and picks the best available driver.
+ */
+
+export interface Point {
+  x: number;
+  y: number;
+}
+
+export interface Size {
+  width: number;
+  height: number;
+}
+
+export interface Region extends Point, Size {}
+
+export type MouseButton = "left" | "right" | "middle";
+
+export interface OverlayPrimitive {
+  kind: "rect" | "circle" | "line" | "point";
+  x: number;
+  y: number;
+  width?: number;
+  height?: number;
+  x2?: number;
+  y2?: number;
+  color?: string;
+  lineWidth?: number;
+}
+
+export interface Display {
+  id: number;
+  bounds: Region;
+  scaleFactor: number;
+  primary: boolean;
+}
+
+export interface ScreenshotOptions {
+  region?: Region;
+  displayId?: number;
+  format?: "png" | "jpeg";
+  scale?: number;
+  /** Native capture downscale. Unlike output encoding scale, drivers may apply
+   * this before raw pixels cross the process boundary. */
+  captureScale?: number;
+}
+
+export interface DriverCapabilities {
+  input: boolean;
+  capture: boolean;
+  windows: boolean;
+  reason?: string;
+}
+
+export interface AccessibilityElement {
+  id: string;
+  role: string;
+  subrole?: string;
+  title?: string;
+  description?: string;
+  value?: string;
+  enabled?: boolean;
+  focused?: boolean;
+  bounds?: Region;
+  actions: string[];
+  childCount: number;
+}
+
+export interface AccessibilityOptions {
+  maxDepth?: number;
+  maxElements?: number;
+  interactiveOnly?: boolean;
+}
+
+/**
+ * The engine seam. Concrete drivers (Rust core, CLI adapter, nut.js demo)
+ * implement this. Registered with the service via `registerDriver`.
+ */
+export interface ComputerDriver {
+  readonly name: string;
+  /**
+   * Lower numbers win when multiple drivers are registered. The default Rust
+   * core registers at 100; a fallback CLI adapter at 500; an experimental demo
+   * adapter can register at e.g. 10 to force selection.
+   */
+  readonly priority?: number;
+
+  capabilities(): Promise<DriverCapabilities>;
+
+  // screen
+  getDisplays(): Promise<Display[]>;
+  screenSize(): Promise<Size>;
+  screenshot(opts?: ScreenshotOptions): Promise<Buffer>;
+  pixelColor(p: Point): Promise<string>;
+
+  // mouse
+  mousePosition(): Promise<Point>;
+  moveMouse(p: Point, opts?: { duration?: number }): Promise<void>;
+  click(button?: MouseButton, opts?: { double?: boolean }): Promise<void>;
+  mouseDown(button?: MouseButton): Promise<void>;
+  mouseUp(button?: MouseButton): Promise<void>;
+  drag(
+    from: Point,
+    to: Point,
+    opts?: { button?: MouseButton; duration?: number }
+  ): Promise<void>;
+  scroll(dx: number, dy: number): Promise<void>;
+
+  // keyboard
+  typeText(text: string, opts?: { delay?: number }): Promise<void>;
+  pressKey(key: string): Promise<void>;
+  hotkey(...keys: string[]): Promise<void>;
+  keyDown(key: string): Promise<void>;
+  keyUp(key: string): Promise<void>;
+
+  // windows (optional / capability-gated)
+  getActiveWindow?(): Promise<{ title: string; app?: string; bounds?: Region } | null>;
+  listWindows?(): Promise<Array<{ title: string; app?: string; bounds?: Region }>>;
+  focusWindow?(match: string): Promise<boolean>;
+
+  accessibilityTrusted?(): Promise<boolean>;
+  accessibilityElements?(options?: AccessibilityOptions): Promise<AccessibilityElement[]>;
+  setAccessibilityValue?(id: string, value: string): Promise<void>;
+  performAccessibilityAction?(id: string, action: string): Promise<void>;
+
+  // click-through visual debugging (optional / platform capability-gated)
+  showOverlay?(primitives: OverlayPrimitive[]): Promise<void>;
+  clearOverlay?(): Promise<void>;
+
+  dispose?(): Promise<void>;
+}
+
+/**
+ * The registered service (`ComputerUse`) surfaced to tools/CLI/MCP. It owns
+ * driver selection and provides the normalized, platform-agnostic verbs.
+ *
+ * We keep this as an interface (not a class) in the core package so the module
+ * owns the concrete implementation; anything reading `context.ComputerUse`
+ * codes against this contract.
+ */
+export interface ComputerUseService {
+  /** Register a driver (called by the base module and any adapter modules). */
+  registerDriver(driver: ComputerDriver): void;
+  /** List the names of all registered drivers. */
+  listDrivers(): string[];
+  /** Resolve/select the active driver (highest priority + capable). */
+  getDriver(): Promise<ComputerDriver>;
+
+  capabilities(): Promise<DriverCapabilities>;
+  getDisplays(): Promise<Display[]>;
+  screenSize(): Promise<Size>;
+  /** Returns encoded image bytes (PNG/JPEG) honoring service defaults. */
+  screenshot(opts?: ScreenshotOptions): Promise<Buffer>;
+  /** Convenience: base64 data (no data: prefix) for multimodal messages. */
+  screenshotBase64(opts?: ScreenshotOptions): Promise<string>;
+  pixelColor(p: Point): Promise<string>;
+
+  mousePosition(): Promise<Point>;
+  moveMouse(p: Point, opts?: { duration?: number }): Promise<void>;
+  click(button?: MouseButton, opts?: { double?: boolean }): Promise<void>;
+  drag(
+    from: Point,
+    to: Point,
+    opts?: { button?: MouseButton; duration?: number }
+  ): Promise<void>;
+  scroll(dx: number, dy: number): Promise<void>;
+  /** Smooth stepped scrolling (repeat a small delta with a pause between). */
+  scrollBy?(
+    dx: number,
+    dy: number,
+    opts?: { repeat?: number; intervalMs?: number }
+  ): Promise<void>;
+
+  typeText(text: string, opts?: { delay?: number }): Promise<void>;
+  pressKey(key: string): Promise<void>;
+  hotkey(...keys: string[]): Promise<void>;
+
+  /** Screenshot with an optional labeled grid and/or crosshair overlay. */
+  screenshotAnnotated?(
+    opts?: ScreenshotOptions & {
+      grid?: boolean;
+      gridStep?: number;
+      crosshair?: Point;
+    }
+  ): Promise<Buffer>;
+
+  getActiveWindow(): Promise<{
+    title: string;
+    app?: string;
+    bounds?: Region;
+  } | null>;
+  listWindows(): Promise<
+    Array<{ title: string; app?: string; bounds?: Region }>
+  >;
+  focusWindow(match: string): Promise<boolean>;
+
+  accessibilityTrusted(): Promise<boolean>;
+  accessibilityElements(options?: AccessibilityOptions): Promise<AccessibilityElement[]>;
+  selectAccessibilityOption(id: string, option: string): Promise<void>;
+  setAccessibilityValue(id: string, value: string): Promise<void>;
+  performAccessibilityAction(id: string, action: string): Promise<void>;
+
+  showOverlay(primitives: OverlayPrimitive[]): Promise<void>;
+  clearOverlay(): Promise<void>;
+
+  dispose(): Promise<void>;
+}

package/src/services/modules/index.ts

@@ -75,6 +75,21 @@ export class ModulesService {
     // modules resolve against cwd/.knowhow/node_modules first. See localResolvePaths()
     // and globalResolvePaths() above.
 
+    // Two-phase load:
+    //   Phase 1 (register): resolve + require EVERY module and run its optional
+    //     `register()`, which may (a) register CLI subcommands on Program and
+    //     (b) inject shared services into `context` (e.g. `ComputerUse`) so that
+    //     sibling/adapter modules loaded later can consume them.
+    //   Phase 2 (init): run every loaded module's `init()`, now that all
+    //     services registered during phase 1 are present. This lets, e.g., a
+    //     `-nutjs` adapter module register a driver against the base
+    //     computer-use service during phase 1, and the base module resolve the
+    //     best driver during phase 2.
+    const loadedModules: {
+      modulePath: string;
+      module: KnowhowModule;
+    }[] = [];
+
     for (const modulePath of allModulePaths) {
       // Build an ordered list of candidate resolutions for this module.
       // Relative paths resolve to a single candidate (relative to cwd), while
@@ -122,15 +137,16 @@ export class ModulesService {
             "ModulesService",
             `🔌 Loading module: ${modulePath} (resolved: ${resolvedPath})`
           );
-          await importedModule.init({
-            config,
-            cwd: process.cwd(),
-            context: context as ModuleContext,
-          });
-          context.Events?.log(
-            "ModulesService",
-            `✅ Module initialized: ${modulePath} (tools: ${importedModule.tools.length}, agents: ${importedModule.agents.length}, plugins: ${importedModule.plugins.length}, clients: ${importedModule.clients.length})`
-          );
+          // Phase 1: register (optional). Injects services into `context` and
+          // registers CLI commands. Must be idempotent (may run in the early
+          // Program-only CLI phase AND the full-services phase).
+          if (typeof importedModule.register === "function") {
+            await importedModule.register({
+              config,
+              cwd: process.cwd(),
+              context: context as ModuleContext,
+            });
+          }
           loaded = true;
           break;
         } catch (err: any) {
@@ -152,6 +168,28 @@ export class ModulesService {
         );
         continue;
       }
+      loadedModules.push({ modulePath, module: importedModule });
+    }
+
+    // Phase 2: init every successfully-loaded module now that all `register`
+    // phases have run and injected their services into `context`.
+    for (const { modulePath, module: importedModule } of loadedModules) {
+      try {
+        await importedModule.init({
+          config,
+          cwd: process.cwd(),
+          context: context as ModuleContext,
+        });
+        context.Events?.log(
+          "ModulesService",
+          `✅ Module initialized: ${modulePath} (tools: ${importedModule.tools.length}, agents: ${importedModule.agents.length}, plugins: ${importedModule.plugins.length}, clients: ${importedModule.clients.length})`
+        );
+      } catch (err: any) {
+        process.stderr.write(
+          `\n⚠️  Failed to init module "${modulePath}": ${err?.message || err}\n\n`
+        );
+        continue;
+      }
       // Only register tools/agents/plugins/clients if the relevant services
       // are available in context (they may not be during early CLI command registration)
       if (context.Agents) {

package/src/services/modules/types.ts

@@ -14,6 +14,7 @@ import { TunnelHandler } from "@tyvm/knowhow-tunnel";
 import { EventService } from "../EventService";
 import { ConversionService } from "../conversion/ConversionService";
 import { BehaviorsService } from "../BehaviorsService";
+import { ComputerUseService } from "./computerUse";
 
 /*
  *
@@ -62,10 +63,31 @@ export interface ModuleContext {
   Conversion?: ConversionService;
   Tunnel?: TunnelHandler;
   Program?: Command;
-  Behaviors?: BehaviorsService
+  Behaviors?: BehaviorsService;
+  ComputerUse?: ComputerUseService;
+  [key: string]: any;
 }
 
 export interface KnowhowModule {
+  /**
+   * Phase 1 (optional): called first, potentially with ONLY `Program` in
+   * context (e.g. during early CLI command registration). Use this to:
+   *   - register CLI subcommands on `context.Program`
+   *   - inject services into the shared context (e.g.
+   *     `context.Tools?.addContext("ComputerUse", svc)`) so that OTHER modules
+   *     (or a sibling adapter module) can consume them during their own init.
+   *
+   * `register` must be side-effect-light and MUST NOT assume the full service
+   * graph is present — only `Program` is guaranteed during the early phase.
+   * It may be called more than once (early CLI phase + full-services phase);
+   * implementations should be idempotent.
+   */
+  register?: (params: InitParams) => Promise<void>;
+  /**
+   * Phase 2: called with the full service graph available. Use this to
+   * actually connect/start things (open drivers, spin up watchers, read
+   * services registered by other modules' `register` phase, etc.).
+   */
   init: (params: InitParams) => Promise<void>;
   commands: ModuleChatCommand[];
   tools: ModuleTool[];

package/src/types.ts

@@ -86,6 +86,9 @@ export type Config = {
   chat?: {
     rootModule?: string;
     renderer?: string;
+    defaultAgent?: string;
+    defaultModel?: string;
+    defaultProvider?: string;
     modules?: string[];
   };
   modules: string[];

package/tests/clients/contextLimits.test.ts

@@ -9,6 +9,8 @@ import {
   GeminiTextPricing,
   AnthropicTextPricing,
   XaiTextPricing,
+  FireworksTextPricing,
+  FireworksEmbeddingPricing,
 } from "../../src/clients/pricing";
 
 // Model ids explicitly marked deprecated across all pricing tables. Deprecated
@@ -21,6 +23,8 @@ const deprecatedModelIds = new Set<string>(
     ...GeminiTextPricing,
     ...AnthropicTextPricing,
     ...XaiTextPricing,
+    ...FireworksTextPricing,
+    ...FireworksEmbeddingPricing,
   } as Record<string, { deprecated?: boolean }>)
     .filter(([, p]) => p && p.deprecated === true)
     .map(([id]) => id)

package/tests/clients/openai.images.test.ts

@@ -0,0 +1,105 @@
+import { GenericOpenAiClient } from "../../src/clients/openai";
+import { Message } from "../../src/clients/types";
+
+describe("GenericOpenAiClient image message conversion", () => {
+  const imageUrl = "data:image/jpeg;base64,/9j/4AAQSkZJRg==";
+  let client: GenericOpenAiClient;
+
+  beforeEach(() => {
+    client = new GenericOpenAiClient("test-key");
+  });
+
+  it("moves tool image output to a user message for Chat Completions", () => {
+    const messages: Message[] = [{
+      role: "tool",
+      tool_call_id: "call_1",
+      name: "loadImageAsBase64",
+      content: [{
+        type: "image_url",
+        image_url: { url: imageUrl, detail: "high" },
+      }],
+    }];
+
+    const converted = client.toChatCompletionMessages(messages);
+
+    expect(converted).toEqual([
+      {
+        role: "tool",
+        tool_call_id: "call_1",
+        content: [{
+          type: "text",
+          text: "Image output attached in the following user message.",
+        }],
+      },
+      {
+        role: "user",
+        content: [{
+          type: "image_url",
+          image_url: { url: imageUrl, detail: "high" },
+        }],
+      },
+    ]);
+  });
+
+  it("uses input_image parts for the Responses API", () => {
+    const converted = client.toResponseContent([
+      { type: "text", text: "Screenshot" },
+      { type: "image_url", image_url: { url: imageUrl, detail: "low" } },
+    ]);
+
+    expect(converted).toEqual([
+      { type: "input_text", text: "Screenshot" },
+      { type: "input_image", image_url: imageUrl, detail: "low" },
+    ]);
+  });
+
+  it.each([
+    { type: "audio_url", audio_url: { url: "data:audio/wav;base64,UklGRg==" } },
+    { type: "video_url", video_url: { url: "data:video/mp4;base64,AAAAIGZ0eXA=" } },
+  ] as const)("rejects unsupported $type Responses API content instead of dropping it", (part) => {
+    expect(() => client.toResponseContent([
+      { type: "text", text: "Media" },
+      part,
+    ])).toThrow(
+      `OpenAI Responses API conversion does not support ${part.type} content.`
+    );
+  });
+
+  it("attaches a tool image as input_image in a Responses API request", async () => {
+    const create = jest.fn().mockResolvedValue({
+      id: "resp_1",
+      output: [],
+      usage: undefined,
+    });
+    (client as any).client = { responses: { create } };
+
+    await client.createChatResponse({
+      model: "gpt-5.4",
+      reasoning_effort: "none",
+      messages: [{
+        role: "tool",
+        tool_call_id: "call_1",
+        content: [{
+          type: "image_url",
+          image_url: { url: imageUrl, detail: "auto" },
+        }],
+      }],
+    });
+
+    expect(create.mock.calls[0][0].input).toEqual([
+      {
+        type: "function_call_output",
+        call_id: "call_1",
+        output: "Image output attached in the following user message.",
+      },
+      {
+        role: "user",
+        content: [{
+          type: "input_image",
+          image_url: imageUrl,
+          detail: "auto",
+        }],
+      },
+    ]);
+  });
+});

package/tests/clients/pricing.test.ts

@@ -30,6 +30,8 @@ import {
   GeminiTextPricing,
   AnthropicTextPricing,
   XaiTextPricing,
+  FireworksTextPricing,
+  FireworksEmbeddingPricing,
   ALL_MODEL_CATALOG,
   XaiImagePricing,
   XaiVideoPricing,
@@ -61,6 +63,8 @@ describe("Model Pricing Coverage", () => {
     ...GeminiTextPricing,
     ...AnthropicTextPricing,
     ...XaiTextPricing,
+    ...FireworksTextPricing,
+    ...FireworksEmbeddingPricing,
   };
 
   describe("Text completion models have pricing", () => {
@@ -166,6 +170,8 @@ describe("Model Catalog Coverage", () => {
       ...GeminiTextPricing,
       ...AnthropicTextPricing,
       ...XaiTextPricing,
+      ...FireworksTextPricing,
+      ...FireworksEmbeddingPricing,
     } as Record<string, { deprecated?: boolean }>)
       .filter(([, p]) => p && p.deprecated === true)
       .map(([id]) => id)