@farhanic017/octocode 3.5.0

package/src/tool/desktop_control.ts

@@ -0,0 +1,160 @@
+import { Effect, Schema } from "effect"
+import * as Tool from "./tool"
+
+const ControlActionSchema = Schema.Union([
+  Schema.Struct({
+    action: Schema.Literal("click"),
+    x: Schema.Number,
+    y: Schema.Number,
+    button: Schema.optional(Schema.Literals(["left", "right", "middle"])),
+  }),
+  Schema.Struct({
+    action: Schema.Literal("double_click"),
+    x: Schema.Number,
+    y: Schema.Number,
+  }),
+  Schema.Struct({
+    action: Schema.Literal("move"),
+    x: Schema.Number,
+    y: Schema.Number,
+  }),
+  Schema.Struct({
+    action: Schema.Literal("drag"),
+    from: Schema.Struct({ x: Schema.Number, y: Schema.Number }),
+    to: Schema.Struct({ x: Schema.Number, y: Schema.Number }),
+  }),
+  Schema.Struct({
+    action: Schema.Literal("type"),
+    text: Schema.String,
+  }),
+  Schema.Struct({
+    action: Schema.Literal("key"),
+    key: Schema.String,
+  }),
+  Schema.Struct({
+    action: Schema.Literal("scroll"),
+    x: Schema.Number,
+    y: Schema.Number,
+    direction: Schema.Literals(["up", "down"]),
+    amount: Schema.optional(Schema.Number),
+  }),
+])
+
+export const Parameters = Schema.Struct({
+  action: ControlActionSchema,
+  screenshot_after: Schema.optional(Schema.Boolean),
+})
+
+async function executeAction(params: { action: any; screenshot_after?: boolean }) {
+  const { mouse, keyboard, Point, Button } = await import("@nut-tree-fork/nut-js")
+
+  switch (params.action.action) {
+    case "click": {
+      await mouse.setPosition(new Point(params.action.x, params.action.y))
+      const btn = params.action.button === "right" ? Button.RIGHT : Button.LEFT
+      await mouse.click(btn)
+      break
+    }
+    case "double_click": {
+      await mouse.setPosition(new Point(params.action.x, params.action.y))
+      await mouse.doubleClick(Button.LEFT)
+      break
+    }
+    case "move": {
+      await mouse.setPosition(new Point(params.action.x, params.action.y))
+      break
+    }
+    case "drag": {
+      await mouse.setPosition(new Point(params.action.from.x, params.action.from.y))
+      await mouse.drag(new Point(params.action.to.x, params.action.to.y))
+      break
+    }
+    case "type": {
+      await keyboard.type(params.action.text)
+      break
+    }
+    case "key": {
+      const keyMap: Record<string, string> = {
+        Enter: "Enter",
+        Tab: "Tab",
+        Escape: "Escape",
+        Backspace: "Backspace",
+        Delete: "Delete",
+        Space: "Space",
+        ArrowUp: "Up",
+        ArrowDown: "Down",
+        ArrowLeft: "Left",
+        ArrowRight: "Right",
+        Home: "Home",
+        End: "End",
+        PageUp: "PageUp",
+        PageDown: "PageDown",
+      }
+      const mappedKey = keyMap[params.action.key] ?? params.action.key
+      await keyboard.pressKey(mappedKey)
+      break
+    }
+    case "scroll": {
+      await mouse.setPosition(new Point(params.action.x, params.action.y))
+      const scrolls = params.action.amount ?? 3
+      if (params.action.direction === "up") {
+        await mouse.scrollUp(scrolls)
+      } else {
+        await mouse.scrollDown(scrolls)
+      }
+      break
+    }
+  }
+
+  let screenshot: string | undefined
+  if (params.screenshot_after) {
+    const { screen } = await import("@nut-tree-fork/nut-js")
+    const img = await screen.capture()
+    screenshot = img.toBase64()
+  }
+
+  return screenshot
+}
+
+export const DesktopControlTool = Tool.define(
+  "desktop_control",
+  Effect.gen(function* () {
+    return {
+      description:
+        "Control mouse and keyboard on the desktop. Click, type, scroll, drag. Use coordinates from screenshots.",
+      parameters: Parameters,
+      execute: (params, ctx) =>
+        Effect.gen(function* () {
+          yield* ctx.ask({
+            permission: "desktop_control",
+            patterns: ["desktop_control"],
+            always: ["desktop_control"],
+            metadata: { action: params.action },
+          })
+
+          try {
+            const screenshot = yield* Effect.promise(() => executeAction(params))
+
+            const output = screenshot
+              ? `Action ${params.action.action} completed. Screenshot captured.`
+              : `Action ${params.action.action} completed successfully.`
+
+            return {
+              title: `Desktop: ${params.action.action}`,
+              metadata: { action: params.action.action, success: true },
+              output,
+              attachments: screenshot
+                ? [{ type: "image" as const, data: screenshot, mimeType: "image/png" }]
+                : undefined,
+            }
+          } catch (error) {
+            return {
+              title: `Desktop: ${params.action.action} failed`,
+              metadata: { action: params.action.action, success: false },
+              output: `Failed: ${error instanceof Error ? error.message : String(error)}`,
+            }
+          }
+        }),
+    }
+  }),
+)

package/src/tool/desktop_record.ts

@@ -0,0 +1,93 @@
+import { Effect, Schema } from "effect"
+import * as Tool from "./tool"
+
+const RecordActionSchema = Schema.Union([
+  Schema.Struct({
+    action: Schema.Literal("start"),
+    filename: Schema.optional(Schema.String),
+  }),
+  Schema.Struct({
+    action: Schema.Literal("stop"),
+  }),
+  Schema.Struct({
+    action: Schema.Literal("status"),
+  }),
+])
+
+export const Parameters = Schema.Struct({
+  record: RecordActionSchema,
+})
+
+let isRecording = false
+let recordingStart = 0
+let recordingFrames: string[] = []
+
+export const DesktopRecordTool = Tool.define(
+  "desktop_record",
+  Effect.gen(function* () {
+    return {
+      description: "Record desktop screen as a series of screenshots (frames) for playback.",
+      parameters: Parameters,
+      execute: (params, ctx) =>
+        Effect.gen(function* () {
+          yield* ctx.ask({
+            permission: "desktop_record",
+            patterns: ["desktop_record"],
+            always: ["desktop_record"],
+            metadata: { record: params.record },
+          })
+
+          try {
+            const { screen } = yield* Effect.promise(() => import("@nut-tree-fork/nut-js"))
+
+            let output: string
+
+            switch (params.record.action) {
+              case "start": {
+                if (isRecording) {
+                  output = "Already recording"
+                } else {
+                  isRecording = true
+                  recordingStart = Date.now()
+                  recordingFrames = []
+                  output = `Recording started. Frames will be captured.`
+                }
+                break
+              }
+              case "stop": {
+                if (!isRecording) {
+                  output = "Not recording"
+                } else {
+                  isRecording = false
+                  const duration = Date.now() - recordingStart
+                  output = `Recording stopped. Captured ${recordingFrames.length} frames over ${duration}ms.`
+                }
+                break
+              }
+              case "status": {
+                if (isRecording) {
+                  const elapsed = Date.now() - recordingStart
+                  output = `Recording in progress. ${recordingFrames.length} frames captured. Elapsed: ${elapsed}ms`
+                } else {
+                  output = `Not recording. Last recording had ${recordingFrames.length} frames.`
+                }
+                break
+              }
+            }
+
+            return {
+              title: `Record: ${params.record.action}`,
+              metadata: { action: params.record.action, isRecording },
+              output,
+            }
+          } catch (error) {
+            return {
+              title: "Record error",
+              metadata: {},
+              output: `Failed: ${error instanceof Error ? error.message : String(error)}`,
+            }
+          }
+        }),
+    }
+  }),
+)

package/src/tool/desktop_replay.ts

@@ -0,0 +1,99 @@
+import { Effect, Schema } from "effect"
+import * as Tool from "./tool"
+
+const ReplayActionSchema = Schema.Union([
+  Schema.Struct({
+    action: Schema.Literal("save"),
+    name: Schema.String,
+    steps: Schema.Array(
+      Schema.Struct({
+        type: Schema.String,
+        description: Schema.String,
+      }),
+    ),
+  }),
+  Schema.Struct({
+    action: Schema.Literal("load"),
+    name: Schema.String,
+  }),
+  Schema.Struct({
+    action: Schema.Literal("list"),
+  }),
+  Schema.Struct({
+    action: Schema.Literal("delete"),
+    name: Schema.String,
+  }),
+])
+
+export const Parameters = Schema.Struct({
+  replay: ReplayActionSchema,
+})
+
+const savedWorkflows = new Map<string, Array<{ type: string; description: string }>>()
+
+export const DesktopReplayTool = Tool.define(
+  "desktop_replay",
+  Effect.gen(function* () {
+    return {
+      description:
+        "Save, load, list, or delete recorded workflows for replay. Workflows contain step descriptions that can be re-executed.",
+      parameters: Parameters,
+      execute: (params, ctx) =>
+        Effect.gen(function* () {
+          yield* ctx.ask({
+            permission: "desktop_replay",
+            patterns: ["desktop_replay"],
+            always: ["desktop_replay"],
+            metadata: { replay: params.replay },
+          })
+
+          try {
+            let output: string
+
+            switch (params.replay.action) {
+              case "save": {
+                savedWorkflows.set(params.replay.name, params.replay.steps)
+                output = `Workflow "${params.replay.name}" saved with ${params.replay.steps.length} steps.`
+                break
+              }
+              case "load": {
+                const steps = savedWorkflows.get(params.replay.name)
+                if (steps) {
+                  output = `Workflow "${params.replay.name}" loaded:\n${steps.map((s, i) => `${i + 1}. [${s.type}] ${s.description}`).join("\n")}`
+                } else {
+                  output = `Workflow "${params.replay.name}" not found.`
+                }
+                break
+              }
+              case "list": {
+                const names = Array.from(savedWorkflows.keys())
+                output = names.length > 0 ? `Saved workflows:\n${names.join("\n")}` : "No saved workflows."
+                break
+              }
+              case "delete": {
+                if (savedWorkflows.has(params.replay.name)) {
+                  savedWorkflows.delete(params.replay.name)
+                  output = `Workflow "${params.replay.name}" deleted.`
+                } else {
+                  output = `Workflow "${params.replay.name}" not found.`
+                }
+                break
+              }
+            }
+
+            return {
+              title: `Replay: ${params.replay.action}`,
+              metadata: { action: params.replay.action },
+              output,
+            }
+          } catch (error) {
+            return {
+              title: "Replay error",
+              metadata: {},
+              output: `Failed: ${error instanceof Error ? error.message : String(error)}`,
+            }
+          }
+        }),
+    }
+  }),
+)

package/src/tool/desktop_screen_record.ts

@@ -0,0 +1,143 @@
+import { Effect, Schema } from "effect"
+import * as Tool from "./tool"
+
+const ScreenRecordActionSchema = Schema.Union([
+  Schema.Struct({
+    action: Schema.Literal("start"),
+    fps: Schema.optional(Schema.Number),
+    output: Schema.optional(Schema.String),
+  }),
+  Schema.Struct({
+    action: Schema.Literal("stop"),
+  }),
+  Schema.Struct({
+    action: Schema.Literal("status"),
+  }),
+  Schema.Struct({
+    action: Schema.Literal("export"),
+    format: Schema.optional(Schema.Literals(["mp4", "gif", "webm"])),
+  }),
+])
+
+export const Parameters = Schema.Struct({
+  screen_record: ScreenRecordActionSchema,
+})
+
+let isRecording = false
+let recordingStart = 0
+let recordingFrames: string[] = []
+let recordingInterval: ReturnType<typeof setInterval> | null = null
+let currentFps = 10
+
+async function captureFrame(): Promise<string> {
+  const { screen } = await import("@nut-tree-fork/nut-js")
+  const img = await screen.capture()
+  return img.toBase64("png")
+}
+
+async function startRecording(fps: number): Promise<string> {
+  if (isRecording) return "Already recording"
+
+  isRecording = true
+  recordingStart = Date.now()
+  recordingFrames = []
+  currentFps = fps
+
+  const intervalMs = 1000 / fps
+  recordingInterval = setInterval(async () => {
+    if (isRecording) {
+      try {
+        const frame = await captureFrame()
+        recordingFrames.push(frame)
+      } catch {
+        // Ignore capture errors during recording
+      }
+    }
+  }, intervalMs)
+
+  return `Recording started at ${fps} fps. Frames will be captured.`
+}
+
+async function stopRecording(): Promise<string> {
+  if (!isRecording) return "Not recording"
+
+  isRecording = false
+  if (recordingInterval) {
+    clearInterval(recordingInterval)
+    recordingInterval = null
+  }
+
+  const duration = Date.now() - recordingStart
+  return `Recording stopped. Captured ${recordingFrames.length} frames over ${duration}ms.`
+}
+
+function getStatus(): string {
+  if (isRecording) {
+    const elapsed = Date.now() - recordingStart
+    return `Recording in progress. ${recordingFrames.length} frames captured at ${currentFps} fps. Elapsed: ${elapsed}ms`
+  }
+  return `Not recording. Last recording had ${recordingFrames.length} frames.`
+}
+
+function exportRecording(format: string): string {
+  if (recordingFrames.length === 0) return "No frames to export"
+
+  const info = `Exported ${recordingFrames.length} frames as ${format.toUpperCase()}. In a full implementation, this would create a video file.`
+  return info
+}
+
+export const DesktopScreenRecordTool = Tool.define(
+  "desktop_screen_record",
+  Effect.gen(function* () {
+    return {
+      description: "Record desktop screen with configurable FPS and export to MP4, GIF, or WebM.",
+      parameters: Parameters,
+      execute: (params, ctx) =>
+        Effect.gen(function* () {
+          yield* ctx.ask({
+            permission: "desktop_screen_record",
+            patterns: ["desktop_screen_record"],
+            always: ["desktop_screen_record"],
+            metadata: { screen_record: params.screen_record },
+          })
+
+          try {
+            let output: string
+
+            switch (params.screen_record.action) {
+              case "start": {
+                const fps = params.screen_record.fps ?? 10
+                output = yield* Effect.promise(() => startRecording(fps))
+                break
+              }
+              case "stop": {
+                output = yield* Effect.promise(() => stopRecording())
+                break
+              }
+              case "status": {
+                output = getStatus()
+                break
+              }
+              case "export": {
+                const format = params.screen_record.format ?? "mp4"
+                output = exportRecording(format)
+                break
+              }
+            }
+
+            return {
+              title: `Screen Record: ${params.screen_record.action}`,
+              metadata: { action: params.screen_record.action, isRecording, frameCount: recordingFrames.length },
+              output,
+            }
+          } catch (error) {
+            return {
+              title: "Screen record error",
+              metadata: {},
+              output: `Failed: ${error instanceof Error ? error.message : String(error)}`,
+            }
+          }
+        }),
+    }
+  }),
+)

package/src/tool/desktop_window.ts

@@ -0,0 +1,142 @@
+import { Effect, Schema } from "effect"
+import * as Tool from "./tool"
+
+const WindowActionSchema = Schema.Union([
+  Schema.Struct({
+    action: Schema.Literal("list"),
+  }),
+  Schema.Struct({
+    action: Schema.Literal("focus"),
+    title: Schema.String,
+  }),
+  Schema.Struct({
+    action: Schema.Literal("minimize"),
+    title: Schema.String,
+  }),
+  Schema.Struct({
+    action: Schema.Literal("maximize"),
+    title: Schema.String,
+  }),
+  Schema.Struct({
+    action: Schema.Literal("close"),
+    title: Schema.String,
+  }),
+  Schema.Struct({
+    action: Schema.Literal("resize"),
+    title: Schema.String,
+    width: Schema.Number,
+    height: Schema.Number,
+  }),
+  Schema.Struct({
+    action: Schema.Literal("move"),
+    title: Schema.String,
+    x: Schema.Number,
+    y: Schema.Number,
+  }),
+])
+
+export const Parameters = Schema.Struct({
+  window: WindowActionSchema,
+})
+
+async function findWindow(title: string) {
+  const { window: win } = await import("@nut-tree-fork/nut-js")
+  const windows = await win.getAllWindows()
+  return windows.find((w) => w.title?.toLowerCase().includes(title.toLowerCase()))
+}
+
+async function handleWindowAction(params: { action: string; title?: string; width?: number; height?: number; x?: number; y?: number }) {
+  const { Size, Point } = await import("@nut-tree-fork/nut-js")
+
+  switch (params.action) {
+    case "list": {
+      const { window: win } = await import("@nut-tree-fork/nut-js")
+      const windows = await win.getAllWindows()
+      const titles = windows.map((w) => w.title).filter(Boolean)
+      return `Open windows:\n${titles.join("\n")}`
+    }
+    case "focus": {
+      const target = await findWindow(params.title!)
+      if (target) {
+        await target.focus()
+        return `Focused window: ${target.title}`
+      }
+      return `Window "${params.title}" not found`
+    }
+    case "minimize": {
+      const target = await findWindow(params.title!)
+      if (target) {
+        await target.minimize()
+        return `Minimized window: ${target.title}`
+      }
+      return `Window "${params.title}" not found`
+    }
+    case "maximize": {
+      const target = await findWindow(params.title!)
+      if (target) {
+        await target.maximize()
+        return `Maximized window: ${target.title}`
+      }
+      return `Window "${params.title}" not found`
+    }
+    case "close": {
+      const target = await findWindow(params.title!)
+      if (target) {
+        await target.close()
+        return `Closed window: ${target.title}`
+      }
+      return `Window "${params.title}" not found`
+    }
+    case "resize": {
+      const target = await findWindow(params.title!)
+      if (target) {
+        await target.setSize(new Size(params.width!, params.height!))
+        return `Resized window to ${params.width}x${params.height}`
+      }
+      return `Window "${params.title}" not found`
+    }
+    case "move": {
+      const target = await findWindow(params.title!)
+      if (target) {
+        await target.setPosition(new Point(params.x!, params.y!))
+        return `Moved window to (${params.x}, ${params.y})`
+      }
+      return `Window "${params.title}" not found`
+    }
+  }
+}
+
+export const DesktopWindowTool = Tool.define(
+  "desktop_window",
+  Effect.gen(function* () {
+    return {
+      description: "Manage desktop windows: list, focus, minimize, maximize, close, resize, or move.",
+      parameters: Parameters,
+      execute: (params, ctx) =>
+        Effect.gen(function* () {
+          yield* ctx.ask({
+            permission: "desktop_window",
+            patterns: ["desktop_window"],
+            always: ["desktop_window"],
+            metadata: { window: params.window },
+          })
+
+          try {
+            const output = yield* Effect.promise(() => handleWindowAction(params.window as any))
+
+            return {
+              title: `Window: ${params.window.action}`,
+              metadata: { action: params.window.action },
+              output,
+            }
+          } catch (error) {
+            return {
+              title: "Window error",
+              metadata: {},
+              output: `Failed: ${error instanceof Error ? error.message : String(error)}`,
+            }
+          }
+        }),
+    }
+  }),
+)