@projectservan8n/cnapse 0.2.1 → 0.4.0

package/src/lib/vision.ts

@@ -0,0 +1,254 @@
+/**
+ * Vision capability - screenshot capture and AI description
+ * Supports Ollama VLMs, OpenRouter, Anthropic, OpenAI
+ */
+
+import { getConfig, getApiKey } from './config.js';
+
+export interface VisionResponse {
+  description: string;
+  screenshot: string; // base64
+}
+
+/**
+ * Capture screenshot and get AI description
+ */
+export async function describeScreen(): Promise<VisionResponse> {
+  const screenshot = await captureScreenshot();
+  if (!screenshot) {
+    throw new Error('Failed to capture screenshot');
+  }
+
+  const config = getConfig();
+  const description = await analyzeWithVision(screenshot, config.provider);
+
+  return { description, screenshot };
+}
+
+/**
+ * Capture screenshot as base64
+ */
+export async function captureScreenshot(): Promise<string | null> {
+  try {
+    // Try screenshot-desktop first (more reliable)
+    const screenshotDesktop = await import('screenshot-desktop');
+    const buffer = await screenshotDesktop.default({ format: 'png' });
+    return buffer.toString('base64');
+  } catch {
+    // Fallback to platform-specific methods
+    return captureScreenFallback();
+  }
+}
+
+async function captureScreenFallback(): Promise<string | null> {
+  const { exec } = await import('child_process');
+  const { promisify } = await import('util');
+  const { tmpdir } = await import('os');
+  const { join } = await import('path');
+  const { readFile, unlink } = await import('fs/promises');
+
+  const execAsync = promisify(exec);
+  const tempFile = join(tmpdir(), `cnapse-screen-${Date.now()}.png`);
+
+  try {
+    const platform = process.platform;
+
+    if (platform === 'win32') {
+      await execAsync(`
+        Add-Type -AssemblyName System.Windows.Forms
+        $screen = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds
+        $bitmap = New-Object System.Drawing.Bitmap($screen.Width, $screen.Height)
+        $graphics = [System.Drawing.Graphics]::FromImage($bitmap)
+        $graphics.CopyFromScreen($screen.Location, [System.Drawing.Point]::Empty, $screen.Size)
+        $bitmap.Save("${tempFile.replace(/\\/g, '\\\\')}")
+        $graphics.Dispose()
+        $bitmap.Dispose()
+      `, { shell: 'powershell.exe' });
+    } else if (platform === 'darwin') {
+      await execAsync(`screencapture -x "${tempFile}"`);
+    } else {
+      await execAsync(`gnome-screenshot -f "${tempFile}" 2>/dev/null || scrot "${tempFile}" 2>/dev/null || import -window root "${tempFile}"`);
+    }
+
+    const imageBuffer = await readFile(tempFile);
+    await unlink(tempFile).catch(() => {});
+    return imageBuffer.toString('base64');
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Analyze screenshot with vision-capable AI
+ */
+async function analyzeWithVision(base64Image: string, provider: string): Promise<string> {
+  const prompt = `Look at this screenshot and describe:
+1. What application or window is visible
+2. Key UI elements you can see (buttons, text fields, menus)
+3. What the user appears to be doing or could do next
+4. Any notable content or state
+
+Be concise but helpful.`;
+
+  switch (provider) {
+    case 'ollama':
+      return analyzeWithOllama(base64Image, prompt);
+    case 'openrouter':
+      return analyzeWithOpenRouter(base64Image, prompt);
+    case 'anthropic':
+      return analyzeWithAnthropic(base64Image, prompt);
+    case 'openai':
+      return analyzeWithOpenAI(base64Image, prompt);
+    default:
+      throw new Error(`Vision not supported for provider: ${provider}`);
+  }
+}
+
+async function analyzeWithOllama(base64Image: string, prompt: string): Promise<string> {
+  const config = getConfig();
+  const ollamaHost = config.ollamaHost || 'http://localhost:11434';
+
+  // Use a vision-capable model (llava, llama3.2-vision, bakllava)
+  const visionModels = ['llava', 'llama3.2-vision', 'bakllava', 'llava-llama3'];
+  const model = visionModels.find(m => config.model.includes(m)) || 'llava';
+
+  const response = await fetch(`${ollamaHost}/api/generate`, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({
+      model,
+      prompt,
+      images: [base64Image],
+      stream: false,
+    }),
+  });
+
+  if (!response.ok) {
+    const text = await response.text();
+    throw new Error(`Ollama vision error: ${text}`);
+  }
+
+  const data = await response.json() as { response: string };
+  return data.response || 'Unable to analyze image';
+}
+
+async function analyzeWithOpenRouter(base64Image: string, prompt: string): Promise<string> {
+  const apiKey = getApiKey('openrouter');
+  if (!apiKey) throw new Error('OpenRouter API key not configured');
+
+  // Use a vision-capable model
+  const model = 'anthropic/claude-3-5-sonnet'; // or 'openai/gpt-4-vision-preview'
+
+  const response = await fetch('https://openrouter.ai/api/v1/chat/completions', {
+    method: 'POST',
+    headers: {
+      'Authorization': `Bearer ${apiKey}`,
+      'Content-Type': 'application/json',
+      'HTTP-Referer': 'https://c-napse.up.railway.app',
+      'X-Title': 'C-napse',
+    },
+    body: JSON.stringify({
+      model,
+      messages: [
+        {
+          role: 'user',
+          content: [
+            { type: 'text', text: prompt },
+            {
+              type: 'image_url',
+              image_url: { url: `data:image/png;base64,${base64Image}` },
+            },
+          ],
+        },
+      ],
+      max_tokens: 1000,
+    }),
+  });
+
+  if (!response.ok) {
+    const text = await response.text();
+    throw new Error(`OpenRouter vision error: ${text}`);
+  }
+
+  const data = await response.json() as { choices: Array<{ message: { content: string } }> };
+  return data.choices?.[0]?.message?.content || 'Unable to analyze image';
+}
+
+async function analyzeWithAnthropic(base64Image: string, prompt: string): Promise<string> {
+  const apiKey = getApiKey('anthropic');
+  if (!apiKey) throw new Error('Anthropic API key not configured');
+
+  const response = await fetch('https://api.anthropic.com/v1/messages', {
+    method: 'POST',
+    headers: {
+      'x-api-key': apiKey,
+      'anthropic-version': '2023-06-01',
+      'Content-Type': 'application/json',
+    },
+    body: JSON.stringify({
+      model: 'claude-3-5-sonnet-20241022',
+      max_tokens: 1000,
+      messages: [
+        {
+          role: 'user',
+          content: [
+            {
+              type: 'image',
+              source: {
+                type: 'base64',
+                media_type: 'image/png',
+                data: base64Image,
+              },
+            },
+            { type: 'text', text: prompt },
+          ],
+        },
+      ],
+    }),
+  });
+
+  if (!response.ok) {
+    const text = await response.text();
+    throw new Error(`Anthropic vision error: ${text}`);
+  }
+
+  const data = await response.json() as { content: Array<{ text: string }> };
+  return data.content?.[0]?.text || 'Unable to analyze image';
+}
+
+async function analyzeWithOpenAI(base64Image: string, prompt: string): Promise<string> {
+  const apiKey = getApiKey('openai');
+  if (!apiKey) throw new Error('OpenAI API key not configured');
+
+  const response = await fetch('https://api.openai.com/v1/chat/completions', {
+    method: 'POST',
+    headers: {
+      'Authorization': `Bearer ${apiKey}`,
+      'Content-Type': 'application/json',
+    },
+    body: JSON.stringify({
+      model: 'gpt-4-vision-preview',
+      messages: [
+        {
+          role: 'user',
+          content: [
+            { type: 'text', text: prompt },
+            {
+              type: 'image_url',
+              image_url: { url: `data:image/png;base64,${base64Image}` },
+            },
+          ],
+        },
+      ],
+      max_tokens: 1000,
+    }),
+  });
+
+  if (!response.ok) {
+    const text = await response.text();
+    throw new Error(`OpenAI vision error: ${text}`);
+  }
+
+  const data = await response.json() as { choices: Array<{ message: { content: string } }> };
+  return data.choices?.[0]?.message?.content || 'Unable to analyze image';
+}

package/src/services/telegram.ts

@@ -0,0 +1,278 @@
+/**
+ * Telegram Bot Service - Remote PC control via Telegram
+ */
+
+import { EventEmitter } from 'events';
+import { getConfig, getApiKey } from '../lib/config.js';
+import { describeScreen, captureScreenshot } from '../lib/vision.js';
+import { runCommand } from '../tools/shell.js';
+
+export interface TelegramMessage {
+  chatId: number;
+  text: string;
+  from: string;
+}
+
+export interface TelegramBotEvents {
+  message: (msg: TelegramMessage) => void;
+  command: (cmd: string, args: string, chatId: number) => void;
+  error: (error: Error) => void;
+  started: () => void;
+  stopped: () => void;
+}
+
+export class TelegramBotService extends EventEmitter {
+  private bot: any = null;
+  private isRunning = false;
+  private allowedChatIds: Set<number> = new Set();
+
+  constructor() {
+    super();
+  }
+
+  /**
+   * Start the Telegram bot
+   */
+  async start(): Promise<void> {
+    if (this.isRunning) {
+      return;
+    }
+
+    const botToken = getApiKey('telegram');
+    if (!botToken) {
+      throw new Error('Telegram bot token not configured. Use: cnapse auth telegram YOUR_BOT_TOKEN');
+    }
+
+    try {
+      // Dynamically import telegraf
+      const { Telegraf } = await import('telegraf');
+      this.bot = new Telegraf(botToken);
+
+      // Load allowed chat IDs from config
+      const config = getConfig();
+      if (config.telegram?.chatId) {
+        this.allowedChatIds.add(config.telegram.chatId);
+      }
+
+      this.setupHandlers();
+
+      // Start polling
+      await this.bot.launch();
+      this.isRunning = true;
+      this.emit('started');
+    } catch (error) {
+      throw new Error(`Failed to start Telegram bot: ${error instanceof Error ? error.message : 'Unknown error'}`);
+    }
+  }
+
+  /**
+   * Stop the Telegram bot
+   */
+  async stop(): Promise<void> {
+    if (!this.isRunning || !this.bot) {
+      return;
+    }
+
+    this.bot.stop('SIGTERM');
+    this.isRunning = false;
+    this.bot = null;
+    this.emit('stopped');
+  }
+
+  /**
+   * Check if bot is running
+   */
+  get running(): boolean {
+    return this.isRunning;
+  }
+
+  /**
+   * Setup message and command handlers
+   */
+  private setupHandlers(): void {
+    if (!this.bot) return;
+
+    // /start command - registers user
+    this.bot.command('start', async (ctx: any) => {
+      const chatId = ctx.chat.id;
+      this.allowedChatIds.add(chatId);
+      await ctx.reply(
+        '🤖 C-napse connected!\n\n' +
+        'Commands:\n' +
+        '/screen - Take screenshot\n' +
+        '/describe - Screenshot + AI description\n' +
+        '/run <cmd> - Execute command\n' +
+        '/status - System status\n\n' +
+        `Your chat ID: ${chatId}`
+      );
+    });
+
+    // /screen command - send screenshot
+    this.bot.command('screen', async (ctx: any) => {
+      if (!this.isAllowed(ctx.chat.id)) {
+        await ctx.reply('⛔ Not authorized. Send /start first.');
+        return;
+      }
+
+      await ctx.reply('📸 Taking screenshot...');
+
+      try {
+        const screenshot = await captureScreenshot();
+        if (!screenshot) {
+          await ctx.reply('❌ Failed to capture screenshot');
+          return;
+        }
+
+        // Send as photo
+        const buffer = Buffer.from(screenshot, 'base64');
+        await ctx.replyWithPhoto({ source: buffer }, { caption: '📸 Current screen' });
+      } catch (error) {
+        await ctx.reply(`❌ Error: ${error instanceof Error ? error.message : 'Unknown error'}`);
+      }
+    });
+
+    // /describe command - screenshot + AI description
+    this.bot.command('describe', async (ctx: any) => {
+      if (!this.isAllowed(ctx.chat.id)) {
+        await ctx.reply('⛔ Not authorized. Send /start first.');
+        return;
+      }
+
+      await ctx.reply('🔍 Analyzing screen...');
+
+      try {
+        const result = await describeScreen();
+        const buffer = Buffer.from(result.screenshot, 'base64');
+
+        // Send photo with description as caption
+        const caption = `🖥️ Screen Analysis:\n\n${result.description}`.slice(0, 1024); // Telegram caption limit
+        await ctx.replyWithPhoto({ source: buffer }, { caption });
+
+        // If description is longer, send the rest as text
+        if (result.description.length > 900) {
+          await ctx.reply(result.description);
+        }
+      } catch (error) {
+        await ctx.reply(`❌ Error: ${error instanceof Error ? error.message : 'Unknown error'}`);
+      }
+    });
+
+    // /run command - execute shell command
+    this.bot.command('run', async (ctx: any) => {
+      if (!this.isAllowed(ctx.chat.id)) {
+        await ctx.reply('⛔ Not authorized. Send /start first.');
+        return;
+      }
+
+      const cmd = ctx.message.text.replace('/run ', '').trim();
+      if (!cmd) {
+        await ctx.reply('Usage: /run <command>\nExample: /run dir');
+        return;
+      }
+
+      await ctx.reply(`⚙️ Running: ${cmd}`);
+
+      try {
+        const result = await runCommand(cmd, 30000);
+        if (result.success) {
+          const output = result.output.slice(0, 4000) || '(no output)';
+          await ctx.reply(`✅ Output:\n\`\`\`\n${output}\n\`\`\``, { parse_mode: 'Markdown' });
+        } else {
+          await ctx.reply(`❌ Error:\n\`\`\`\n${result.error}\n\`\`\``, { parse_mode: 'Markdown' });
+        }
+      } catch (error) {
+        await ctx.reply(`❌ Error: ${error instanceof Error ? error.message : 'Unknown error'}`);
+      }
+    });
+
+    // /status command - system status
+    this.bot.command('status', async (ctx: any) => {
+      if (!this.isAllowed(ctx.chat.id)) {
+        await ctx.reply('⛔ Not authorized. Send /start first.');
+        return;
+      }
+
+      const config = getConfig();
+      const status = [
+        '📊 C-napse Status',
+        '',
+        `Provider: ${config.provider}`,
+        `Model: ${config.model}`,
+        `Platform: ${process.platform}`,
+        `Node: ${process.version}`,
+      ].join('\n');
+
+      await ctx.reply(status);
+    });
+
+    // Handle text messages - forward to AI
+    this.bot.on('text', async (ctx: any) => {
+      if (!this.isAllowed(ctx.chat.id)) {
+        return;
+      }
+
+      // Skip commands
+      if (ctx.message.text.startsWith('/')) {
+        return;
+      }
+
+      const message: TelegramMessage = {
+        chatId: ctx.chat.id,
+        text: ctx.message.text,
+        from: ctx.from.username || ctx.from.first_name || 'User',
+      };
+
+      this.emit('message', message);
+      this.emit('command', 'chat', ctx.message.text, ctx.chat.id);
+    });
+
+    // Error handling
+    this.bot.catch((err: Error) => {
+      this.emit('error', err);
+    });
+  }
+
+  /**
+   * Check if chat is authorized
+   */
+  private isAllowed(chatId: number): boolean {
+    // If no chat IDs configured, allow all (first-come authorization)
+    if (this.allowedChatIds.size === 0) {
+      return true;
+    }
+    return this.allowedChatIds.has(chatId);
+  }
+
+  /**
+   * Send a message to a specific chat
+   */
+  async sendMessage(chatId: number, text: string): Promise<void> {
+    if (!this.bot || !this.isRunning) {
+      throw new Error('Telegram bot is not running');
+    }
+    await this.bot.telegram.sendMessage(chatId, text);
+  }
+
+  /**
+   * Send a photo to a specific chat
+   */
+  async sendPhoto(chatId: number, base64Image: string, caption?: string): Promise<void> {
+    if (!this.bot || !this.isRunning) {
+      throw new Error('Telegram bot is not running');
+    }
+    const buffer = Buffer.from(base64Image, 'base64');
+    await this.bot.telegram.sendPhoto(chatId, { source: buffer }, { caption });
+  }
+}
+
+// Singleton instance
+let instance: TelegramBotService | null = null;
+
+export function getTelegramBot(): TelegramBotService {
+  if (!instance) {
+    instance = new TelegramBotService();
+  }
+  return instance;
+}
+
+export default TelegramBotService;

package/src/tools/clipboard.ts

@@ -0,0 +1,55 @@
+/**
+ * Clipboard tools
+ */
+
+import clipboardy from 'clipboardy';
+import { ToolResult, ok, err } from './index.js';
+
+/**
+ * Get clipboard contents
+ */
+export async function getClipboard(): Promise<ToolResult> {
+  try {
+    const text = await clipboardy.read();
+    return ok(text);
+  } catch (error: any) {
+    return err(`Failed to read clipboard: ${error.message}`);
+  }
+}
+
+/**
+ * Set clipboard contents
+ */
+export async function setClipboard(text: string): Promise<ToolResult> {
+  try {
+    await clipboardy.write(text);
+    return ok(`Copied ${text.length} characters to clipboard`);
+  } catch (error: any) {
+    return err(`Failed to write clipboard: ${error.message}`);
+  }
+}
+
+/**
+ * Clipboard tool definitions for agents
+ */
+export const clipboardTools = [
+  {
+    name: 'get_clipboard',
+    description: 'Get clipboard contents',
+    parameters: {
+      type: 'object',
+      properties: {},
+    },
+  },
+  {
+    name: 'set_clipboard',
+    description: 'Set clipboard contents',
+    parameters: {
+      type: 'object',
+      properties: {
+        text: { type: 'string', description: 'Text to copy' },
+      },
+      required: ['text'],
+    },
+  },
+];