opc-agent 4.0.0 → 4.0.1

package/src/tools/web-scraper.ts

@@ -0,0 +1,179 @@
+/**
+ * Web Scraper - v0.10.0
+ * Fetch URL content and extract readable text in markdown format.
+ * Uses a simple readability-style extraction (no external dependencies).
+ */
+
+export interface ScrapedContent {
+  title: string;
+  content: string;  // markdown
+  url: string;
+  wordCount: number;
+}
+
+const MAX_CONTENT_LENGTH = 5000;
+
+/**
+ * Fetch a URL and extract readable content as markdown.
+ */
+export async function scrapeUrl(url: string, maxLength = MAX_CONTENT_LENGTH): Promise<ScrapedContent> {
+  const response = await fetch(url, {
+    headers: {
+      'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
+      Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+    },
+    signal: AbortSignal.timeout(15000),
+    redirect: 'follow',
+  });
+
+  const contentType = response.headers.get('content-type') || '';
+  const text = await response.text();
+
+  // If not HTML, return raw text
+  if (!contentType.includes('html')) {
+    const truncated = text.slice(0, maxLength);
+    return {
+      title: url,
+      content: truncated,
+      url,
+      wordCount: truncated.split(/\s+/).length,
+    };
+  }
+
+  return extractReadableContent(text, url, maxLength);
+}
+
+/**
+ * Extract readable content from HTML using simple heuristics.
+ */
+export function extractReadableContent(html: string, url: string, maxLength = MAX_CONTENT_LENGTH): ScrapedContent {
+  // Extract title
+  const titleMatch = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
+  const title = titleMatch ? decodeEntities(titleMatch[1]).trim() : url;
+
+  // Remove non-content elements
+  let content = html;
+
+  // Remove script, style, nav, header, footer, aside, iframe
+  const removePatterns = [
+    /<script[\s\S]*?<\/script>/gi,
+    /<style[\s\S]*?<\/style>/gi,
+    /<nav[\s\S]*?<\/nav>/gi,
+    /<footer[\s\S]*?<\/footer>/gi,
+    /<aside[\s\S]*?<\/aside>/gi,
+    /<iframe[\s\S]*?<\/iframe>/gi,
+    /<noscript[\s\S]*?<\/noscript>/gi,
+    /<!--[\s\S]*?-->/g,
+  ];
+
+  for (const pattern of removePatterns) {
+    content = content.replace(pattern, '');
+  }
+
+  // Try to find main content area
+  const mainContent = findMainContent(content);
+  content = mainContent || content;
+
+  // Convert to markdown-ish text
+  content = htmlToMarkdown(content);
+
+  // Clean up whitespace
+  content = content
+    .replace(/\n{3,}/g, '\n\n')
+    .replace(/[ \t]+/g, ' ')
+    .trim();
+
+  // Truncate
+  if (content.length > maxLength) {
+    content = content.slice(0, maxLength) + '\n\n...[truncated]';
+  }
+
+  return {
+    title,
+    content,
+    url,
+    wordCount: content.split(/\s+/).filter(Boolean).length,
+  };
+}
+
+/**
+ * Try to find the main content area of the page.
+ */
+function findMainContent(html: string): string | null {
+  // Try common content selectors
+  const patterns = [
+    /<article[^>]*>([\s\S]*?)<\/article>/i,
+    /<main[^>]*>([\s\S]*?)<\/main>/i,
+    /<div[^>]*class="[^"]*(?:content|article|post|entry|main)[^"]*"[^>]*>([\s\S]*?)<\/div>/i,
+    /<div[^>]*id="[^"]*(?:content|article|post|entry|main)[^"]*"[^>]*>([\s\S]*?)<\/div>/i,
+  ];
+
+  for (const pattern of patterns) {
+    const match = html.match(pattern);
+    if (match && match[1] && match[1].length > 200) {
+      return match[1];
+    }
+  }
+
+  // Fallback: find body content
+  const bodyMatch = html.match(/<body[^>]*>([\s\S]*?)<\/body>/i);
+  return bodyMatch ? bodyMatch[1] : null;
+}
+
+/**
+ * Simple HTML to Markdown conversion.
+ */
+function htmlToMarkdown(html: string): string {
+  let md = html;
+
+  // Headers
+  md = md.replace(/<h1[^>]*>([\s\S]*?)<\/h1>/gi, '\n# $1\n');
+  md = md.replace(/<h2[^>]*>([\s\S]*?)<\/h2>/gi, '\n## $1\n');
+  md = md.replace(/<h3[^>]*>([\s\S]*?)<\/h3>/gi, '\n### $1\n');
+  md = md.replace(/<h4[^>]*>([\s\S]*?)<\/h4>/gi, '\n#### $1\n');
+  md = md.replace(/<h5[^>]*>([\s\S]*?)<\/h5>/gi, '\n##### $1\n');
+  md = md.replace(/<h6[^>]*>([\s\S]*?)<\/h6>/gi, '\n###### $1\n');
+
+  // Paragraphs and line breaks
+  md = md.replace(/<p[^>]*>/gi, '\n');
+  md = md.replace(/<\/p>/gi, '\n');
+  md = md.replace(/<br\s*\/?>/gi, '\n');
+
+  // Links
+  md = md.replace(/<a[^>]*href="([^"]*)"[^>]*>([\s\S]*?)<\/a>/gi, '[$2]($1)');
+
+  // Bold and italic
+  md = md.replace(/<(?:strong|b)[^>]*>([\s\S]*?)<\/(?:strong|b)>/gi, '**$1**');
+  md = md.replace(/<(?:em|i)[^>]*>([\s\S]*?)<\/(?:em|i)>/gi, '*$1*');
+
+  // Code
+  md = md.replace(/<code[^>]*>([\s\S]*?)<\/code>/gi, '`$1`');
+  md = md.replace(/<pre[^>]*>([\s\S]*?)<\/pre>/gi, '\n```\n$1\n```\n');
+
+  // Lists
+  md = md.replace(/<li[^>]*>([\s\S]*?)<\/li>/gi, '- $1\n');
+
+  // Blockquote
+  md = md.replace(/<blockquote[^>]*>([\s\S]*?)<\/blockquote>/gi, '\n> $1\n');
+
+  // Remove remaining HTML tags
+  md = md.replace(/<[^>]+>/g, '');
+
+  // Decode entities
+  md = decodeEntities(md);
+
+  return md;
+}
+
+function decodeEntities(text: string): string {
+  return text
+    .replace(/&amp;/g, '&')
+    .replace(/&lt;/g, '<')
+    .replace(/&gt;/g, '>')
+    .replace(/&quot;/g, '"')
+    .replace(/&#x27;/g, "'")
+    .replace(/&#39;/g, "'")
+    .replace(/&nbsp;/g, ' ')
+    .replace(/&#(\d+);/g, (_, n) => String.fromCharCode(parseInt(n)))
+    .replace(/&#x([0-9a-fA-F]+);/g, (_, n) => String.fromCharCode(parseInt(n, 16)));
+}

package/src/tools/web-search.ts

@@ -0,0 +1,180 @@
+/**
+ * Web Search Engine Manager - v0.10.0
+ * Supports multiple search backends with automatic fallback.
+ * Default: DuckDuckGo (free, no API key required).
+ */
+
+export interface SearchResult {
+  title: string;
+  url: string;
+  snippet: string;
+}
+
+export interface SearchOptions {
+  maxResults?: number;
+  engine?: SearchEngine;
+}
+
+export type SearchEngine = 'duckduckgo' | 'brave' | 'searxng' | 'google';
+
+export interface SearchEngineConfig {
+  enabled: boolean;
+  apiKey?: string;
+  baseUrl?: string;  // For SearXNG self-hosted
+}
+
+export interface WebSearchConfig {
+  defaultEngine: SearchEngine;
+  enabled: boolean;
+  engines: Partial<Record<SearchEngine, SearchEngineConfig>>;
+}
+
+export const DEFAULT_SEARCH_CONFIG: WebSearchConfig = {
+  defaultEngine: 'duckduckgo',
+  enabled: true,
+  engines: {
+    duckduckgo: { enabled: true },
+  },
+};
+
+/**
+ * Parse DuckDuckGo HTML search results.
+ */
+export function parseDuckDuckGoHTML(html: string): SearchResult[] {
+  const results: SearchResult[] = [];
+  // Match result blocks: <a class="result__a" href="...">title</a> ... <a class="result__snippet">snippet</a>
+  const resultBlocks = html.split(/class="result__body"/);
+
+  for (let i = 1; i < resultBlocks.length && results.length < 10; i++) {
+    const block = resultBlocks[i];
+
+    // Extract URL and title from result__a
+    const linkMatch = block.match(/class="result__a"[^>]*href="([^"]*)"[^>]*>([\s\S]*?)<\/a>/);
+    if (!linkMatch) continue;
+
+    let url = linkMatch[1];
+    const title = stripHTML(linkMatch[2]).trim();
+
+    // DuckDuckGo wraps URLs in redirect, extract actual URL
+    const uddgMatch = url.match(/[?&]uddg=([^&]+)/);
+    if (uddgMatch) {
+      url = decodeURIComponent(uddgMatch[1]);
+    }
+
+    // Extract snippet
+    const snippetMatch = block.match(/class="result__snippet"[^>]*>([\s\S]*?)<\/a>/);
+    const snippet = snippetMatch ? stripHTML(snippetMatch[1]).trim() : '';
+
+    if (title && url) {
+      results.push({ title, url, snippet });
+    }
+  }
+
+  return results;
+}
+
+/**
+ * Search using DuckDuckGo HTML interface (no API key needed).
+ */
+export async function searchDuckDuckGo(query: string, maxResults = 5): Promise<SearchResult[]> {
+  const url = `https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`;
+  const response = await fetch(url, {
+    headers: {
+      'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
+    },
+    signal: AbortSignal.timeout(15000),
+  });
+  const html = await response.text();
+  return parseDuckDuckGoHTML(html).slice(0, maxResults);
+}
+
+/**
+ * Search using Brave Search API.
+ */
+export async function searchBrave(query: string, apiKey: string, maxResults = 5): Promise<SearchResult[]> {
+  const url = `https://api.search.brave.com/res/v1/web/search?q=${encodeURIComponent(query)}&count=${maxResults}`;
+  const response = await fetch(url, {
+    headers: { 'X-Subscription-Token': apiKey, Accept: 'application/json' },
+    signal: AbortSignal.timeout(15000),
+  });
+  const data = await response.json() as any;
+  return (data.web?.results || []).slice(0, maxResults).map((r: any) => ({
+    title: r.title || '',
+    url: r.url || '',
+    snippet: r.description || '',
+  }));
+}
+
+/**
+ * Search using SearXNG instance.
+ */
+export async function searchSearXNG(query: string, baseUrl: string, maxResults = 5): Promise<SearchResult[]> {
+  const url = `${baseUrl.replace(/\/$/, '')}/search?q=${encodeURIComponent(query)}&format=json`;
+  const response = await fetch(url, { signal: AbortSignal.timeout(15000) });
+  const data = await response.json() as any;
+  return (data.results || []).slice(0, maxResults).map((r: any) => ({
+    title: r.title || '',
+    url: r.url || '',
+    snippet: r.content || '',
+  }));
+}
+
+/**
+ * Search using Google Custom Search API.
+ */
+export async function searchGoogle(query: string, apiKey: string, maxResults = 5): Promise<SearchResult[]> {
+  // apiKey format: "key:cx" (API key and Custom Search Engine ID)
+  const [key, cx] = apiKey.split(':');
+  const url = `https://www.googleapis.com/customsearch/v1?q=${encodeURIComponent(query)}&key=${key}&cx=${cx}&num=${maxResults}`;
+  const response = await fetch(url, { signal: AbortSignal.timeout(15000) });
+  const data = await response.json() as any;
+  return (data.items || []).slice(0, maxResults).map((r: any) => ({
+    title: r.title || '',
+    url: r.link || '',
+    snippet: r.snippet || '',
+  }));
+}
+
+/**
+ * Unified search function with fallback.
+ */
+export async function webSearch(query: string, config?: WebSearchConfig, options?: SearchOptions): Promise<SearchResult[]> {
+  const cfg = config || DEFAULT_SEARCH_CONFIG;
+  if (!cfg.enabled) return [];
+
+  const maxResults = options?.maxResults || 5;
+  const engine = options?.engine || cfg.defaultEngine;
+
+  // Try requested engine first, then fallback chain
+  const fallbackOrder: SearchEngine[] = [engine, 'duckduckgo', 'brave', 'searxng', 'google']
+    .filter((e, i, arr) => arr.indexOf(e) === i) as SearchEngine[];
+
+  for (const eng of fallbackOrder) {
+    const engCfg = cfg.engines[eng];
+    if (engCfg && !engCfg.enabled) continue;
+
+    try {
+      switch (eng) {
+        case 'duckduckgo':
+          return await searchDuckDuckGo(query, maxResults);
+        case 'brave':
+          if (engCfg?.apiKey) return await searchBrave(query, engCfg.apiKey, maxResults);
+          continue;
+        case 'searxng':
+          if (engCfg?.baseUrl) return await searchSearXNG(query, engCfg.baseUrl, maxResults);
+          continue;
+        case 'google':
+          if (engCfg?.apiKey) return await searchGoogle(query, engCfg.apiKey, maxResults);
+          continue;
+      }
+    } catch {
+      continue; // Fallback to next engine
+    }
+  }
+
+  return [];
+}
+
+function stripHTML(html: string): string {
+  return html.replace(/<[^>]+>/g, '').replace(/&amp;/g, '&').replace(/&lt;/g, '<').replace(/&gt;/g, '>').replace(/&quot;/g, '"').replace(/&#x27;/g, "'").replace(/&nbsp;/g, ' ');
+}

package/tests/cron-engine.test.ts

@@ -0,0 +1,101 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+import { CronEngine, frequencyToCron } from '../src/scheduler/cron-engine';
+import type { ScheduleTask } from '../src/scheduler/cron-engine';
+import { existsSync, unlinkSync } from 'fs';
+import { join } from 'path';
+import * as os from 'os';
+
+describe('CronEngine', () => {
+  let engine: CronEngine;
+  const schedulesPath = join(os.homedir(), '.opc', 'schedules.json');
+
+  beforeEach(() => {
+    engine = new CronEngine(async () => {});
+  });
+
+  afterEach(() => {
+    engine.stop();
+  });
+
+  describe('frequencyToCron', () => {
+    it('converts daily + time to cron', () => {
+      expect(frequencyToCron('daily', '08:30')).toBe('30 8 * * *');
+    });
+
+    it('converts weekly to Monday cron', () => {
+      expect(frequencyToCron('weekly', '09:00')).toBe('0 9 * * 1');
+    });
+
+    it('converts monthly to 1st of month', () => {
+      expect(frequencyToCron('monthly', '10:15')).toBe('15 10 1 * *');
+    });
+
+    it('defaults to 9:00 when no time given', () => {
+      expect(frequencyToCron('daily')).toBe('0 9 * * *');
+    });
+  });
+
+  describe('CRUD operations', () => {
+    it('creates a task', () => {
+      const task = engine.createTask({
+        name: 'Test Task',
+        schedule: '0 9 * * *',
+        description: 'Test description',
+        frequency: 'daily',
+        time: '09:00',
+        outputChannel: 'web',
+        enabled: true,
+      });
+      expect(task.id).toBeTruthy();
+      expect(task.name).toBe('Test Task');
+      expect(task.createdAt).toBeTruthy();
+    });
+
+    it('lists tasks', () => {
+      engine.createTask({ name: 'A', schedule: '0 9 * * *', description: '', frequency: 'daily', outputChannel: 'web', enabled: true });
+      engine.createTask({ name: 'B', schedule: '0 10 * * *', description: '', frequency: 'daily', outputChannel: 'telegram', enabled: false });
+      const tasks = engine.listTasks();
+      expect(tasks.length).toBeGreaterThanOrEqual(2);
+    });
+
+    it('updates a task', () => {
+      const task = engine.createTask({ name: 'Original', schedule: '0 9 * * *', description: '', frequency: 'daily', outputChannel: 'web', enabled: true });
+      const updated = engine.updateTask(task.id, { name: 'Updated', enabled: false });
+      expect(updated).not.toBeNull();
+      expect(updated!.name).toBe('Updated');
+      expect(updated!.enabled).toBe(false);
+    });
+
+    it('deletes a task', () => {
+      const task = engine.createTask({ name: 'ToDelete', schedule: '0 9 * * *', description: '', frequency: 'daily', outputChannel: 'web', enabled: true });
+      expect(engine.deleteTask(task.id)).toBe(true);
+      expect(engine.getTask(task.id)).toBeUndefined();
+    });
+
+    it('returns false for deleting non-existent task', () => {
+      expect(engine.deleteTask('nonexistent')).toBe(false);
+    });
+
+    it('returns null for updating non-existent task', () => {
+      expect(engine.updateTask('nonexistent', { name: 'x' })).toBeNull();
+    });
+  });
+
+  describe('run task', () => {
+    it('runs a task immediately', async () => {
+      const handler = vi.fn();
+      const eng = new CronEngine(handler);
+      const task = eng.createTask({ name: 'RunMe', schedule: '0 9 * * *', description: 'test', frequency: 'daily', outputChannel: 'web', enabled: true });
+      eng.start();
+      const result = await eng.runTask(task.id);
+      expect(result).toBe(true);
+      expect(handler).toHaveBeenCalled();
+      eng.stop();
+    });
+
+    it('returns false for running non-existent task', async () => {
+      const result = await engine.runTask('nonexistent');
+      expect(result).toBe(false);
+    });
+  });
+});

package/tests/document-processor.test.ts

@@ -0,0 +1,69 @@
+import { describe, it, expect } from 'vitest';
+import { DocumentProcessor } from '../src/tools/document-processor';
+
+describe('DocumentProcessor', () => {
+  const processor = new DocumentProcessor();
+
+  it('should process plain text', async () => {
+    const text = 'Hello world. This is a test document.\n\nSecond paragraph here.';
+    const buffer = Buffer.from(text, 'utf-8');
+    const doc = await processor.process(buffer, 'test.txt');
+
+    expect(doc.filename).toBe('test.txt');
+    expect(doc.format).toBe('txt');
+    expect(doc.chunks.length).toBeGreaterThan(0);
+    expect(doc.chunks[0].content).toContain('Hello world');
+    expect(doc.chunks[0].metadata.source).toBe('test.txt');
+  });
+
+  it('should process markdown with headings', async () => {
+    const md = `# Title\n\nFirst section content.\n\n## Section Two\n\nSecond section content.\n\n## Section Three\n\nThird section.`;
+    const buffer = Buffer.from(md, 'utf-8');
+    const doc = await processor.process(buffer, 'test.md');
+
+    expect(doc.format).toBe('md');
+    expect(doc.chunks.length).toBeGreaterThan(0);
+  });
+
+  it('should process CSV', async () => {
+    const csv = 'Name,Age,City\nAlice,30,Beijing\nBob,25,Shanghai';
+    const buffer = Buffer.from(csv, 'utf-8');
+    const doc = await processor.process(buffer, 'data.csv');
+
+    expect(doc.format).toBe('csv');
+    expect(doc.chunks.length).toBeGreaterThan(0);
+    expect(doc.chunks[0].content).toContain('Alice');
+  });
+
+  it('should process JSON array', async () => {
+    const json = JSON.stringify([{ name: 'Alice', role: 'engineer' }, { name: 'Bob', role: 'designer' }]);
+    const buffer = Buffer.from(json, 'utf-8');
+    const doc = await processor.process(buffer, 'data.json');
+
+    expect(doc.format).toBe('json');
+    expect(doc.chunks[0].content).toContain('Alice');
+  });
+
+  it('should chunk large text properly', async () => {
+    const bigText = Array(200).fill('Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.').join('\n\n');
+    const buffer = Buffer.from(bigText, 'utf-8');
+    const doc = await processor.process(buffer, 'big.txt');
+
+    expect(doc.chunks.length).toBeGreaterThan(1);
+    for (const chunk of doc.chunks) {
+      expect(chunk.content.length).toBeLessThanOrEqual(5000); // some tolerance
+      expect(chunk.metadata.totalChunks).toBe(doc.chunks.length);
+    }
+  });
+
+  it('should reject files over 50MB', async () => {
+    const bigBuffer = Buffer.alloc(51 * 1024 * 1024);
+    await expect(processor.process(bigBuffer, 'huge.txt')).rejects.toThrow('too large');
+  });
+
+  it('should handle empty content', async () => {
+    const buffer = Buffer.from('', 'utf-8');
+    const doc = await processor.process(buffer, 'empty.txt');
+    expect(doc.chunks.length).toBe(0);
+  });
+});