opc-agent 3.0.0 → 4.0.0

package/src/core/vision.ts

@@ -0,0 +1,180 @@
+import * as fs from 'fs';
+import * as path from 'path';
+
+// ─── Types ───────────────────────────────────────────────────
+
+export interface ImageInput {
+  type: 'base64' | 'url' | 'file';
+  data: string;
+  mimeType?: string;
+}
+
+export interface VisionResult {
+  description: string;
+  text_content?: string;
+  objects?: string[];
+}
+
+// ─── MIME detection from magic bytes ─────────────────────────
+
+const MAGIC_BYTES: Array<{ bytes: number[]; mime: string }> = [
+  { bytes: [0x89, 0x50, 0x4E, 0x47], mime: 'image/png' },
+  { bytes: [0xFF, 0xD8, 0xFF], mime: 'image/jpeg' },
+  { bytes: [0x47, 0x49, 0x46, 0x38], mime: 'image/gif' },
+  { bytes: [0x52, 0x49, 0x46, 0x46], mime: 'image/webp' }, // RIFF header (WebP)
+];
+
+export function detectMimeType(buffer: Buffer): string {
+  for (const { bytes, mime } of MAGIC_BYTES) {
+    if (bytes.every((b, i) => buffer[i] === b)) {
+      return mime;
+    }
+  }
+  return 'application/octet-stream';
+}
+
+// ─── VisionManager ───────────────────────────────────────────
+
+export interface VisionManagerConfig {
+  model?: string;
+  apiKey?: string;
+  baseURL?: string;
+  maxImageSize?: number; // bytes, default 20MB
+}
+
+export class VisionManager {
+  private config: VisionManagerConfig;
+
+  constructor(config: VisionManagerConfig = {}) {
+    this.config = {
+      model: config.model ?? 'gpt-4o',
+      apiKey: config.apiKey ?? process.env.OPENAI_API_KEY ?? '',
+      baseURL: config.baseURL ?? 'https://api.openai.com/v1',
+      maxImageSize: config.maxImageSize ?? 20 * 1024 * 1024,
+    };
+  }
+
+  /**
+   * Load image data as base64, detecting MIME type.
+   */
+  async loadImage(input: ImageInput): Promise<{ base64: string; mimeType: string }> {
+    if (input.type === 'base64') {
+      const buf = Buffer.from(input.data, 'base64');
+      return { base64: input.data, mimeType: input.mimeType ?? detectMimeType(buf) };
+    }
+    if (input.type === 'file') {
+      const buf = fs.readFileSync(input.data);
+      if (buf.length > this.config.maxImageSize!) {
+        throw new Error(`Image exceeds max size: ${buf.length} > ${this.config.maxImageSize}`);
+      }
+      return { base64: buf.toString('base64'), mimeType: input.mimeType ?? detectMimeType(buf) };
+    }
+    // URL — return as-is for API
+    return { base64: '', mimeType: input.mimeType ?? 'image/jpeg' };
+  }
+
+  /**
+   * Prepare messages in OpenAI multimodal format.
+   */
+  prepareMessage(images: ImageInput[], text: string): Array<Record<string, unknown>> {
+    const content: Array<Record<string, unknown>> = [];
+    if (text) {
+      content.push({ type: 'text', text });
+    }
+    for (const img of images) {
+      if (img.type === 'url') {
+        content.push({
+          type: 'image_url',
+          image_url: { url: img.data },
+        });
+      } else {
+        // base64 or file — will need to be loaded first
+        const mime = img.mimeType ?? 'image/jpeg';
+        const data = img.type === 'base64' ? img.data : fs.readFileSync(img.data).toString('base64');
+        content.push({
+          type: 'image_url',
+          image_url: { url: `data:${mime};base64,${data}` },
+        });
+      }
+    }
+    return [{ role: 'user', content }];
+  }
+
+  /**
+   * Analyze an image with optional prompt.
+   */
+  async analyze(image: ImageInput, prompt?: string): Promise<VisionResult> {
+    const messages = this.prepareMessage([image], prompt ?? 'Describe this image in detail.');
+
+    if (!this.config.apiKey) {
+      throw new Error('Vision API key not configured. Set OPENAI_API_KEY or pass apiKey in config.');
+    }
+
+    const response = await fetch(`${this.config.baseURL}/chat/completions`, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        'Authorization': `Bearer ${this.config.apiKey}`,
+      },
+      body: JSON.stringify({
+        model: this.config.model,
+        messages,
+        max_tokens: 1000,
+      }),
+    });
+
+    if (!response.ok) {
+      throw new Error(`Vision API error: ${response.status} ${response.statusText}`);
+    }
+
+    const data = await response.json() as any;
+    const text = data.choices?.[0]?.message?.content ?? '';
+
+    return {
+      description: text,
+      text_content: undefined,
+      objects: [],
+    };
+  }
+
+  /**
+   * Extract text (OCR) from an image.
+   */
+  async extractText(image: ImageInput): Promise<string> {
+    const result = await this.analyze(image, 'Extract all visible text from this image. Return only the text, nothing else.');
+    return result.description;
+  }
+
+  /**
+   * Compare multiple images.
+   */
+  async compareImages(images: ImageInput[], prompt?: string): Promise<string> {
+    if (images.length < 2) throw new Error('Need at least 2 images to compare');
+
+    const messages = this.prepareMessage(images, prompt ?? 'Compare these images and describe the differences.');
+
+    if (!this.config.apiKey) {
+      throw new Error('Vision API key not configured.');
+    }
+
+    const response = await fetch(`${this.config.baseURL}/chat/completions`, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        'Authorization': `Bearer ${this.config.apiKey}`,
+      },
+      body: JSON.stringify({
+        model: this.config.model,
+        messages,
+        max_tokens: 1000,
+      }),
+    });
+
+    if (!response.ok) {
+      throw new Error(`Vision API error: ${response.status} ${response.statusText}`);
+    }
+
+    const data = await response.json() as any;
+    return data.choices?.[0]?.message?.content ?? '';
+  }
+}

package/src/index.ts

@@ -78,6 +78,26 @@ export { TextAnalysisTool } from './tools/text-analysis';
 export { HttpSkill } from './skills/http';
 export { WebhookTriggerSkill } from './skills/webhook-trigger';
 export type { WebhookTarget } from './skills/webhook-trigger';
+
+// v0.9.0 channels
+export { DingTalkChannel } from './channels/dingtalk';
+export type { DingTalkChannelConfig } from './channels/dingtalk';
+export { MattermostChannel } from './channels/mattermost';
+export type { MattermostChannelConfig } from './channels/mattermost';
+export { GoogleChatChannel } from './channels/googlechat';
+export type { GoogleChatChannelConfig } from './channels/googlechat';
+export { TwitchChannel } from './channels/twitch';
+export type { TwitchChannelConfig } from './channels/twitch';
+export { IRCChannel } from './channels/irc';
+export type { IRCChannelConfig } from './channels/irc';
+
+// v0.9.0 core
+export { ContextDiscovery } from './core/context-discovery';
+export type { ContextFile } from './core/context-discovery';
+export { SessionManager } from './core/session-manager';
+export type { Session } from './core/session-manager';
+export { HeartbeatManager } from './core/heartbeat';
+export type { HeartbeatConfig } from './core/heartbeat';
 export { SchedulerSkill } from './skills/scheduler';
 export type { ScheduledTask } from './skills/scheduler';
 export { DocumentSkill } from './skills/document';
@@ -111,6 +131,20 @@ export { contentFilterPlugin, createContentFilterPlugin } from './plugins/conten
 export { GuardrailManager, createGuardrailsFromConfig } from './security/guardrails';
 export type { GuardrailConfig, GuardrailRule, GuardrailResult, GuardrailViolation } from './security/guardrails';
 
+// v3.1.0 — Exec Approvals + Elevated + Secrets
+export { ExecApprovalManager } from './security/approvals';
+export type { ExecApprovalPolicy, ExecApprovalRequest, ExecApprovalHistory, ApprovalRequestCallback } from './security/approvals';
+export { ElevatedManager } from './security/elevated';
+export type { ElevationMode, ElevationAuditEntry } from './security/elevated';
+export { SecretsManager } from './security/secrets';
+export type { SecretsStore } from './security/secrets';
+
+// v3.1.0 — Hooks + Audio
+export { HookManager, ALL_HOOK_EVENTS } from './core/hooks';
+export type { HookEvent, HookContext, HookHandler } from './core/hooks';
+export { AudioProcessor } from './core/audio';
+export type { AudioFormat, TranscribeOptions, SynthesizeOptions, TranscribeResult, SynthesizeResult } from './core/audio';
+
 // v1.1.0 modules
 export { FeishuChannel } from './channels/feishu';
 export type { FeishuChannelConfig } from './channels/feishu';
@@ -142,7 +176,7 @@ export { Scheduler, parseCron, cronMatches } from './core/scheduler';
 export type { CronJob, JobHandler } from './core/scheduler';
 
 // v1.5.0 — built-in tools + MCP client
-export { getBuiltinTools, getBuiltinToolsByName } from './tools/builtin';
+export { getBuiltinTools, getBuiltinToolsByName, rlTools, homeAssistantTools, configureHomeAssistant } from './tools/builtin';
 export { MCPClient } from './tools/mcp-client';
 export type { MCPServerConfig } from './tools/mcp-client';
 
@@ -186,6 +220,55 @@ export type {
   StepStartedEvent, StepFinishedEvent, CustomEvent,
 } from './protocols/agui';
 
+// v2.1.0 — API Server + Context References
+export { APIServer } from './core/api-server';
+export type { APIServerConfig } from './core/api-server';
+export { ContextRefResolver } from './core/context-refs';
+export type { ContextRef, RefType } from './core/context-refs';
+
+// v2.2.0 — Vision
+export { VisionManager, detectMimeType } from './core/vision';
+export type { ImageInput, VisionResult, VisionManagerConfig } from './core/vision';
+export { visionTools, visionAnalyzeTool, visionExtractTextTool, visionCompareTool } from './tools/builtin/vision';
+
+// v2.2.0 — Additional channels
+export { WhatsAppChannel } from './channels/whatsapp';
+export type { WhatsAppChannelConfig } from './channels/whatsapp';
+export { SignalChannel } from './channels/signal';
+export type { SignalChannelConfig } from './channels/signal';
+export { MatrixChannel } from './channels/matrix';
+export type { MatrixChannelConfig } from './channels/matrix';
+export { IMessageChannel } from './channels/imessage';
+export type { IMessageChannelConfig } from './channels/imessage';
+export { LINEChannel } from './channels/line';
+export type { LINEChannelConfig } from './channels/line';
+export { MSTeamsChannel } from './channels/msteams';
+export type { MSTeamsChannelConfig } from './channels/msteams';
+export { QQChannel } from './channels/qq';
+export type { QQChannelConfig } from './channels/qq';
+export { NostrChannel } from './channels/nostr';
+export type { NostrChannelConfig } from './channels/nostr';
+export { SMSChannel } from './channels/sms';
+export type { SMSChannelConfig } from './channels/sms';
+
+// v2.3.0 — Voice Calls + IDE + Node Network + Gateway
+export { VoiceCallManager } from './channels/voice-call';
+export type { VoiceCallConfig } from './channels/voice-call';
+export { IDEBridge } from './core/ide-bridge';
+export type { IDEConfig, Diagnostic, TextEdit, Range, SearchOptions, SearchResult } from './core/ide-bridge';
+export { NodeNetwork } from './core/node-network';
+export type { RemoteNode } from './core/node-network';
+export { Gateway } from './core/gateway';
+export type { AgentConfig as GatewayAgentConfig, ChannelConfig as GatewayChannelConfig, GatewayConfig } from './core/gateway';
+
+// v2.2.0 — Remote Sandbox
+export { SandboxManager } from './core/sandbox';
+export type { RemoteSandboxConfig, ExecResult } from './core/sandbox';
+
+// v2.2.0 — Profiles
+export { ProfileManager } from './core/profiles';
+export type { Profile, ProfileConfig } from './core/profiles';
+
 // v2.0.0 - Pre-built tool integrations (20 tools)
 export {
   SlackTool, EmailSendTool, WebhookTool,

package/src/memory/context-compressor.ts

@@ -0,0 +1,189 @@
+import type { Message } from '../core/types';
+
+export interface CompressorConfig {
+  maxTokens: number;
+  compressThreshold: number;
+  preserveRecent: number;
+  brain?: any;
+}
+
+export interface CompressResult {
+  messages: Message[];
+  learnedCount: number;
+  savedTokens: number;
+  summary: string;
+}
+
+const DEFAULT_CONFIG: CompressorConfig = {
+  maxTokens: 8000,
+  compressThreshold: 0.8,
+  preserveRecent: 10,
+};
+
+/**
+ * Context compression with optional DeepBrain memory offloading.
+ */
+export class ContextCompressor {
+  private config: CompressorConfig;
+
+  constructor(config: Partial<CompressorConfig> = {}) {
+    this.config = { ...DEFAULT_CONFIG, ...config };
+  }
+
+  /**
+   * Estimate token count using language-aware heuristic.
+   * English: ~1 token per 4 chars. Chinese: ~1 token per 2 chars.
+   */
+  estimateTokens(text: string): number {
+    let tokens = 0;
+    for (const char of text) {
+      // CJK Unicode range detection
+      const code = char.codePointAt(0) ?? 0;
+      if (
+        (code >= 0x4e00 && code <= 0x9fff) ||  // CJK Unified
+        (code >= 0x3400 && code <= 0x4dbf) ||  // CJK Extension A
+        (code >= 0x3000 && code <= 0x303f)     // CJK Punctuation
+      ) {
+        tokens += 0.5; // 1 token per 2 chars
+      } else {
+        tokens += 0.25; // 1 token per 4 chars
+      }
+    }
+    return Math.ceil(tokens);
+  }
+
+  private estimateMessagesTokens(messages: Message[]): number {
+    return messages.reduce((sum, m) => sum + this.estimateTokens(m.content), 0);
+  }
+
+  /**
+   * Extract key insights from messages for brain storage.
+   */
+  private extractInsights(messages: Message[]): Array<{ content: string; type: string }> {
+    const insights: Array<{ content: string; type: string }> = [];
+    for (const msg of messages) {
+      const c = msg.content;
+      // Decisions
+      if (/\b(decided|decision|choose|chose|will use|going with|let's go|确定|决定)\b/i.test(c)) {
+        insights.push({ content: c.slice(0, 500), type: 'decision' });
+      }
+      // Facts / definitions
+      else if (/\b(is defined as|means|equals|refers to|是指|定义)\b/i.test(c)) {
+        insights.push({ content: c.slice(0, 500), type: 'fact' });
+      }
+      // Preferences
+      else if (/\b(prefer|like|want|don't like|不喜欢|喜欢|偏好)\b/i.test(c)) {
+        insights.push({ content: c.slice(0, 500), type: 'preference' });
+      }
+      // Code snippets
+      else if (/```[\s\S]{20,}```/.test(c)) {
+        insights.push({ content: c.slice(0, 800), type: 'code' });
+      }
+      // Long assistant messages likely contain useful info
+      else if (msg.role === 'assistant' && c.length > 200) {
+        insights.push({ content: c.slice(0, 500), type: 'knowledge' });
+      }
+    }
+    return insights;
+  }
+
+  /**
+   * Generate a simple summary from messages (no-brain fallback).
+   */
+  private summarize(messages: Message[]): string {
+    const topics = new Set<string>();
+    const keyLines: string[] = [];
+
+    for (const msg of messages) {
+      // Extract first meaningful sentence
+      const firstLine = msg.content.split(/[.\n!?。！？]/)[0]?.trim();
+      if (firstLine && firstLine.length > 10 && firstLine.length < 200) {
+        if (keyLines.length < 5) keyLines.push(`[${msg.role}] ${firstLine}`);
+      }
+      // Extract topic words (capitalized words, Chinese phrases)
+      const words = msg.content.match(/[A-Z][a-z]{2,}/g) ?? [];
+      words.forEach(w => topics.add(w));
+    }
+
+    const topicStr = [...topics].slice(0, 10).join(', ');
+    const linesStr = keyLines.join('; ');
+    return `Topics: ${topicStr || 'general discussion'}. Key points: ${linesStr || 'varied conversation'}`;
+  }
+
+  /**
+   * Compress messages when token count exceeds threshold.
+   */
+  async compress(messages: Message[], config?: Partial<CompressorConfig>): Promise<CompressResult> {
+    const cfg = { ...this.config, ...config };
+    const totalTokens = this.estimateMessagesTokens(messages);
+    const threshold = cfg.maxTokens * cfg.compressThreshold;
+
+    // Under threshold — return as-is
+    if (totalTokens <= threshold) {
+      return {
+        messages: [...messages],
+        learnedCount: 0,
+        savedTokens: 0,
+        summary: '',
+      };
+    }
+
+    const recentCount = Math.min(cfg.preserveRecent, messages.length);
+    const splitIdx = messages.length - recentCount;
+    const oldMessages = messages.slice(0, splitIdx);
+    const recentMessages = messages.slice(splitIdx);
+
+    if (oldMessages.length === 0) {
+      return { messages: [...messages], learnedCount: 0, savedTokens: 0, summary: '' };
+    }
+
+    const oldTokens = this.estimateMessagesTokens(oldMessages);
+    let learnedCount = 0;
+    let summary: string;
+
+    if (cfg.brain) {
+      // Extract and learn insights
+      const insights = this.extractInsights(oldMessages);
+      for (const insight of insights) {
+        try {
+          await cfg.brain.learn(insight.content, { insight_type: insight.type });
+          learnedCount++;
+        } catch { /* non-critical */ }
+      }
+      summary = `${oldMessages.length} messages compressed. Extracted ${learnedCount} insights (${insights.map(i => i.type).filter((v, i, a) => a.indexOf(v) === i).join(', ')}).`;
+    } else {
+      summary = this.summarize(oldMessages);
+    }
+
+    const compressionMessage: Message = {
+      id: `compressed-${Date.now()}`,
+      role: 'system',
+      content: `[Context compressed: ${oldMessages.length} messages → ${summary}${cfg.brain ? ' Details stored in Brain, use recall() to retrieve.' : ''}]`,
+      timestamp: Date.now(),
+    };
+
+    return {
+      messages: [compressionMessage, ...recentMessages],
+      learnedCount,
+      savedTokens: oldTokens - this.estimateTokens(compressionMessage.content),
+      summary,
+    };
+  }
+
+  /**
+   * Restore context from brain for a given query.
+   */
+  async restore(query: string, brain: any): Promise<string[]> {
+    if (!brain?.recall) return [];
+    try {
+      const results = await brain.recall(query);
+      if (Array.isArray(results)) {
+        return results.map((r: any) => typeof r === 'string' ? r : r.content ?? JSON.stringify(r));
+      }
+      if (typeof results === 'string') return [results];
+      return [];
+    } catch {
+      return [];
+    }
+  }
+}

package/src/memory/index.ts

@@ -2,6 +2,10 @@ import type { Message, MemoryStore } from '../core/types';
 
 export { BrainSeedLoader, KnowledgeEvolver } from './seed-loader';
 export type { BrainSeedConfig, SeedPage, SeedResult, PromotionResult, PromotionCandidate } from './seed-loader';
+export { ContextCompressor } from './context-compressor';
+export type { CompressorConfig, CompressResult } from './context-compressor';
+export { UserProfiler } from './user-profiler';
+export type { UserProfile } from './user-profiler';
 
 export class InMemoryStore implements MemoryStore {
   private store: Map<string, unknown> = new Map();