@push.rocks/smartai 0.6.1 → 0.7.0

package/ts/abstract.classes.multimodal.ts

@@ -50,6 +50,60 @@ export interface ResearchResponse {
   metadata?: any;
 }
 
+/**
+ * Options for image generation
+ */
+export interface ImageGenerateOptions {
+  prompt: string;
+  model?: 'gpt-image-1' | 'dall-e-3' | 'dall-e-2';
+  quality?: 'low' | 'medium' | 'high' | 'standard' | 'hd' | 'auto';
+  size?: '256x256' | '512x512' | '1024x1024' | '1536x1024' | '1024x1536' | '1792x1024' | '1024x1792' | 'auto';
+  style?: 'vivid' | 'natural';
+  background?: 'transparent' | 'opaque' | 'auto';
+  outputFormat?: 'png' | 'jpeg' | 'webp';
+  outputCompression?: number; // 0-100 for webp/jpeg
+  moderation?: 'low' | 'auto';
+  n?: number; // Number of images to generate
+  stream?: boolean;
+  partialImages?: number; // 0-3 for streaming
+}
+
+/**
+ * Options for image editing
+ */
+export interface ImageEditOptions {
+  image: Buffer;
+  prompt: string;
+  mask?: Buffer;
+  model?: 'gpt-image-1' | 'dall-e-2';
+  quality?: 'low' | 'medium' | 'high' | 'standard' | 'auto';
+  size?: '256x256' | '512x512' | '1024x1024' | '1536x1024' | '1024x1536' | 'auto';
+  background?: 'transparent' | 'opaque' | 'auto';
+  outputFormat?: 'png' | 'jpeg' | 'webp';
+  outputCompression?: number;
+  n?: number;
+  stream?: boolean;
+  partialImages?: number;
+}
+
+/**
+ * Response format for image operations
+ */
+export interface ImageResponse {
+  images: Array<{
+    b64_json?: string;
+    url?: string;
+    revisedPrompt?: string;
+  }>;
+  metadata?: {
+    model: string;
+    quality?: string;
+    size?: string;
+    outputFormat?: string;
+    tokensUsed?: number;
+  };
+}
+
 /**
  * Abstract base class for multi-modal AI models.
  * Provides a common interface for different AI providers (OpenAI, Anthropic, Perplexity, Ollama)
@@ -131,4 +185,20 @@ export abstract class MultiModalModel {
    * @throws Error if the provider doesn't support research capabilities
    */
   public abstract research(optionsArg: ResearchOptions): Promise<ResearchResponse>;
+
+  /**
+   * Image generation from text prompts
+   * @param optionsArg Options containing the prompt and generation parameters
+   * @returns Promise resolving to the generated image(s)
+   * @throws Error if the provider doesn't support image generation
+   */
+  public abstract imageGenerate(optionsArg: ImageGenerateOptions): Promise<ImageResponse>;
+
+  /**
+   * Image editing and inpainting
+   * @param optionsArg Options containing the image, prompt, and editing parameters
+   * @returns Promise resolving to the edited image(s)
+   * @throws Error if the provider doesn't support image editing
+   */
+  public abstract imageEdit(optionsArg: ImageEditOptions): Promise<ImageResponse>;
 }

package/ts/provider.anthropic.ts

@@ -1,7 +1,16 @@
 import * as plugins from './plugins.js';
 import * as paths from './paths.js';
 import { MultiModalModel } from './abstract.classes.multimodal.js';
-import type { ChatOptions, ChatResponse, ChatMessage, ResearchOptions, ResearchResponse } from './abstract.classes.multimodal.js';
+import type {
+  ChatOptions,
+  ChatResponse,
+  ChatMessage,
+  ResearchOptions,
+  ResearchResponse,
+  ImageGenerateOptions,
+  ImageEditOptions,
+  ImageResponse
+} from './abstract.classes.multimodal.js';
 import type { ImageBlockParam, TextBlockParam } from '@anthropic-ai/sdk/resources/messages';
 
 type ContentBlock = ImageBlockParam | TextBlockParam;
@@ -68,7 +77,7 @@ export class AnthropicProvider extends MultiModalModel {
         // If we have a complete message, send it to Anthropic
         if (currentMessage) {
           const stream = await this.anthropicApiClient.messages.create({
-            model: 'claude-3-opus-20240229',
+            model: 'claude-sonnet-4-5-20250929',
             messages: [{ role: currentMessage.role, content: currentMessage.content }],
             system: '',
             stream: true,
@@ -112,7 +121,7 @@ export class AnthropicProvider extends MultiModalModel {
     }));
 
     const result = await this.anthropicApiClient.messages.create({
-      model: 'claude-3-opus-20240229',
+      model: 'claude-sonnet-4-5-20250929',
       system: optionsArg.systemMessage,
       messages: [
         ...messages,
@@ -159,7 +168,7 @@ export class AnthropicProvider extends MultiModalModel {
     ];
 
     const result = await this.anthropicApiClient.messages.create({
-      model: 'claude-3-opus-20240229',
+      model: 'claude-sonnet-4-5-20250929',
       messages: [{
         role: 'user',
         content
@@ -218,7 +227,7 @@ export class AnthropicProvider extends MultiModalModel {
     }
 
     const result = await this.anthropicApiClient.messages.create({
-      model: 'claude-3-opus-20240229',
+      model: 'claude-sonnet-4-5-20250929',
       system: optionsArg.systemMessage,
       messages: [
         ...messages,
@@ -251,23 +260,27 @@ export class AnthropicProvider extends MultiModalModel {
 
     try {
       // Build the tool configuration for web search
-      const tools = this.options.enableWebSearch ? [
-        {
-          type: 'web_search_20250305' as const,
-          name: 'web_search',
-          description: 'Search the web for current information',
-          input_schema: {
-            type: 'object' as const,
-            properties: {
-              query: {
-                type: 'string',
-                description: 'The search query'
-              }
-            },
-            required: ['query']
-          }
+      const tools: any[] = [];
+
+      if (this.options.enableWebSearch) {
+        const webSearchTool: any = {
+          type: 'web_search_20250305',
+          name: 'web_search'
+        };
+
+        // Add optional parameters
+        if (optionsArg.maxSources) {
+          webSearchTool.max_uses = optionsArg.maxSources;
         }
-      ] : [];
+
+        if (this.options.searchDomainAllowList?.length) {
+          webSearchTool.allowed_domains = this.options.searchDomainAllowList;
+        } else if (this.options.searchDomainBlockList?.length) {
+          webSearchTool.blocked_domains = this.options.searchDomainBlockList;
+        }
+
+        tools.push(webSearchTool);
+      }
 
       // Configure the request based on search depth
       const maxTokens = optionsArg.searchDepth === 'deep' ? 8192 :
@@ -275,7 +288,7 @@ export class AnthropicProvider extends MultiModalModel {
 
       // Create the research request
       const requestParams: any = {
-        model: 'claude-3-opus-20240229',
+        model: 'claude-sonnet-4-5-20250929',
         system: systemMessage,
         messages: [
           {
@@ -290,7 +303,6 @@ export class AnthropicProvider extends MultiModalModel {
       // Add tools if web search is enabled
       if (tools.length > 0) {
         requestParams.tools = tools;
-        requestParams.tool_choice = { type: 'auto' };
       }
 
       // Execute the research request
@@ -304,54 +316,71 @@ export class AnthropicProvider extends MultiModalModel {
       // Process content blocks
       for (const block of result.content) {
         if ('text' in block) {
+          // Accumulate text content
           answer += block.text;
-        }
-      }
 
-      // Parse sources from the answer (Claude includes citations in various formats)
-      const urlRegex = /\[([^\]]+)\]\(([^)]+)\)/g;
-      let match: RegExpExecArray | null;
-
-      while ((match = urlRegex.exec(answer)) !== null) {
-        sources.push({
-          title: match[1],
-          url: match[2],
-          snippet: ''
-        });
+          // Extract citations if present
+          if ('citations' in block && Array.isArray(block.citations)) {
+            for (const citation of block.citations) {
+              if (citation.type === 'web_search_result_location') {
+                sources.push({
+                  title: citation.title || '',
+                  url: citation.url || '',
+                  snippet: citation.cited_text || ''
+                });
+              }
+            }
+          }
+        } else if ('type' in block && block.type === 'server_tool_use') {
+          // Extract search queries from server tool use
+          if (block.name === 'web_search' && block.input && typeof block.input === 'object' && 'query' in block.input) {
+            searchQueries.push((block.input as any).query);
+          }
+        } else if ('type' in block && block.type === 'web_search_tool_result') {
+          // Extract sources from web search results
+          if (Array.isArray(block.content)) {
+            for (const result of block.content) {
+              if (result.type === 'web_search_result') {
+                // Only add if not already in sources (avoid duplicates from citations)
+                if (!sources.some(s => s.url === result.url)) {
+                  sources.push({
+                    title: result.title || '',
+                    url: result.url || '',
+                    snippet: '' // Search results don't include snippets, only citations do
+                  });
+                }
+              }
+            }
+          }
+        }
       }
 
-      // Also look for plain URLs
-      const plainUrlRegex = /https?:\/\/[^\s\)]+/g;
-      const plainUrls = answer.match(plainUrlRegex) || [];
+      // Fallback: Parse markdown-style links if no citations found
+      if (sources.length === 0) {
+        const urlRegex = /\[([^\]]+)\]\(([^)]+)\)/g;
+        let match: RegExpExecArray | null;
 
-      for (const url of plainUrls) {
-        // Check if this URL is already in sources
-        if (!sources.some(s => s.url === url)) {
+        while ((match = urlRegex.exec(answer)) !== null) {
           sources.push({
-            title: new URL(url).hostname,
-            url: url,
+            title: match[1],
+            url: match[2],
             snippet: ''
           });
         }
       }
 
-      // Extract tool use information if available
-      if ('tool_use' in result && Array.isArray(result.tool_use)) {
-        for (const toolUse of result.tool_use) {
-          if (toolUse.name === 'web_search' && toolUse.input?.query) {
-            searchQueries.push(toolUse.input.query);
-          }
-        }
-      }
+      // Check if web search was used based on usage info
+      const webSearchCount = result.usage?.server_tool_use?.web_search_requests || 0;
 
       return {
         answer,
         sources,
         searchQueries: searchQueries.length > 0 ? searchQueries : undefined,
         metadata: {
-          model: 'claude-3-opus-20240229',
+          model: 'claude-sonnet-4-5-20250929',
           searchDepth: optionsArg.searchDepth || 'basic',
-          tokensUsed: result.usage?.output_tokens
+          tokensUsed: result.usage?.output_tokens,
+          webSearchesPerformed: webSearchCount
         }
       };
     } catch (error) {
@@ -359,4 +388,18 @@ export class AnthropicProvider extends MultiModalModel {
       throw new Error(`Failed to perform research: ${error.message}`);
     }
   }
+
+  /**
+   * Image generation is not supported by Anthropic
+   */
+  public async imageGenerate(optionsArg: ImageGenerateOptions): Promise<ImageResponse> {
+    throw new Error('Image generation is not supported by Anthropic. Claude can only analyze images, not generate them. Please use OpenAI provider for image generation.');
+  }
+
+  /**
+   * Image editing is not supported by Anthropic
+   */
+  public async imageEdit(optionsArg: ImageEditOptions): Promise<ImageResponse> {
+    throw new Error('Image editing is not supported by Anthropic. Claude can only analyze images, not edit them. Please use OpenAI provider for image editing.');
+  }
 }

package/ts/provider.exo.ts

@@ -1,7 +1,16 @@
 import * as plugins from './plugins.js';
 import * as paths from './paths.js';
 import { MultiModalModel } from './abstract.classes.multimodal.js';
-import type { ChatOptions, ChatResponse, ChatMessage, ResearchOptions, ResearchResponse } from './abstract.classes.multimodal.js';
+import type {
+  ChatOptions,
+  ChatResponse,
+  ChatMessage,
+  ResearchOptions,
+  ResearchResponse,
+  ImageGenerateOptions,
+  ImageEditOptions,
+  ImageResponse
+} from './abstract.classes.multimodal.js';
 import type { ChatCompletionMessageParam } from 'openai/resources/chat/completions';
 
 export interface IExoProviderOptions {
@@ -129,4 +138,18 @@ export class ExoProvider extends MultiModalModel {
   public async research(optionsArg: ResearchOptions): Promise<ResearchResponse> {
     throw new Error('Research capabilities are not yet supported by Exo provider.');
   }
+
+  /**
+   * Image generation is not supported by Exo
+   */
+  public async imageGenerate(optionsArg: ImageGenerateOptions): Promise<ImageResponse> {
+    throw new Error('Image generation is not supported by Exo. Please use OpenAI provider for image generation.');
+  }
+
+  /**
+   * Image editing is not supported by Exo
+   */
+  public async imageEdit(optionsArg: ImageEditOptions): Promise<ImageResponse> {
+    throw new Error('Image editing is not supported by Exo. Please use OpenAI provider for image editing.');
+  }
 }

package/ts/provider.groq.ts

@@ -1,7 +1,16 @@
 import * as plugins from './plugins.js';
 import * as paths from './paths.js';
 import { MultiModalModel } from './abstract.classes.multimodal.js';
-import type { ChatOptions, ChatResponse, ChatMessage, ResearchOptions, ResearchResponse } from './abstract.classes.multimodal.js';
+import type {
+  ChatOptions,
+  ChatResponse,
+  ChatMessage,
+  ResearchOptions,
+  ResearchResponse,
+  ImageGenerateOptions,
+  ImageEditOptions,
+  ImageResponse
+} from './abstract.classes.multimodal.js';
 
 export interface IGroqProviderOptions {
   groqToken: string;
@@ -193,4 +202,18 @@ export class GroqProvider extends MultiModalModel {
   public async research(optionsArg: ResearchOptions): Promise<ResearchResponse> {
     throw new Error('Research capabilities are not yet supported by Groq provider.');
   }
+
+  /**
+   * Image generation is not supported by Groq
+   */
+  public async imageGenerate(optionsArg: ImageGenerateOptions): Promise<ImageResponse> {
+    throw new Error('Image generation is not supported by Groq. Please use OpenAI provider for image generation.');
+  }
+
+  /**
+   * Image editing is not supported by Groq
+   */
+  public async imageEdit(optionsArg: ImageEditOptions): Promise<ImageResponse> {
+    throw new Error('Image editing is not supported by Groq. Please use OpenAI provider for image editing.');
+  }
 }

package/ts/provider.ollama.ts

@@ -1,7 +1,16 @@
 import * as plugins from './plugins.js';
 import * as paths from './paths.js';
 import { MultiModalModel } from './abstract.classes.multimodal.js';
-import type { ChatOptions, ChatResponse, ChatMessage, ResearchOptions, ResearchResponse } from './abstract.classes.multimodal.js';
+import type {
+  ChatOptions,
+  ChatResponse,
+  ChatMessage,
+  ResearchOptions,
+  ResearchResponse,
+  ImageGenerateOptions,
+  ImageEditOptions,
+  ImageResponse
+} from './abstract.classes.multimodal.js';
 
 export interface IOllamaProviderOptions {
   baseUrl?: string;
@@ -255,4 +264,18 @@ export class OllamaProvider extends MultiModalModel {
   public async research(optionsArg: ResearchOptions): Promise<ResearchResponse> {
     throw new Error('Research capabilities are not yet supported by Ollama provider.');
   }
+
+  /**
+   * Image generation is not supported by Ollama
+   */
+  public async imageGenerate(optionsArg: ImageGenerateOptions): Promise<ImageResponse> {
+    throw new Error('Image generation is not supported by Ollama. Please use OpenAI provider for image generation.');
+  }
+
+  /**
+   * Image editing is not supported by Ollama
+   */
+  public async imageEdit(optionsArg: ImageEditOptions): Promise<ImageResponse> {
+    throw new Error('Image editing is not supported by Ollama. Please use OpenAI provider for image editing.');
+  }
 }