outlet-orm 7.0.0 → 9.0.0

package/src/AI/Providers/CustomOpenAIProvider.js

@@ -0,0 +1,238 @@
+'use strict';
+
+const JsonSchemaValidator = require('../Support/JsonSchemaValidator');
+
+/**
+ * CustomOpenAIProvider
+ * Fully configurable OpenAI-compatible provider.
+ * Works with Azure OpenAI, proxies, OpenRouter, self-hosted endpoints, etc.
+ * Supports chat, streaming (SSE), embeddings, images, audio TTS/STT, and models.
+ */
+class CustomOpenAIProvider {
+  /**
+   * @param {string} apiKey
+   * @param {string} baseUrl
+   * @param {Object} [paths={}]
+   * @param {string} [authHeader='Authorization']
+   * @param {string} [authPrefix='Bearer ']
+   * @param {Object} [extraHeaders={}]
+   */
+  constructor(apiKey, baseUrl, paths = {}, authHeader = 'Authorization', authPrefix = 'Bearer ', extraHeaders = {}) {
+    this.apiKey = apiKey;
+    this.baseUrl = baseUrl.replace(/\/+$/, '');
+    this.paths = paths;
+    this.authHeader = authHeader;
+    this.authPrefix = authPrefix;
+    this.extraHeaders = extraHeaders;
+  }
+
+  /** @private */
+  _endpoint(key) {
+    const p = this.paths[key] || '';
+    return this.baseUrl + p;
+  }
+
+  /** @private */
+  _headers() {
+    return Object.assign({
+      [this.authHeader]: this.authPrefix + this.apiKey,
+      'Content-Type': 'application/json',
+      'Accept': 'application/json',
+    }, this.extraHeaders);
+  }
+
+  /** @private */
+  async _post(url, body, stream = false) {
+    const opts = { method: 'POST', headers: this._headers(), body: JSON.stringify(body) };
+    if (stream) return fetch(url, opts);
+    const res = await fetch(url, opts);
+    return res.json();
+  }
+
+  /** @private */
+  async _get(url) {
+    const res = await fetch(url, { method: 'GET', headers: this._headers() });
+    return res.json();
+  }
+
+  // ─── Chat ───
+  async chat(messages, options = {}) {
+    const payload = this._buildChatPayload(messages, options);
+    const res = await this._post(this._endpoint('chat'), payload);
+    this._normalizeToolCallsOnResponse(res);
+    return res || {};
+  }
+
+  /** @private */
+  _buildChatPayload(messages, options) {
+    const payload = {
+      model: options.model || options.deployment || 'gpt-like',
+      messages,
+    };
+    this._applySamplingOptions(payload, options);
+    this._applyResponseFormatOptions(payload, options);
+    this._applyToolsOptions(payload, options);
+    return payload;
+  }
+
+  /** @private */
+  _applySamplingOptions(payload, options) {
+    for (const k of ['temperature', 'top_p', 'max_tokens', 'frequency_penalty', 'presence_penalty', 'stop', 'seed', 'user']) {
+      if (options[k] !== undefined) payload[k] = options[k];
+    }
+  }
+
+  /** @private */
+  _applyResponseFormatOptions(payload, options) {
+    if (options.response_format === 'json') {
+      const schema = (options.json_schema || {}).schema || { type: 'object' };
+      payload.response_format = {
+        type: 'json_schema',
+        json_schema: options.json_schema || { name: 'auto_schema', schema },
+      };
+    }
+  }
+
+  /** @private */
+  _applyToolsOptions(payload, options) {
+    if (!options.tools || !Array.isArray(options.tools)) return;
+    payload.tools = options.tools.map(tool => ({
+      type: 'function',
+      function: {
+        name: tool.name,
+        description: tool.description || '',
+        parameters: tool.parameters || tool.schema || { type: 'object', properties: {} },
+      },
+    }));
+    if (options.tool_choice) payload.tool_choice = options.tool_choice;
+  }
+
+  /** @private */
+  _normalizeToolCallsOnResponse(res) {
+    if (!res || !res.choices?.[0]?.message?.tool_calls) return;
+    res.tool_calls = res.choices[0].message.tool_calls.map(tc => ({
+      id: tc.id || null,
+      name: (tc.function || {}).name || null,
+      arguments: (() => { try { return JSON.parse((tc.function || {}).arguments || '{}'); } catch { return {}; } })(),
+    }));
+  }
+
+  // ─── Streaming ───
+  async *stream(messages, options = {}) {
+    const payload = { model: options.model || 'gpt-like', messages, stream: true };
+    const res = await this._post(this._endpoint('chat'), payload, true);
+    yield* this._readSse(res.body);
+  }
+
+  async *streamEvents(messages, options = {}) {
+    const payload = { model: options.model || 'gpt-like', messages, stream: true };
+    const res = await this._post(this._endpoint('chat'), payload, true);
+    for await (const delta of this._readSse(res.body)) {
+      yield { type: 'delta', data: delta };
+    }
+    yield { type: 'end', data: null };
+  }
+
+  /** @private */
+  async *_readSse(body) {
+    const reader = body.getReader();
+    const decoder = new TextDecoder();
+    let buffer = '';
+    try {
+      while (true) {
+        const { done, value } = await reader.read();
+        if (done) break;
+        buffer += decoder.decode(value, { stream: true });
+        const lines = buffer.split(/\r?\n/);
+        buffer = lines.pop() || '';
+        for (const line of lines) {
+          const trimmed = line.trim();
+          if (trimmed === '' || trimmed.startsWith(':') || !trimmed.startsWith('data:')) continue;
+          const json = trimmed.slice(5).trim();
+          if (json === '[DONE]') return;
+          try {
+            const decoded = JSON.parse(json);
+            const delta = decoded?.choices?.[0]?.delta?.content || null;
+            if (delta !== null) yield delta;
+          } catch { /* skip */ }
+        }
+      }
+    } finally {
+      reader.releaseLock();
+    }
+  }
+
+  supportsStreaming() { return true; }
+
+  // ─── Models ───
+  async listModels() {
+    const url = this.baseUrl + this._modelsPath();
+    return (await this._get(url)) || {};
+  }
+
+  async getModel(id) {
+    const url = this.baseUrl + this._modelsPath() + '/' + encodeURIComponent(id);
+    return (await this._get(url)) || {};
+  }
+
+  /** @private */
+  _modelsPath() {
+    if (this.paths.models) return this.paths.models;
+    if (/\/v\d+(?:$|\/)/.test(this.baseUrl)) return '/models';
+    return '/v1/models';
+  }
+
+  // ─── Embeddings ───
+  async embeddings(inputs, options = {}) {
+    const payload = { model: options.model || 'embedding-model', input: inputs };
+    const res = await this._post(this._endpoint('embeddings'), payload);
+    return {
+      embeddings: (res.data || []).map(d => d.embedding || []),
+      usage: res.usage || {},
+      raw: res,
+    };
+  }
+
+  // ─── Images ───
+  async generateImage(prompt, options = {}) {
+    const payload = { prompt, model: options.model || 'image-model', n: 1 };
+    const res = await this._post(this._endpoint('image'), payload);
+    return { images: res.data || [], raw: res };
+  }
+
+  // ─── Audio ───
+  async textToSpeech(text, options = {}) {
+    const payload = {
+      model: options.model || 'tts-model',
+      input: text,
+      voice: options.voice || 'alloy',
+      format: options.format || 'mp3',
+    };
+    const res = await fetch(this._endpoint('tts'), {
+      method: 'POST',
+      headers: this._headers(),
+      body: JSON.stringify(payload),
+    });
+    const arrayBuf = await res.arrayBuffer();
+    return { audio: Buffer.from(arrayBuf).toString('base64'), mime: 'audio/mpeg' };
+  }
+
+  async speechToText(filePath, options = {}) {
+    const fs = require('fs');
+    const path = require('path');
+    const formData = new FormData();
+    const fileBuffer = fs.readFileSync(filePath);
+    const blob = new Blob([fileBuffer], { type: 'application/octet-stream' });
+    formData.append('file', blob, path.basename(filePath));
+    formData.append('model', options.model || 'stt-model');
+    formData.append('response_format', 'json');
+
+    const headers = { [this.authHeader]: this.authPrefix + this.apiKey };
+    Object.assign(headers, this.extraHeaders);
+    const res = await fetch(this._endpoint('stt'), { method: 'POST', headers, body: formData });
+    const data = await res.json();
+    return { text: data.text || '', raw: data };
+  }
+}
+
+module.exports = CustomOpenAIProvider;

package/src/AI/Providers/GeminiProvider.js

@@ -0,0 +1,68 @@
+'use strict';
+
+/**
+ * GeminiProvider
+ * Google Generative Language API. Supports chat, simulated streaming, and embeddings.
+ */
+class GeminiProvider {
+  /**
+   * @param {string} apiKey
+   * @param {string} [chatEndpoint='https://generativelanguage.googleapis.com/v1beta/models/gemini-pro:generateContent']
+   */
+  constructor(apiKey, chatEndpoint = 'https://generativelanguage.googleapis.com/v1beta/models/gemini-pro:generateContent') {
+    this.apiKey = apiKey;
+    this.chatEndpoint = chatEndpoint;
+    this.embedEndpoint = 'https://generativelanguage.googleapis.com/v1beta/models/text-embedding-004:embedContent';
+  }
+
+  /** @private */
+  _keyQuery() { return `?key=${this.apiKey}`; }
+
+  async chat(messages, options = {}) {
+    const userTexts = messages
+      .filter(m => (m.role || '') !== 'system')
+      .map(m => m.content);
+
+    const payload = {
+      contents: [{ parts: [{ text: userTexts.join('\n') }] }],
+    };
+
+    const res = await fetch(this.chatEndpoint + this._keyQuery(), {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify(payload),
+    });
+    const data = await res.json();
+    return data || {};
+  }
+
+  async *stream(messages, options = {}) {
+    const full = await this.chat(messages, options);
+    const text = full?.candidates?.[0]?.content?.parts?.[0]?.text || '';
+    for (let i = 0; i < text.length; i += 80) {
+      yield text.slice(i, i + 80);
+    }
+  }
+
+  supportsStreaming() { return true; } // simulated
+
+  async embeddings(inputs, options = {}) {
+    const vectors = [];
+    for (const input of inputs) {
+      const payload = {
+        model: 'text-embedding-004',
+        content: { parts: [{ text: input }] },
+      };
+      const res = await fetch(this.embedEndpoint + this._keyQuery(), {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify(payload),
+      });
+      const data = await res.json();
+      vectors.push((data?.embedding || {}).values || []);
+    }
+    return { embeddings: vectors };
+  }
+}
+
+module.exports = GeminiProvider;

package/src/AI/Providers/GrokProvider.js

@@ -0,0 +1,46 @@
+'use strict';
+
+/**
+ * GrokProvider
+ * Prompt-based chat with simulated streaming (80-char chunks).
+ */
+class GrokProvider {
+  /**
+   * @param {string} apiKey
+   * @param {string} [endpoint='https://api.grok.com/v1/chat']
+   */
+  constructor(apiKey, endpoint = 'https://api.grok.com/v1/chat') {
+    this.apiKey = apiKey;
+    this.endpoint = endpoint;
+  }
+
+  async chat(messages, options = {}) {
+    const joined = messages.map(m => m.content).join('\n');
+    const payload = {
+      prompt: joined,
+      model: options.model || 'grok-default',
+    };
+    const res = await fetch(this.endpoint, {
+      method: 'POST',
+      headers: {
+        'Authorization': `Bearer ${this.apiKey}`,
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify(payload),
+    });
+    const data = await res.json();
+    return data || {};
+  }
+
+  async *stream(messages, options = {}) {
+    const full = await this.chat(messages, options);
+    const text = full.response || '';
+    for (let i = 0; i < text.length; i += 80) {
+      yield text.slice(i, i + 80);
+    }
+  }
+
+  supportsStreaming() { return true; } // simulated
+}
+
+module.exports = GrokProvider;

package/src/AI/Providers/MistralProvider.js

@@ -0,0 +1,21 @@
+'use strict';
+
+const OpenAIProvider = require('./OpenAIProvider');
+
+/**
+ * MistralProvider
+ * Targets Mistral AI API (https://api.mistral.ai) — OpenAI-compatible endpoints.
+ */
+class MistralProvider extends OpenAIProvider {
+  /**
+   * @param {string} apiKey
+   * @param {string} [chatEndpoint='https://api.mistral.ai/v1/chat/completions']
+   */
+  constructor(apiKey, chatEndpoint = 'https://api.mistral.ai/v1/chat/completions') {
+    super(apiKey, chatEndpoint);
+    this.modelsEndpoint = 'https://api.mistral.ai/v1/models';
+    this.embeddingsEndpoint = 'https://api.mistral.ai/v1/embeddings';
+  }
+}
+
+module.exports = MistralProvider;

package/src/AI/Providers/OllamaProvider.js

@@ -0,0 +1,249 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const DocumentAttachmentMapper = require('../Support/DocumentAttachmentMapper');
+const FileSecurity = require('../Support/FileSecurity');
+const ProviderError = require('../Support/Exceptions/ProviderError');
+
+/**
+ * OllamaProvider
+ * Implements chat, NDJSON streaming, embeddings, basic image generation,
+ * structured JSON output (format=json), and multimodal vision input.
+ */
+class OllamaProvider {
+  /**
+   * @param {string} [endpoint='http://localhost:11434']
+   */
+  constructor(endpoint = 'http://localhost:11434') {
+    this.base = endpoint.replace(/\/+$/, '');
+    this.chatEndpoint = `${this.base}/api/chat`;
+    this.embeddingsEndpoint = `${this.base}/api/embeddings`;
+    this.generateEndpoint = `${this.base}/api/generate`;
+  }
+
+  /** @protected — override in subclass to add auth */
+  _decorateHeaders(headers) {
+    return headers;
+  }
+
+  /** @private */
+  async _post(url, body, stream = false) {
+    const headers = this._decorateHeaders({ 'Content-Type': 'application/json' });
+    const fetchOpts = { method: 'POST', headers, body: JSON.stringify(body) };
+    if (stream) return fetch(url, fetchOpts);
+    const res = await fetch(url, fetchOpts);
+    return res.json();
+  }
+
+  // ─── Chat ───
+  async chat(messages, options = {}) {
+    let accFiles = [];
+    let accImages = [];
+
+    // Process attachments
+    for (let i = 0; i < messages.length; i++) {
+      const atts = messages[i].attachments || [];
+      if (atts.length > 0) {
+        const mapped = DocumentAttachmentMapper.toOllamaOptions(atts);
+        if (mapped.inlineTexts.length > 0) {
+          messages[i].content = ((messages[i].content || '') + '\n' + mapped.inlineTexts.join('\n\n')).trim();
+        }
+        accFiles = accFiles.concat(mapped.files);
+        accImages = accImages.concat(mapped.image_files);
+        delete messages[i].attachments;
+      }
+    }
+
+    const payload = {
+      model: options.model || 'gemma3:4b',
+      messages,
+      stream: false,
+    };
+
+    // Options
+    if (options.temperature) { payload.options = payload.options || {}; payload.options.temperature = options.temperature; }
+    if (options.top_p) { payload.options = payload.options || {}; payload.options.top_p = options.top_p; }
+    if (options.top_k) { payload.options = payload.options || {}; payload.options.top_k = options.top_k; }
+    if (options.repeat_penalty) { payload.options = payload.options || {}; payload.options.repeat_penalty = options.repeat_penalty; }
+    if (options.stop) { payload.options = payload.options || {}; payload.options.stop = Array.isArray(options.stop) ? options.stop : [options.stop]; }
+
+    // JSON format
+    if (options.response_format === 'json') {
+      payload.format = 'json';
+      const hasSystem = payload.messages.some(m => m.role === 'system');
+      if (!hasSystem) {
+        payload.messages.unshift({
+          role: 'system',
+          content: 'You must respond only with valid JSON without additional text.',
+        });
+      }
+    }
+
+    // Files and images
+    const optFiles = (options.files && Array.isArray(options.files)) ? this._prepareFiles(options.files) : [];
+    const optImages = (options.image_files && Array.isArray(options.image_files)) ? this._prepareImageFiles(options.image_files) : [];
+    const files = optFiles.concat(accFiles);
+    const images = optImages.concat(accImages);
+    if (files.length > 0) payload.files = files;
+    if (images.length > 0) {
+      const lastIndex = payload.messages.length - 1;
+      if (lastIndex >= 0 && payload.messages[lastIndex].role === 'user') {
+        payload.messages[lastIndex].images = images;
+      } else {
+        payload.messages.push({ role: 'user', content: '', images });
+      }
+    }
+
+    const data = await this._post(this.chatEndpoint, payload);
+    return data || {};
+  }
+
+  // ─── Streaming (NDJSON) ───
+  async *stream(messages, options = {}) {
+    let accFiles = [];
+    let accImages = [];
+
+    for (let i = 0; i < messages.length; i++) {
+      const atts = messages[i].attachments || [];
+      if (atts.length > 0) {
+        const mapped = DocumentAttachmentMapper.toOllamaOptions(atts);
+        if (mapped.inlineTexts.length > 0) {
+          messages[i].content = ((messages[i].content || '') + '\n' + mapped.inlineTexts.join('\n\n')).trim();
+        }
+        accFiles = accFiles.concat(mapped.files);
+        accImages = accImages.concat(mapped.image_files);
+        delete messages[i].attachments;
+      }
+    }
+
+    const payload = {
+      model: options.model || 'gemma3:4b',
+      messages,
+      stream: true,
+    };
+
+    if (options.temperature) { payload.options = payload.options || {}; payload.options.temperature = options.temperature; }
+    if (options.top_p) { payload.options = payload.options || {}; payload.options.top_p = options.top_p; }
+    if (options.top_k) { payload.options = payload.options || {}; payload.options.top_k = options.top_k; }
+    if (options.repeat_penalty) { payload.options = payload.options || {}; payload.options.repeat_penalty = options.repeat_penalty; }
+    if (options.stop) { payload.options = payload.options || {}; payload.options.stop = Array.isArray(options.stop) ? options.stop : [options.stop]; }
+    if (options.response_format === 'json') payload.format = 'json';
+
+    const optFiles = (options.files && Array.isArray(options.files)) ? this._prepareFiles(options.files) : [];
+    const optImages = (options.image_files && Array.isArray(options.image_files)) ? this._prepareImageFiles(options.image_files) : [];
+    const files = optFiles.concat(accFiles);
+    const images = optImages.concat(accImages);
+    if (files.length > 0) payload.files = files;
+    if (images.length > 0) {
+      const lastIndex = payload.messages.length - 1;
+      if (lastIndex >= 0 && payload.messages[lastIndex].role === 'user') {
+        payload.messages[lastIndex].images = images;
+      } else {
+        payload.messages.push({ role: 'user', content: '', images });
+      }
+    }
+
+    const res = await this._post(this.chatEndpoint, payload, true);
+    const reader = res.body.getReader();
+    const decoder = new TextDecoder();
+    let buffer = '';
+    try {
+      while (true) {
+        const { done, value } = await reader.read();
+        if (done) break;
+        buffer += decoder.decode(value, { stream: true });
+        const lines = buffer.split(/\r?\n/);
+        buffer = lines.pop() || '';
+        for (const line of lines) {
+          const trimmed = line.trim();
+          if (!trimmed) continue;
+          try {
+            const decoded = JSON.parse(trimmed);
+            if (decoded?.message?.content) yield decoded.message.content;
+          } catch { /* skip invalid JSON lines */ }
+        }
+      }
+    } finally {
+      reader.releaseLock();
+    }
+  }
+
+  supportsStreaming() { return true; }
+
+  // ─── Embeddings ───
+  async embeddings(inputs, options = {}) {
+    const model = options.model || 'nomic-embed-text';
+    const vectors = [];
+    for (const text of inputs) {
+      const res = await this._post(this.embeddingsEndpoint, { model, prompt: text });
+      vectors.push(res.embedding || []);
+    }
+    return { embeddings: vectors, raw: null };
+  }
+
+  // ─── File helpers ───
+  /** @private */
+  _prepareFiles(files) {
+    const out = [];
+    const security = FileSecurity.fromConfig();
+    for (const file of files) {
+      if (typeof file !== 'string' || !fs.existsSync(file)) continue;
+      if (!security.validateFile(file, false)) continue;
+      out.push({
+        name: path.basename(file),
+        type: 'application/octet-stream',
+        content: fs.readFileSync(file).toString('base64'),
+      });
+    }
+    return out;
+  }
+
+  /** @private */
+  _prepareImageFiles(files) {
+    const images = [];
+    const security = FileSecurity.fromConfig();
+    for (const file of files) {
+      if (typeof file !== 'string' || !fs.existsSync(file)) continue;
+      const ext = path.extname(file).toLowerCase();
+      const imageExts = ['.png', '.jpg', '.jpeg', '.gif', '.bmp', '.webp', '.svg'];
+      if (imageExts.includes(ext) && security.validateFile(file, true)) {
+        images.push(fs.readFileSync(file).toString('base64'));
+      }
+    }
+    return images;
+  }
+
+  // ─── Images ───
+  async generateImage(prompt, options = {}) {
+    const payload = {
+      model: options.model || 'stable-diffusion',
+      prompt,
+      stream: false,
+    };
+    if (options.negative_prompt) payload.negative = options.negative_prompt;
+    const res = await this._post(this.generateEndpoint, payload);
+
+    const images = [];
+    if (Array.isArray(res.images)) {
+      for (const img of res.images) images.push({ b64: img });
+    } else if (res.image) {
+      images.push({ b64: res.image });
+    } else if (typeof res.response === 'string' && res.response.startsWith('data:image')) {
+      const match = res.response.match(/base64,(.*)$/);
+      if (match) images.push({ b64: match[1] });
+    }
+    return { images, meta: { model: payload.model }, raw: res };
+  }
+
+  // ─── Audio (not supported) ───
+  async textToSpeech(_text, _options = {}) {
+    throw ProviderError.unsupported('ollama', 'tts');
+  }
+
+  async speechToText(_filePath, _options = {}) {
+    throw ProviderError.unsupported('ollama', 'stt');
+  }
+}
+
+module.exports = OllamaProvider;

package/src/AI/Providers/OllamaTurboProvider.js

@@ -0,0 +1,32 @@
+'use strict';
+
+const OllamaProvider = require('./OllamaProvider');
+
+/**
+ * OllamaTurboProvider
+ * Targets Ollama Turbo (SaaS) at https://ollama.com with API key auth.
+ */
+class OllamaTurboProvider extends OllamaProvider {
+  /**
+   * @param {string|null} apiKey
+   * @param {string} [endpoint='https://ollama.com']
+   */
+  constructor(apiKey, endpoint = 'https://ollama.com') {
+    super(endpoint);
+    this.apiKey = apiKey || null;
+  }
+
+  /** @override */
+  _decorateHeaders(headers) {
+    headers = super._decorateHeaders(headers);
+    if (this.apiKey) {
+      const value = this.apiKey.toLowerCase().startsWith('bearer ')
+        ? this.apiKey
+        : `Bearer ${this.apiKey}`;
+      headers['Authorization'] = value;
+    }
+    return headers;
+  }
+}
+
+module.exports = OllamaTurboProvider;