neoagent 2.3.1-beta.0 → 2.3.1-beta.100

package/server/services/ai/providers/githubCopilot.js

@@ -0,0 +1,97 @@
+const { OpenAIProvider } = require('./openai');
+
+class GithubCopilotProvider extends OpenAIProvider {
+  constructor(config = {}) {
+    // GitHub Copilot base URL defaults to the individual endpoint
+    const defaultBaseUrl = 'https://api.individual.githubcopilot.com';
+    const baseUrl = config.baseUrl || process.env.GITHUB_COPILOT_BASE_URL || defaultBaseUrl;
+
+    super({
+      ...config,
+      apiKey: config.apiKey || process.env.GITHUB_COPILOT_ACCESS_TOKEN,
+      baseUrl,
+      // Pass special headers required by GitHub Copilot
+      defaultHeaders: {
+        'Editor-Version': 'vscode/1.90.0',
+        'Editor-Plugin-Version': 'copilot-chat/0.15.0',
+        'User-Agent': 'GithubCopilot/1.155.0',
+        'X-Github-Api-Version': '2023-07-07',
+        'Copilot-Integration-Id': 'vscode-chat'
+      }
+    });
+    this.name = 'github-copilot';
+    this.githubToken = config.apiKey || process.env.GITHUB_COPILOT_ACCESS_TOKEN;
+    this.copilotToken = null;
+    this.tokenExpiresAt = 0;
+    this._refreshPromise = null;
+  }
+
+  async _refreshCopilotToken() {
+    if (this._refreshPromise) return this._refreshPromise;
+
+    const now = Math.floor(Date.now() / 1000);
+    // Refresh token if missing or expiring in less than 5 minutes
+    if (this.copilotToken && this.tokenExpiresAt >= now + 300) {
+      return;
+    }
+
+    this._refreshPromise = (async () => {
+      try {
+        if (!this.githubToken) {
+          throw new Error('GitHub Copilot access token is missing. Please run `neoagent login github-copilot`.');
+        }
+
+        const res = await fetch('https://api.github.com/copilot_internal/v2/token', {
+          headers: {
+            'Authorization': `token ${this.githubToken}`,
+            'Accept': 'application/json',
+            'User-Agent': 'NeoAgent/1.0.0'
+          }
+        });
+
+        if (!res.ok) {
+          const errorText = await res.text().catch(() => 'Unknown error');
+          throw new Error(`Failed to refresh GitHub Copilot token: HTTP ${res.status} - ${errorText}`);
+        }
+
+        const data = await res.json();
+        if (!data || typeof data.token !== 'string' || !data.token) {
+          throw new Error('Invalid token response from GitHub Copilot.');
+        }
+
+        this.copilotToken = data.token;
+        this.tokenExpiresAt = typeof data.expires_at === 'number'
+          ? data.expires_at
+          : Math.floor(new Date(data.expires_at).getTime() / 1000);
+
+        if (isNaN(this.tokenExpiresAt)) {
+          this.tokenExpiresAt = Math.floor(Date.now() / 1000) + 1800;
+        }
+
+        // Update the client's API key
+        this.client.apiKey = this.copilotToken;
+      } finally {
+        this._refreshPromise = null;
+      }
+    })();
+
+    return this._refreshPromise;
+  }
+
+  async chat(messages, tools = [], options = {}) {
+    await this._refreshCopilotToken();
+    return super.chat(messages, tools, options);
+  }
+
+  async *stream(messages, tools = [], options = {}) {
+    await this._refreshCopilotToken();
+    yield* super.stream(messages, tools, options);
+  }
+
+  async analyzeImage(options = {}) {
+    await this._refreshCopilotToken();
+    return super.analyzeImage(options);
+  }
+}
+
+module.exports = { GithubCopilotProvider };

package/server/services/ai/providers/openai.js

@@ -29,7 +29,8 @@ class OpenAIProvider extends BaseProvider {
     };
     this.client = new OpenAI({
       apiKey: config.apiKey || process.env.OPENAI_API_KEY,
-      baseURL: config.baseUrl || process.env.OPENAI_BASE_URL || undefined
+      baseURL: config.baseUrl || process.env.OPENAI_BASE_URL || undefined,
+      defaultHeaders: config.defaultHeaders || undefined
     });
   }
 

package/server/services/ai/providers/openaiCodex.js

@@ -0,0 +1,384 @@
+const OpenAI = require('openai');
+const { BaseProvider } = require('./base');
+
+const DEFAULT_BASE_URL = 'https://chatgpt.com/backend-api/codex';
+
+function isCodexBackendBaseUrl(baseURL) {
+  const trimmed = String(baseURL || '').trim();
+  if (!trimmed) return false;
+
+  try {
+    const url = new URL(trimmed);
+    const path = url.pathname.replace(/\/+$/, '');
+    return url.hostname === 'chatgpt.com'
+      && (path === '/backend-api' || path === '/backend-api/codex' || path === '/backend-api/codex/v1');
+  } catch {
+    return false;
+  }
+}
+
+function isOpenAIApiBaseUrl(baseURL) {
+  const trimmed = String(baseURL || '').trim();
+  if (!trimmed) return false;
+
+  try {
+    const url = new URL(trimmed);
+    const path = url.pathname.replace(/\/+$/, '');
+    return url.hostname === 'api.openai.com' && (path === '' || path === '/v1');
+  } catch {
+    return false;
+  }
+}
+
+function normalizeCodexBaseUrl(baseURL) {
+  if (!baseURL || isOpenAIApiBaseUrl(baseURL) || isCodexBackendBaseUrl(baseURL)) {
+    return DEFAULT_BASE_URL;
+  }
+  return baseURL;
+}
+
+function normalizeContent(content) {
+  if (content == null) return '';
+  if (typeof content === 'string') return content;
+  if (Array.isArray(content)) {
+    return content.map((part) => {
+      if (!part) return '';
+      if (typeof part === 'string') return part;
+      if (part.type === 'text' && typeof part.text === 'string') return part.text;
+      if (part.type === 'input_text' && typeof part.text === 'string') return part.text;
+      return '';
+    }).join('');
+  }
+  return String(content);
+}
+
+function normalizeInputContent(content) {
+  if (content == null) return [];
+
+  if (typeof content === 'string') {
+    return [{ type: 'input_text', text: content }];
+  }
+
+  if (!Array.isArray(content)) {
+    const text = String(content);
+    return text ? [{ type: 'input_text', text }] : [];
+  }
+
+  const parts = [];
+  for (const part of content) {
+    if (!part) continue;
+    if (typeof part === 'string') {
+      if (part.trim()) parts.push({ type: 'input_text', text: part });
+      continue;
+    }
+    if (part.type === 'text' && typeof part.text === 'string') {
+      parts.push({ type: 'input_text', text: part.text });
+      continue;
+    }
+    if (part.type === 'input_text' && typeof part.text === 'string') {
+      parts.push({ type: 'input_text', text: part.text });
+      continue;
+    }
+    if (part.type === 'image_url') {
+      const imageUrl = typeof part.image_url === 'string' ? part.image_url : part.image_url?.url;
+      if (imageUrl) {
+        parts.push({
+          type: 'input_image',
+          image_url: imageUrl,
+          detail: part.detail || 'auto',
+        });
+      }
+      continue;
+    }
+    if (part.type === 'input_image') {
+      parts.push({
+        type: 'input_image',
+        image_url: part.image_url || null,
+        file_id: part.file_id || null,
+        detail: part.detail || 'auto',
+      });
+    }
+  }
+
+  return parts;
+}
+
+function toFunctionCallOutput(toolCallId, content) {
+  return {
+    type: 'function_call_output',
+    call_id: toolCallId,
+    output: normalizeContent(content),
+  };
+}
+
+function extractResponseText(response) {
+  if (typeof response?.output_text === 'string' && response.output_text.length > 0) {
+    return response.output_text;
+  }
+
+  const parts = [];
+  for (const item of response?.output || []) {
+    if (item?.type !== 'message') continue;
+    for (const content of item.content || []) {
+      if (content?.type === 'output_text' && typeof content.text === 'string') {
+        parts.push(content.text);
+      }
+    }
+  }
+  return parts.join('');
+}
+
+function extractToolCalls(response) {
+  const toolCalls = [];
+  for (const item of response?.output || []) {
+    if (item?.type !== 'function_call') continue;
+    toolCalls.push({
+      id: item.call_id || item.id || '',
+      type: 'function',
+      function: {
+        name: item.name || '',
+        arguments: item.arguments || '',
+      },
+    });
+  }
+  return toolCalls.filter((toolCall) => toolCall.id && toolCall.function.name);
+}
+
+function formatOpenAIError(err) {
+  if (!err || typeof err !== 'object') return 'Unknown OpenAI error';
+  const parts = [];
+  if (typeof err.status === 'number') parts.push(`HTTP ${err.status}`);
+  if (err.type) parts.push(`type=${err.type}`);
+  if (err.code) parts.push(`code=${err.code}`);
+  if (err.param) parts.push(`param=${err.param}`);
+  if (err.request_id) parts.push(`request_id=${err.request_id}`);
+  const message = err.message || 'Unknown OpenAI error';
+  return parts.length > 0 ? `${message} (${parts.join(', ')})` : message;
+}
+
+class OpenAICodexProvider extends BaseProvider {
+  constructor(config = {}) {
+    super(config);
+
+    const configuredBaseURL = config.baseUrl || process.env.OPENAI_CODEX_BASE_URL || DEFAULT_BASE_URL;
+    const baseURL = normalizeCodexBaseUrl(configuredBaseURL);
+
+    this.baseURL = baseURL;
+    this.usesCodexBackend = isCodexBackendBaseUrl(baseURL);
+
+    if (!this.usesCodexBackend && !baseURL.includes('api.openai.com')) {
+      console.warn(`[OpenAICodex] Using non-official base URL: ${baseURL}`);
+    } else if (this.usesCodexBackend) {
+      console.info(`[OpenAICodex] Using ChatGPT Codex endpoint: ${baseURL}`);
+    }
+
+    this.name = 'openai-codex';
+    this.models = [
+      'gpt-5.5',
+      'gpt-5.4',
+      'gpt-5.4-mini',
+    ];
+    this.reasoningModels = new Set([
+      'gpt-5.5',
+      'gpt-5.4',
+      'gpt-5.4-mini',
+    ]);
+    const defaultHeaders = this.usesCodexBackend
+      ? {
+          'Editor-Version': process.env.OPENAI_CODEX_EDITOR_VERSION || 'vscode/1.99.0',
+          'Editor-Plugin-Version': process.env.OPENAI_CODEX_EDITOR_PLUGIN_VERSION || 'neoagent/1.0.0',
+          'User-Agent': process.env.OPENAI_CODEX_USER_AGENT || 'NeoAgent/1.0.0',
+        }
+      : undefined;
+
+    this.client = new OpenAI({
+      apiKey: config.apiKey || process.env.OPENAI_CODEX_ACCESS_TOKEN,
+      baseURL,
+      defaultHeaders,
+    });
+  }
+
+  formatTools(tools) {
+    return tools.map((tool) => ({
+      type: 'function',
+      name: tool.name,
+      description: tool.description || '',
+      parameters: tool.parameters || { type: 'object', properties: {} },
+      strict: false,
+    }));
+  }
+
+  _isReasoningModel(model) {
+    if (!model) return false;
+    for (const id of this.reasoningModels) {
+      if (model === id || model.startsWith(`${id}-`)) return true;
+    }
+    return false;
+  }
+
+  _buildRequest(messages = [], tools = [], options = {}, model = '') {
+    const instructions = [];
+    const input = [];
+
+    for (const msg of messages || []) {
+      if (!msg || !msg.role) continue;
+
+      if ((msg.role === 'system' || msg.role === 'developer') && msg.content != null) {
+        const text = normalizeContent(msg.content).trim();
+        if (text) instructions.push(text);
+        continue;
+      }
+
+      if (msg.role === 'tool') {
+        const toolCallId = String(msg.tool_call_id || '').trim();
+        if (!toolCallId) continue;
+        input.push(toFunctionCallOutput(toolCallId, msg.content));
+        continue;
+      }
+
+      const content = normalizeInputContent(msg.content);
+      if (content.length > 0) {
+        input.push({
+          type: 'message',
+          role: msg.role === 'assistant' ? 'assistant' : 'user',
+          content,
+        });
+      }
+
+      if (msg.role === 'assistant' && Array.isArray(msg.tool_calls) && msg.tool_calls.length > 0) {
+        for (const toolCall of msg.tool_calls) {
+          const name = String(toolCall?.function?.name || '').trim();
+          const argumentsText = String(toolCall?.function?.arguments || '');
+          const callId = String(toolCall?.id || toolCall?.call_id || '').trim();
+          if (!name || !callId) continue;
+          input.push({
+            type: 'function_call',
+            call_id: callId,
+            name,
+            arguments: argumentsText,
+          });
+        }
+      }
+    }
+
+    const request = {
+      input,
+    };
+
+    if (instructions.length > 0) {
+      request.instructions = instructions.join('\n\n');
+    }
+
+    if (tools && tools.length > 0) {
+      request.tools = this.formatTools(tools);
+      request.tool_choice = options.toolChoice || 'auto';
+    }
+
+    if (!this.usesCodexBackend) {
+      request.max_output_tokens = options.maxTokens || 16384;
+
+      if (options.temperature !== undefined && options.temperature !== null) {
+        request.temperature = options.temperature;
+      }
+    }
+
+    const reasoningEffort = options.reasoningEffort || options.reasoning_effort;
+    if (reasoningEffort || this._isReasoningModel(model)) {
+      request.reasoning = {
+        effort: reasoningEffort || 'medium',
+      };
+    }
+
+    return request;
+  }
+
+  async chat(messages, tools = [], options = {}) {
+    const model = options.model || this.config.model || this.getDefaultModel();
+    const request = this._buildRequest(messages, tools, options, model);
+    let response;
+    try {
+      response = await this.client.responses.create({
+        model,
+        ...request,
+      });
+    } catch (err) {
+      throw new Error(`OpenAI Codex request failed: ${formatOpenAIError(err)}`);
+    }
+
+    const toolCalls = extractToolCalls(response);
+
+    return {
+      content: extractResponseText(response),
+      toolCalls,
+      finishReason: toolCalls.length > 0 ? 'tool_calls' : 'stop',
+      usage: response.usage ? {
+        promptTokens: response.usage.input_tokens,
+        completionTokens: response.usage.output_tokens,
+        totalTokens: response.usage.total_tokens,
+      } : null,
+      model: response.model,
+    };
+  }
+
+  async *stream(messages, tools = [], options = {}) {
+    const model = options.model || this.config.model || this.getDefaultModel();
+    const request = this._buildRequest(messages, tools, options, model);
+    let stream;
+    try {
+      stream = await this.client.responses.create({
+        model,
+        ...request,
+        stream: true,
+      });
+    } catch (err) {
+      throw new Error(`OpenAI Codex request failed: ${formatOpenAIError(err)}`);
+    }
+
+    let content = '';
+    let finalResponse = null;
+
+    for await (const event of stream) {
+      if (event.type === 'response.output_text.delta' && typeof event.delta === 'string') {
+        content += event.delta;
+        yield { type: 'content', content: event.delta };
+        continue;
+      }
+
+      if (event.type === 'response.completed') {
+        finalResponse = event.response;
+      }
+    }
+
+    const response = finalResponse || {};
+    const toolCalls = extractToolCalls(response);
+    const finalContent = extractResponseText(response) || content;
+
+    if (toolCalls.length > 0) {
+      yield {
+        type: 'tool_calls',
+        content: finalContent,
+        toolCalls,
+        usage: response.usage ? {
+          promptTokens: response.usage.input_tokens,
+          completionTokens: response.usage.output_tokens,
+          totalTokens: response.usage.total_tokens,
+        } : null,
+      };
+      return;
+    }
+
+    yield {
+      type: 'done',
+      content: finalContent,
+      toolCalls: [],
+      finishReason: 'stop',
+      usage: response.usage ? {
+        promptTokens: response.usage.input_tokens,
+        completionTokens: response.usage.output_tokens,
+        totalTokens: response.usage.total_tokens,
+      } : null,
+    };
+  }
+}
+
+module.exports = { OpenAICodexProvider };

package/server/services/ai/recordingInsights.js

@@ -6,23 +6,21 @@ const { getSupportedModels } = require('./models');
 const { getAiSettings } = require('./settings');
 const { parseJsonObject } = require('./taskAnalysis');
 
-const INSIGHTS_SYSTEM_PROMPT = `You are an expert audio transcript analyzer. Your job is to read the provided transcript and extract structured insights.
+const INSIGHTS_SYSTEM_PROMPT = `Return JSON only. No markdown, no prose, no code fences.
 
-You must output valid JSON ONLY, with the following exact structure:
+You are a precise conversation analyst. Read the transcript and extract exactly what happened: who said what, what was decided, what needs to happen next, and when.
+
+Schema:
 {
-  "summary": "A concise, 1-2 paragraph summary of the entire conversation.",
-  "action_items": [
-    "List of any action items, tasks, or follow-ups mentioned.",
-    "Be specific and include who is responsible if mentioned."
-  ],
-  "events": [
-    "List of any events, meetings, or dates mentioned in the transcript."
-  ]
+  "summary": "1-2 paragraph factual summary. Name speakers if identifiable. Cover the main topic, key decisions, outcome, and any unresolved items.",
+  "action_items": ["Each item as: '[Owner if named] — specific action'. One item per string. Empty array if none."],
+  "events": ["Each as: '[date/time if stated] — event description'. One event per string. Empty array if none."]
 }
 
-If no action items or events are found, return empty arrays for those fields.
-Do NOT wrap the output in markdown \`\`\`json blocks. ONLY return the raw JSON object.
-`;
+Rules:
+- Report only what the transcript explicitly contains. Do not infer or add context not present in the recording.
+- Be specific: "Alice will send the contract by Friday" beats "follow-up needed".
+- If a field has no data, use an empty array.`;
 
 async function extractRecordingInsights(userId, transcriptText, options = {}) {
   if (!transcriptText || !transcriptText.trim()) {

package/server/services/ai/runEvents.js

@@ -0,0 +1,100 @@
+'use strict';
+
+const db = require('../../db/database');
+
+function parseJsonObject(value, fallback = {}) {
+  if (!value) return { ...fallback };
+  if (typeof value === 'object' && !Array.isArray(value)) return { ...value };
+  try {
+    const parsed = JSON.parse(String(value));
+    return parsed && typeof parsed === 'object' && !Array.isArray(parsed)
+      ? parsed
+      : { ...fallback };
+  } catch {
+    return { ...fallback };
+  }
+}
+
+function recordRunEvent({
+  runId,
+  userId,
+  agentId = null,
+  eventType,
+  requestId = null,
+  stepId = null,
+  sequenceIndex = null,
+  payload = {},
+}) {
+  if (!runId || !userId || !eventType) return null;
+  const payloadJson = JSON.stringify(
+    payload && typeof payload === 'object' && !Array.isArray(payload) ? payload : {},
+  );
+  const row = db.transaction(() => {
+    const resolvedSequence = Number.isInteger(sequenceIndex) && sequenceIndex > 0
+      ? sequenceIndex
+      : Number(
+        db.prepare(
+          'SELECT COALESCE(MAX(sequence_index), 0) AS max_sequence FROM agent_run_events WHERE run_id = ?'
+        ).get(runId)?.max_sequence || 0,
+      ) + 1;
+    const result = db.prepare(
+      `INSERT INTO agent_run_events (
+        run_id, user_id, agent_id, event_type, request_id, step_id, sequence_index, payload_json
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
+    ).run(
+      runId,
+      userId,
+      agentId || null,
+      eventType,
+      requestId || null,
+      stepId || null,
+      resolvedSequence,
+      payloadJson,
+    );
+    return db.prepare(
+      `SELECT id, run_id, user_id, agent_id, event_type, request_id, step_id, sequence_index, payload_json, created_at
+       FROM agent_run_events
+       WHERE id = ?`
+    ).get(result.lastInsertRowid);
+  })();
+
+  return row ? {
+    id: Number(row.id),
+    runId: row.run_id,
+    userId: row.user_id,
+    agentId: row.agent_id || null,
+    eventType: row.event_type,
+    requestId: row.request_id || null,
+    stepId: row.step_id || null,
+    sequenceIndex: Number(row.sequence_index || 0),
+    payload: parseJsonObject(row.payload_json, {}),
+    createdAt: row.created_at,
+  } : null;
+}
+
+function listRunEvents(runId) {
+  if (!runId) return [];
+  const rows = db.prepare(
+    `SELECT id, run_id, user_id, agent_id, event_type, request_id, step_id, sequence_index, payload_json, created_at
+     FROM agent_run_events
+     WHERE run_id = ?
+     ORDER BY sequence_index ASC, id ASC`
+  ).all(runId);
+  return rows.map((row) => ({
+    id: Number(row.id),
+    runId: row.run_id,
+    userId: row.user_id,
+    agentId: row.agent_id || null,
+    eventType: row.event_type,
+    requestId: row.request_id || null,
+    stepId: row.step_id || null,
+    sequenceIndex: Number(row.sequence_index || 0),
+    payload: parseJsonObject(row.payload_json, {}),
+    createdAt: row.created_at,
+  }));
+}
+
+module.exports = {
+  recordRunEvent,
+  listRunEvents,
+};

package/server/services/ai/settings.js

@@ -59,6 +59,26 @@ const AI_PROVIDER_DEFINITIONS = Object.freeze({
     defaultEnabled: false,
     defaultBaseUrl: 'https://api.minimax.io/anthropic'
   },
+  'github-copilot': {
+    id: 'github-copilot',
+    label: 'GitHub Copilot',
+    description: 'Use your GitHub Copilot subscription as an AI provider.',
+    envKey: 'GITHUB_COPILOT_ACCESS_TOKEN',
+    supportsApiKey: true,
+    supportsBaseUrl: true,
+    defaultEnabled: false,
+    defaultBaseUrl: 'https://api.githubcopilot.com'
+  },
+  'openai-codex': {
+    id: 'openai-codex',
+    label: 'OpenAI Codex',
+    description: 'Use Codex models through ChatGPT Codex authentication.',
+    envKey: 'OPENAI_CODEX_ACCESS_TOKEN',
+    supportsApiKey: true,
+    supportsBaseUrl: true,
+    defaultEnabled: false,
+    defaultBaseUrl: 'https://chatgpt.com/backend-api/codex'
+  },
   ollama: {
     id: 'ollama',
     label: 'Ollama',