mixdog 0.7.18 → 0.8.0

package/src/search/lib/backends/openai-oauth.mjs

@@ -1,102 +0,0 @@
-/**
- * OpenAI OAuth (Codex backend) search backend.
- *
- * Reuses agent.providers.openai-oauth credentials (ChatGPT Pro bearer).
- * Calls Codex WebSocket endpoint via sendViaWebSocket with web_search server
- * tool. Model is config-driven (search.models.openai default 'gpt-5.4-mini').
- */
-import {
-  OpenAIOAuthProvider,
-  ensureLatestCodexModel,
-  codexModelSupportsServiceTier,
-} from '../../../agent/orchestrator/providers/openai-oauth.mjs'
-import {
-  OPENAI_SEARCH_SYSTEM_INSTRUCTIONS,
-  buildOpenAISearchPrompt,
-  buildOpenAIWebSearchTool,
-  citationsFromText,
-  citationsFromWebSearchCalls,
-} from './openai-web-search.mjs'
-
-const SEARCH_POOL_KEY = 'mixdog-search-openai-oauth'
-const SEARCH_CACHE_KEY = 'mixdog-codex-search'
-
-// Reuse one provider instance across searches so its in-memory token cache
-// survives between calls — avoids re-loading credentials on every search.
-// ensureAuth() still re-validates via mtime + expiry per use.
-let _sharedProvider = null
-function getSharedProvider() {
-  if (!_sharedProvider) _sharedProvider = new OpenAIOAuthProvider({})
-  return _sharedProvider
-}
-
-export async function searchViaOpenAIOAuth({
-  query,
-  model,
-  effort,
-  fast = false,
-  site,
-  type = 'web',
-  maxResults = 5,
-  locale,
-  contextSize = 'low',
-  warnings = [],
-  signal,
-}) {
-  const t0 = Date.now()
-  const provider = getSharedProvider()
-  const useModel = model || await ensureLatestCodexModel(provider)
-  await provider.ensureAuth({ reason: 'search' })
-  const tokens = provider.tokens
-  if (!tokens?.access_token) throw new Error('[search:openai-oauth] no access_token available')
-
-  // Match the bridge invariant: poolKey is the local socket bucket, cacheKey is
-  // the server-side prompt-cache shard. Keep them stable and distinct.
-  const poolKey = SEARCH_POOL_KEY
-  const cacheKey = SEARCH_CACHE_KEY
-  // Effort defaults to 'low' when caller doesn't specify — preserves the
-  // prior hard-coded behavior so older configs without modelOptions still
-  // route through the same low-latency path.
-  const body = {
-    model: useModel,
-    instructions: OPENAI_SEARCH_SYSTEM_INSTRUCTIONS,
-    input: [{ type: 'message', role: 'user', content: [{ type: 'input_text', text: buildOpenAISearchPrompt(query, maxResults) }] }],
-    store: false,
-    stream: true,
-    prompt_cache_key: cacheKey,
-    reasoning: { effort: effort || 'low' },
-    text: { verbosity: 'low' },
-    tool_choice: 'auto',
-    parallel_tool_calls: false,
-    tools: [buildOpenAIWebSearchTool({ site, type, locale, contextSize })],
-  }
-  if (fast === true && codexModelSupportsServiceTier(useModel, 'priority')) body.service_tier = 'priority'
-  // Route through provider.send() (not sendViaWebSocket directly) so the search
-  // request inherits the 401/403 force-refresh retry + HTTP/SSE fallback. A
-  // stale token or unhealthy WebSocket then recovers instead of hard-failing.
-  // _prebuiltBody bypasses buildRequestBody — the web_search server-tool body
-  // shape it can't express is shipped verbatim. poolKey/cacheKey map onto
-  // sessionId/providerCacheKey to preserve the prior socket/cache sharding.
-  const result = await provider.send(null, useModel, null, {
-    _prebuiltBody: body,
-    sessionId: poolKey,
-    providerCacheKey: cacheKey,
-    iteration: 0,
-    signal,
-  })
-  const answer = String(result?.content || '').trim()
-  let citations = Array.isArray(result?.citations) ? result.citations.slice(0, maxResults) : []
-  if (!citations.length) citations = citationsFromText(answer, maxResults, 'openai-oauth')
-  if (!citations.length) citations = citationsFromWebSearchCalls(result?.webSearchCalls, maxResults, 'openai-oauth')
-  return {
-    backend: 'openai-oauth',
-    model: useModel,
-    query,
-    answer,
-    citations,
-    durationMs: Date.now() - t0,
-    usage: result?.usage || null,
-    webSearchCalls: result?.webSearchCalls || [],
-    warnings,
-  }
-}

package/src/search/lib/backends/openai-web-search.mjs

@@ -1,76 +0,0 @@
-export const OPENAI_SEARCH_SYSTEM_INSTRUCTIONS = 'You can use the web_search server tool to look up live information. Reply concisely with the answer and citations.'
-
-export function buildOpenAIWebSearchTool({ site, type = 'web', locale, contextSize = 'low' } = {}) {
-  const tool = {
-    type: 'web_search',
-    external_web_access: true,
-    search_context_size: contextSize || 'low',
-  }
-  if (site) tool.filters = { allowed_domains: [site] }
-  if (locale?.country || locale?.city || locale?.region || locale?.timezone) {
-    tool.user_location = {
-      type: 'approximate',
-      ...(locale.country ? { country: locale.country } : {}),
-      ...(locale.city ? { city: locale.city } : {}),
-      ...(locale.region ? { region: locale.region } : {}),
-      ...(locale.timezone ? { timezone: locale.timezone } : {}),
-    }
-  }
-  if (type === 'images') tool.search_content_types = ['text', 'image']
-  return tool
-}
-
-export function buildOpenAISearchPrompt(query, maxResults = 5) {
-  return [
-    String(query),
-    '',
-    `Return a concise answer and cite at most ${maxResults} source URLs.`,
-    'Put source URLs as plain full URLs if annotations are not available.',
-  ].join('\n')
-}
-
-export function citationsFromText(text, maxResults = 5, source = 'openai-oauth') {
-  const urls = String(text || '').match(/https?:\/\/[^\s<>()\[\]{}"`']+/g) || []
-  const seen = new Set()
-  const citations = []
-  for (const raw of urls) {
-    let parsed
-    try {
-      parsed = new URL(raw.replace(/[.,;:!?]+$/g, ''))
-    } catch {
-      continue
-    }
-    const url = parsed.toString()
-    if (seen.has(url)) continue
-    seen.add(url)
-    citations.push({ title: parsed.hostname, url, snippet: '', source })
-    if (citations.length >= maxResults) break
-  }
-  return citations
-}
-
-export function citationsFromWebSearchCalls(calls, maxResults = 5, source = 'openai-oauth') {
-  const seen = new Set()
-  const citations = []
-  for (const call of Array.isArray(calls) ? calls : []) {
-    const action = call?.action || {}
-    const entries = [
-      ...(action.url ? [action.url] : []),
-      ...(Array.isArray(action.urls) ? action.urls : []),
-    ]
-    for (const raw of entries) {
-      let parsed
-      try {
-        parsed = new URL(String(raw))
-      } catch {
-        continue
-      }
-      const url = parsed.toString()
-      if (seen.has(url)) continue
-      seen.add(url)
-      citations.push({ title: parsed.hostname, url, snippet: '', source })
-      if (citations.length >= maxResults) return citations
-    }
-  }
-  return citations
-}

package/src/search/lib/backends/tavily.mjs

@@ -1,55 +0,0 @@
-/**
- * Tavily search backend.
- *
- * Uses search.credentials.tavily.apiKey. POST /search with include_answer:true
- * for built-in AI synthesis. RAG-specialized index.
- */
-import { providerHttpError } from '../state.mjs'
-import { getSearchApiKey } from '../../../shared/config.mjs'
-import { tavilyCountryName } from '../search-intent.mjs'
-
-const URL = 'https://api.tavily.com/search'
-
-export async function searchViaTavily({ query, limit = 5, site, type = 'web', locale, signal }) {
-  const t0 = Date.now()
-  const key = getSearchApiKey('tavily')
-  if (!key) throw new Error('[search:tavily] no api key — register via mixdog-search setup -> search-keys')
-  const topic = type === 'news' ? 'news' : 'general'
-  const body = {
-    api_key: key,
-    query: String(query),
-    topic,
-    search_depth: 'basic',
-    max_results: limit,
-    include_answer: true,
-    include_raw_content: false,
-    ...(site ? { include_domains: [site] } : {}),
-  }
-  if (topic === 'general' && locale?.country) body.country = tavilyCountryName(locale.country)
-
-  const res = await fetch(URL, {
-    method: 'POST',
-    headers: { 'Content-Type': 'application/json' },
-    body: JSON.stringify(body),
-    signal,
-  })
-  if (res.status !== 200) {
-    const text = await res.text()
-    throw providerHttpError('tavily', res.status, text)
-  }
-  const j = await res.json()
-  const citations = (j?.results || []).slice(0, limit).map(h => ({
-    title: h.title || '',
-    url: h.url || '',
-    snippet: (h.content || '').slice(0, 240),
-    publishedDate: h.published_date || null,
-    source: 'tavily',
-  }))
-  return {
-    backend: 'tavily',
-    query,
-    answer: (j?.answer || '').trim(),
-    citations,
-    durationMs: Date.now() - t0,
-  }
-}

package/src/search/lib/backends/xai-api.mjs

@@ -1,113 +0,0 @@
-/**
- * xAI API key search backend.
- *
- * API key via getAgentApiKey('xai') (env/keychain/setup). Calls Responses API + web_search
- * (Agent Tools API) — Live Search (Chat Completions) is deprecated.
- */
-import { providerHttpError } from '../state.mjs'
-import { getAgentApiKey } from '../../../shared/config.mjs'
-import { resolveLatestGrokModel } from '../../../agent/orchestrator/providers/grok-oauth.mjs'
-
-const URL = 'https://api.x.ai/v1/responses'
-const MODELS_URL = 'https://api.x.ai/v1/models'
-
-// Match resolveLatestGrokModel() in grok-oauth.mjs (do not write OAuth shared cache).
-const NON_CHAT_MODEL_RE = /imagine|image|video/i
-const PROXY_EXACT_MODELS = new Set(['grok-build'])
-function _isProxyOnlyModel(model) {
-  const m = String(model || '')
-  return /^grok-composer/i.test(m) || PROXY_EXACT_MODELS.has(m)
-}
-
-function _normalizeGrokApiModel(m) {
-  const id = m?.id
-  if (!id) return null
-  return {
-    id,
-    name: id,
-    display: id,
-    provider: 'grok-oauth',
-    family: 'grok',
-    tier: 'version',
-    latest: false,
-    contextWindow: m?.context_window || 0,
-    created: typeof m?.created === 'number' ? m.created : null,
-  }
-}
-
-function _pickLatestChatModelId(models) {
-  let best = null
-  for (const m of models) {
-    if (!m?.id || NON_CHAT_MODEL_RE.test(m.id) || _isProxyOnlyModel(m.id) || !(Number(m.created) > 0)) continue
-    if (!best || Number(m.created) > Number(best.created)) best = m
-  }
-  return best?.id || null
-}
-
-async function _fetchGrokCatalogModels(apiKey, signal) {
-  const res = await fetch(MODELS_URL, {
-    method: 'GET',
-    headers: { Authorization: `Bearer ${apiKey}` },
-    redirect: 'error',
-    signal,
-  })
-  if (!res.ok) {
-    const text = await res.text().catch(() => '')
-    throw new Error(`[search:xai-api] model catalog fetch failed (${res.status}${text ? `: ${text.slice(0, 200)}` : ''})`)
-  }
-  const data = await res.json()
-  if (!Array.isArray(data?.data)) {
-    throw new Error('[search:xai-api] unexpected /models response shape (no data[])')
-  }
-  return data.data.map(_normalizeGrokApiModel).filter(Boolean)
-}
-
-async function ensureLatestGrokModelForApiKey(apiKey, signal) {
-  let m = resolveLatestGrokModel()
-  if (m) return m
-  const models = await _fetchGrokCatalogModels(apiKey, signal)
-  m = _pickLatestChatModelId(models)
-  if (m) return m
-  throw new Error(
-    '[search:xai-api] model catalog unavailable after warmup — set search.models.xai to an explicit model id',
-  )
-}
-
-export async function searchViaXAIApi({ query, model, maxResults = 5, warnings = [], signal }) {
-  const t0 = Date.now()
-  const key = getAgentApiKey('xai')
-  if (!key) throw new Error('[search:xai-api] no api key — set XAI_API_KEY or the xAI provider key in setup')
-  const useModel = model || await ensureLatestGrokModelForApiKey(key, signal)
-
-  const res = await fetch(URL, {
-    method: 'POST',
-    headers: { Authorization: `Bearer ${key}`, 'Content-Type': 'application/json' },
-    body: JSON.stringify({
-      model: useModel,
-      input: String(query),
-      tools: [{ type: 'web_search' }],
-    }),
-    signal,
-  })
-  if (res.status !== 200) {
-    const text = await res.text()
-    throw providerHttpError('xai-api', res.status, text)
-  }
-  const j = await res.json()
-  const items = j?.output || []
-  const msg = items.find(it => it.type === 'message')?.content || []
-  const tb = msg.find(c => c.type === 'output_text')
-  const annotations = (tb?.annotations || [])
-    .filter(a => a?.url)
-    .map(a => ({ title: a.title || '', url: a.url || '', snippet: '', source: 'xai-api' }))
-  return {
-    backend: 'xai-api',
-    model: useModel,
-    query,
-    answer: (tb?.text || '').trim(),
-    citations: annotations.slice(0, maxResults),
-    durationMs: Date.now() - t0,
-    usage: j?.usage || null,
-    warnings,
-  }
-}

package/src/search/lib/config.mjs

@@ -1,192 +0,0 @@
-import fs from 'fs'
-import path from 'path'
-import { fileURLToPath } from 'url'
-import { resolvePluginData } from '../../shared/plugin-paths.mjs'
-import { renameWithRetrySync, writeJsonAtomicSync } from '../../shared/atomic-file.mjs'
-import { readSection, updateSection, stripGeneratedMarker, CONFIG_PATH as MIXDOG_CONFIG_PATH, getSearchApiKey } from '../../shared/config.mjs'
-
-const currentDir = path.dirname(fileURLToPath(import.meta.url))
-// src/search/lib/config.mjs → plugin root is three levels up (src/search/lib → src/search → src → plugin root).
-export const PLUGIN_ROOT = process.env.CLAUDE_PLUGIN_ROOT || path.resolve(currentDir, '..', '..', '..')
-
-// Unified mode: search shares the plugin data dir with the rest of mixdog.
-const SHARED_DATA_DIR = resolvePluginData()
-if (!SHARED_DATA_DIR) throw new Error('[search-config] resolvePluginData() returned falsy — plugin data dir not configured')
-export const DATA_DIR = SHARED_DATA_DIR
-export const USAGE_PATH = path.join(DATA_DIR, 'usage.local.json')
-export const CACHE_PATH = path.join(DATA_DIR, 'cache.local.json')
-
-// Per-provider default models. Single source of truth for any site that
-// needs a fallback when the user config lacks an explicit override.
-export const DEFAULT_MODELS = {
-  openai: 'gpt-5.5',
-  gemini: 'gemini-3-flash-preview',
-  xai:    'grok-4.3',
-}
-
-export const DEFAULT_CONFIG = {
-  // Single active search backend. Switching here changes which backend the
-  // `search` MCP tool calls — does not require credentials for the unused
-  // backends. No priority chain / fallback — the selected provider is the
-  // only one tried; on failure the call throws.
-  provider: 'anthropic-oauth',
-  // Per-provider model override (only used by providers that take a model arg).
-  // Anthropic OAuth is haiku-fixed (third-party policy + practical quota).
-  models: {},
-  rawSearch: {
-    maxResults: 10,
-    credentials: {
-      firecrawl: {
-        apiKey: '',
-      },
-      tavily: {
-        apiKey: '',
-      },
-      exa: {
-        apiKey: '',
-      },
-    },
-  },
-  requestTimeoutMs: 120000,
-  crawl: {
-    maxPages: 10,
-    maxDepth: 2,
-    sameDomainOnly: true,
-  },
-}
-
-export function ensureDir(dirPath) {
-  fs.mkdirSync(dirPath, { recursive: true })
-}
-
-export function ensureDataDir() {
-  ensureDir(DATA_DIR)
-}
-
-export function readJson(filePath, fallback) {
-  try {
-    const raw = fs.readFileSync(filePath, 'utf8')
-    try {
-      return JSON.parse(raw)
-    } catch (parseErr) {
-      try { renameWithRetrySync(filePath, filePath + '.corrupt.' + Date.now()) } catch {}
-      process.stderr.write(`[search-config] corrupt JSON backed up: ${filePath}\n`)
-      throw parseErr
-    }
-  } catch (err) {
-    if (err.code === 'ENOENT') return fallback
-    throw err
-  }
-}
-
-export function writeJson(filePath, value) {
-  ensureDir(path.dirname(filePath))
-  writeJsonAtomicSync(filePath, value, { lock: true, fsyncDir: true })
-}
-
-function hasKeys(value) {
-  return !!value && typeof value === 'object' && !Array.isArray(value) && Object.keys(value).length > 0
-}
-
-export function saveConfig(config) {
-  updateSection('search', () => stripGeneratedMarker(config) || {})
-}
-
-function finiteInt(value, { min, max, def }) {
-  const n = typeof value === 'number' ? value : Number(value)
-  if (!Number.isFinite(n)) return def
-  const i = Math.trunc(n)
-  if (i < min) return min
-  if (i > max) return max
-  return i
-}
-
-function strictBool(value, def) {
-  if (value === true) return true
-  if (value === false) return false
-  return def
-}
-
-export function loadConfig() {
-  ensureDataDir()
-  let config = readSection('search')
-  if (!hasKeys(config)) {
-    saveConfig(DEFAULT_CONFIG)
-    config = DEFAULT_CONFIG
-    process.stderr.write(
-      `mixdog-search: default config created in ${MIXDOG_CONFIG_PATH} (section: search)\n` +
-      '  use /setup to change provider priority and crawl defaults.\n',
-    )
-  }
-  const merged = {
-    ...DEFAULT_CONFIG,
-    ...config,
-    rawSearch: {
-      ...DEFAULT_CONFIG.rawSearch,
-      ...(config?.rawSearch || {}),
-      credentials: {
-        ...DEFAULT_CONFIG.rawSearch.credentials,
-        ...(config?.rawSearch?.credentials || {}),
-      },
-    },
-    crawl: {
-      ...DEFAULT_CONFIG.crawl,
-      ...(config?.crawl || {}),
-    },
-  }
-  merged.requestTimeoutMs = finiteInt(merged.requestTimeoutMs, {
-    min: 1000,
-    max: 300000,
-    def: DEFAULT_CONFIG.requestTimeoutMs,
-  })
-  merged.crawl.maxPages = finiteInt(merged.crawl.maxPages, {
-    min: 1,
-    max: 200,
-    def: DEFAULT_CONFIG.crawl.maxPages,
-  })
-  merged.crawl.maxDepth = finiteInt(merged.crawl.maxDepth, {
-    min: 0,
-    max: 5,
-    def: DEFAULT_CONFIG.crawl.maxDepth,
-  })
-  merged.crawl.sameDomainOnly = strictBool(merged.crawl.sameDomainOnly, true)
-  return merged
-}
-
-export function getRawSearchMaxResults(config) {
-  return config.rawSearch?.maxResults || DEFAULT_CONFIG.rawSearch.maxResults
-}
-
-export function getRawProviderApiKey(_config, provider) {
-  return getSearchApiKey(provider)
-}
-
-export function getRawProviderCredentialSource(config, provider, env = process.env) {
-  if (getRawProviderApiKey(config, provider)) {
-    return 'config'
-  }
-
-  const envKeyByProvider = {
-    firecrawl: 'FIRECRAWL_API_KEY',
-    tavily: 'TAVILY_API_KEY',
-    'xai-api': ['XAI_API_KEY', 'GROK_API_KEY'],
-  }
-
-  const envKey = envKeyByProvider[provider]
-  if (envKey) {
-    const keys = Array.isArray(envKey) ? envKey : [envKey]
-    if (keys.some(k => env?.[k])) {
-      return 'env'
-    }
-  }
-
-  return null
-}
-
-export function getRequestTimeoutMs(config) {
-  return config.requestTimeoutMs || DEFAULT_CONFIG.requestTimeoutMs
-}
-
-export function getFirecrawlApiKey(config) {
-  return getRawProviderApiKey(config, 'firecrawl') || ''
-}

package/src/search/lib/provider-usage.mjs

@@ -1,67 +0,0 @@
-async function fetchJson(url, apiKey) {
-  const response = await fetch(url, {
-    signal: AbortSignal.timeout(10_000),
-    headers: {
-      Authorization: `Bearer ${apiKey}`,
-    },
-  })
-
-  if (!response.ok) {
-    // Drain/cancel the body before throwing so the underlying socket isn't
-    // held open until GC.
-    try { await response.body?.cancel() } catch {}
-    throw new Error(`Usage request failed: ${response.status}`)
-  }
-
-  return response.json()
-}
-
-async function fetchFirecrawlUsage(apiKey) {
-  if (!apiKey) return null
-  const payload = await fetchJson('https://api.firecrawl.dev/v2/team/credit-usage', apiKey)
-  const data = payload?.data
-  if (!data) return null
-
-  return {
-    remaining: typeof data.remainingCredits === 'number' ? data.remainingCredits : null,
-    limit:
-      typeof data.planCredits === 'number' && data.planCredits > 0
-        ? data.planCredits
-        : null,
-    resetAt: data.billingPeriodEnd || null,
-  }
-}
-
-async function fetchTavilyUsage(apiKey) {
-  if (!apiKey) return null
-  const payload = await fetchJson('https://api.tavily.com/usage', apiKey)
-  const key = payload?.key
-  if (!key) return null
-
-  const usage = typeof key.usage === 'number' ? key.usage : null
-  const limit =
-    typeof key.limit === 'number' && key.limit > 0
-      ? key.limit
-      : null
-
-  return {
-    remaining: usage !== null && limit !== null ? Math.max(limit - usage, 0) : null,
-    limit,
-    resetAt: null,
-  }
-}
-
-export async function fetchProviderUsageSnapshot(provider, env = process.env) {
-  switch (provider) {
-    case 'firecrawl':
-      return fetchFirecrawlUsage(env.FIRECRAWL_API_KEY)
-    case 'tavily':
-      return fetchTavilyUsage(env.TAVILY_API_KEY)
-    default:
-      // No telemetry endpoint just means "no usage patch to apply" — it does
-      // NOT mean the provider is unavailable. Returning {available:false}
-      // here would spread over the {available:true} baseline in
-      // writeStartupSnapshot and mark a configured provider as down.
-      return null
-  }
-}

package/src/search/lib/providers.mjs

@@ -1,47 +0,0 @@
-export const PROVIDER_OVERRIDE_ENUM = [
-  'anthropic-oauth',
-  'openai-oauth',
-  'openai-api',
-  'gemini-api',
-  'xai-api',
-  'grok-oauth',
-  'tavily',
-  'firecrawl',
-  'exa',
-]
-
-export const RAW_PROVIDER_CAPABILITIES = {
-  firecrawl: {
-    usageSupport: {
-      available: true,
-      timestamps: true,
-      cost: false,
-      quota: true,
-    },
-  },
-  tavily: {
-    usageSupport: {
-      available: true,
-      timestamps: true,
-      cost: false,
-      quota: true,
-    },
-  },
-  'xai-api': {
-    usageSupport: {
-      available: true,
-      timestamps: true,
-      cost: true,
-      quota: false,
-    },
-  },
-}
-
-export function getProvidersWithApiKeys(env = process.env) {
-  const providers = []
-  if (env.FIRECRAWL_API_KEY) providers.push('firecrawl')
-  if (env.TAVILY_API_KEY) providers.push('tavily')
-  if (env.EXA_API_KEY) providers.push('exa')
-  if (env.XAI_API_KEY || env.GROK_API_KEY) providers.push('xai-api')
-  return providers
-}