mixdog 0.7.1

package/src/search/lib/backends/openai-oauth.mjs

@@ -0,0 +1,98 @@
+/**
+ * OpenAI OAuth (Codex backend) search backend.
+ *
+ * Reuses agent.providers.openai-oauth credentials (ChatGPT Pro bearer).
+ * Calls Codex WebSocket endpoint via sendViaWebSocket with web_search server
+ * tool. Model is config-driven (search.models.openai default 'gpt-5.4-mini').
+ */
+import { OpenAIOAuthProvider, ensureLatestCodexModel } from '../../../agent/orchestrator/providers/openai-oauth.mjs'
+import {
+  OPENAI_SEARCH_SYSTEM_INSTRUCTIONS,
+  buildOpenAISearchPrompt,
+  buildOpenAIWebSearchTool,
+  citationsFromText,
+  citationsFromWebSearchCalls,
+} from './openai-web-search.mjs'
+
+const SEARCH_POOL_KEY = 'mixdog-search-openai-oauth'
+const SEARCH_CACHE_KEY = 'mixdog-codex-search'
+
+// Reuse one provider instance across searches so its in-memory token cache
+// survives between calls — avoids re-loading credentials on every search.
+// ensureAuth() still re-validates via mtime + expiry per use.
+let _sharedProvider = null
+function getSharedProvider() {
+  if (!_sharedProvider) _sharedProvider = new OpenAIOAuthProvider({})
+  return _sharedProvider
+}
+
+export async function searchViaOpenAIOAuth({
+  query,
+  model,
+  effort,
+  fast = false,
+  site,
+  type = 'web',
+  maxResults = 5,
+  locale,
+  contextSize = 'low',
+  warnings = [],
+  signal,
+}) {
+  const t0 = Date.now()
+  const provider = getSharedProvider()
+  const useModel = model || await ensureLatestCodexModel(provider)
+  await provider.ensureAuth({ reason: 'search' })
+  const tokens = provider.tokens
+  if (!tokens?.access_token) throw new Error('[search:openai-oauth] no access_token available')
+
+  // Match the bridge invariant: poolKey is the local socket bucket, cacheKey is
+  // the server-side prompt-cache shard. Keep them stable and distinct.
+  const poolKey = SEARCH_POOL_KEY
+  const cacheKey = SEARCH_CACHE_KEY
+  // Effort defaults to 'low' when caller doesn't specify — preserves the
+  // prior hard-coded behavior so older configs without modelOptions still
+  // route through the same low-latency path.
+  const body = {
+    model: useModel,
+    instructions: OPENAI_SEARCH_SYSTEM_INSTRUCTIONS,
+    input: [{ type: 'message', role: 'user', content: [{ type: 'input_text', text: buildOpenAISearchPrompt(query, maxResults) }] }],
+    store: false,
+    stream: true,
+    prompt_cache_key: cacheKey,
+    reasoning: { effort: effort || 'low' },
+    text: { verbosity: 'low' },
+    tool_choice: 'auto',
+    parallel_tool_calls: false,
+    tools: [buildOpenAIWebSearchTool({ site, type, locale, contextSize })],
+  }
+  if (fast === true) body.service_tier = 'priority'
+  // Route through provider.send() (not sendViaWebSocket directly) so the search
+  // request inherits the 401/403 force-refresh retry + HTTP/SSE fallback. A
+  // stale token or unhealthy WebSocket then recovers instead of hard-failing.
+  // _prebuiltBody bypasses buildRequestBody — the web_search server-tool body
+  // shape it can't express is shipped verbatim. poolKey/cacheKey map onto
+  // sessionId/providerCacheKey to preserve the prior socket/cache sharding.
+  const result = await provider.send(null, useModel, null, {
+    _prebuiltBody: body,
+    sessionId: poolKey,
+    providerCacheKey: cacheKey,
+    iteration: 0,
+    signal,
+  })
+  const answer = String(result?.content || '').trim()
+  let citations = Array.isArray(result?.citations) ? result.citations.slice(0, maxResults) : []
+  if (!citations.length) citations = citationsFromText(answer, maxResults, 'openai-oauth')
+  if (!citations.length) citations = citationsFromWebSearchCalls(result?.webSearchCalls, maxResults, 'openai-oauth')
+  return {
+    backend: 'openai-oauth',
+    model: useModel,
+    query,
+    answer,
+    citations,
+    durationMs: Date.now() - t0,
+    usage: result?.usage || null,
+    webSearchCalls: result?.webSearchCalls || [],
+    warnings,
+  }
+}

package/src/search/lib/backends/openai-web-search.mjs

@@ -0,0 +1,76 @@
+export const OPENAI_SEARCH_SYSTEM_INSTRUCTIONS = 'You can use the web_search server tool to look up live information. Reply concisely with the answer and citations.'
+
+export function buildOpenAIWebSearchTool({ site, type = 'web', locale, contextSize = 'low' } = {}) {
+  const tool = {
+    type: 'web_search',
+    external_web_access: true,
+    search_context_size: contextSize || 'low',
+  }
+  if (site) tool.filters = { allowed_domains: [site] }
+  if (locale?.country || locale?.city || locale?.region || locale?.timezone) {
+    tool.user_location = {
+      type: 'approximate',
+      ...(locale.country ? { country: locale.country } : {}),
+      ...(locale.city ? { city: locale.city } : {}),
+      ...(locale.region ? { region: locale.region } : {}),
+      ...(locale.timezone ? { timezone: locale.timezone } : {}),
+    }
+  }
+  if (type === 'images') tool.search_content_types = ['text', 'image']
+  return tool
+}
+
+export function buildOpenAISearchPrompt(query, maxResults = 5) {
+  return [
+    String(query),
+    '',
+    `Return a concise answer and cite at most ${maxResults} source URLs.`,
+    'Put source URLs as plain full URLs if annotations are not available.',
+  ].join('\n')
+}
+
+export function citationsFromText(text, maxResults = 5, source = 'openai-oauth') {
+  const urls = String(text || '').match(/https?:\/\/[^\s<>()\[\]{}"`']+/g) || []
+  const seen = new Set()
+  const citations = []
+  for (const raw of urls) {
+    let parsed
+    try {
+      parsed = new URL(raw.replace(/[.,;:!?]+$/g, ''))
+    } catch {
+      continue
+    }
+    const url = parsed.toString()
+    if (seen.has(url)) continue
+    seen.add(url)
+    citations.push({ title: parsed.hostname, url, snippet: '', source })
+    if (citations.length >= maxResults) break
+  }
+  return citations
+}
+
+export function citationsFromWebSearchCalls(calls, maxResults = 5, source = 'openai-oauth') {
+  const seen = new Set()
+  const citations = []
+  for (const call of Array.isArray(calls) ? calls : []) {
+    const action = call?.action || {}
+    const entries = [
+      ...(action.url ? [action.url] : []),
+      ...(Array.isArray(action.urls) ? action.urls : []),
+    ]
+    for (const raw of entries) {
+      let parsed
+      try {
+        parsed = new URL(String(raw))
+      } catch {
+        continue
+      }
+      const url = parsed.toString()
+      if (seen.has(url)) continue
+      seen.add(url)
+      citations.push({ title: parsed.hostname, url, snippet: '', source })
+      if (citations.length >= maxResults) return citations
+    }
+  }
+  return citations
+}

package/src/search/lib/backends/tavily.mjs

@@ -0,0 +1,55 @@
+/**
+ * Tavily search backend.
+ *
+ * Uses search.credentials.tavily.apiKey. POST /search with include_answer:true
+ * for built-in AI synthesis. RAG-specialized index.
+ */
+import { providerHttpError } from '../state.mjs'
+import { getSearchApiKey } from '../../../shared/config.mjs'
+import { tavilyCountryName } from '../search-intent.mjs'
+
+const URL = 'https://api.tavily.com/search'
+
+export async function searchViaTavily({ query, limit = 5, site, type = 'web', locale, signal }) {
+  const t0 = Date.now()
+  const key = getSearchApiKey('tavily')
+  if (!key) throw new Error('[search:tavily] no api key — register via mixdog-search setup -> search-keys')
+  const topic = type === 'news' ? 'news' : 'general'
+  const body = {
+    api_key: key,
+    query: String(query),
+    topic,
+    search_depth: 'basic',
+    max_results: limit,
+    include_answer: true,
+    include_raw_content: false,
+    ...(site ? { include_domains: [site] } : {}),
+  }
+  if (topic === 'general' && locale?.country) body.country = tavilyCountryName(locale.country)
+
+  const res = await fetch(URL, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify(body),
+    signal,
+  })
+  if (res.status !== 200) {
+    const text = await res.text()
+    throw providerHttpError('tavily', res.status, text)
+  }
+  const j = await res.json()
+  const citations = (j?.results || []).slice(0, limit).map(h => ({
+    title: h.title || '',
+    url: h.url || '',
+    snippet: (h.content || '').slice(0, 240),
+    publishedDate: h.published_date || null,
+    source: 'tavily',
+  }))
+  return {
+    backend: 'tavily',
+    query,
+    answer: (j?.answer || '').trim(),
+    citations,
+    durationMs: Date.now() - t0,
+  }
+}

package/src/search/lib/backends/xai-api.mjs

@@ -0,0 +1,113 @@
+/**
+ * xAI API key search backend.
+ *
+ * API key via getAgentApiKey('xai') (env/keychain/setup). Calls Responses API + web_search
+ * (Agent Tools API) — Live Search (Chat Completions) is deprecated.
+ */
+import { providerHttpError } from '../state.mjs'
+import { getAgentApiKey } from '../../../shared/config.mjs'
+import { resolveLatestGrokModel } from '../../../agent/orchestrator/providers/grok-oauth.mjs'
+
+const URL = 'https://api.x.ai/v1/responses'
+const MODELS_URL = 'https://api.x.ai/v1/models'
+
+// Match resolveLatestGrokModel() in grok-oauth.mjs (do not write OAuth shared cache).
+const NON_CHAT_MODEL_RE = /imagine|image|video/i
+const PROXY_EXACT_MODELS = new Set(['grok-build'])
+function _isProxyOnlyModel(model) {
+  const m = String(model || '')
+  return /^grok-composer/i.test(m) || PROXY_EXACT_MODELS.has(m)
+}
+
+function _normalizeGrokApiModel(m) {
+  const id = m?.id
+  if (!id) return null
+  return {
+    id,
+    name: id,
+    display: id,
+    provider: 'grok-oauth',
+    family: 'grok',
+    tier: 'version',
+    latest: false,
+    contextWindow: m?.context_window || 0,
+    created: typeof m?.created === 'number' ? m.created : null,
+  }
+}
+
+function _pickLatestChatModelId(models) {
+  let best = null
+  for (const m of models) {
+    if (!m?.id || NON_CHAT_MODEL_RE.test(m.id) || _isProxyOnlyModel(m.id) || !(Number(m.created) > 0)) continue
+    if (!best || Number(m.created) > Number(best.created)) best = m
+  }
+  return best?.id || null
+}
+
+async function _fetchGrokCatalogModels(apiKey, signal) {
+  const res = await fetch(MODELS_URL, {
+    method: 'GET',
+    headers: { Authorization: `Bearer ${apiKey}` },
+    redirect: 'error',
+    signal,
+  })
+  if (!res.ok) {
+    const text = await res.text().catch(() => '')
+    throw new Error(`[search:xai-api] model catalog fetch failed (${res.status}${text ? `: ${text.slice(0, 200)}` : ''})`)
+  }
+  const data = await res.json()
+  if (!Array.isArray(data?.data)) {
+    throw new Error('[search:xai-api] unexpected /models response shape (no data[])')
+  }
+  return data.data.map(_normalizeGrokApiModel).filter(Boolean)
+}
+
+async function ensureLatestGrokModelForApiKey(apiKey, signal) {
+  let m = resolveLatestGrokModel()
+  if (m) return m
+  const models = await _fetchGrokCatalogModels(apiKey, signal)
+  m = _pickLatestChatModelId(models)
+  if (m) return m
+  throw new Error(
+    '[search:xai-api] model catalog unavailable after warmup — set search.models.xai to an explicit model id',
+  )
+}
+
+export async function searchViaXAIApi({ query, model, maxResults = 5, warnings = [], signal }) {
+  const t0 = Date.now()
+  const key = getAgentApiKey('xai')
+  if (!key) throw new Error('[search:xai-api] no api key — set XAI_API_KEY or the xAI provider key in setup')
+  const useModel = model || await ensureLatestGrokModelForApiKey(key, signal)
+
+  const res = await fetch(URL, {
+    method: 'POST',
+    headers: { Authorization: `Bearer ${key}`, 'Content-Type': 'application/json' },
+    body: JSON.stringify({
+      model: useModel,
+      input: String(query),
+      tools: [{ type: 'web_search' }],
+    }),
+    signal,
+  })
+  if (res.status !== 200) {
+    const text = await res.text()
+    throw providerHttpError('xai-api', res.status, text)
+  }
+  const j = await res.json()
+  const items = j?.output || []
+  const msg = items.find(it => it.type === 'message')?.content || []
+  const tb = msg.find(c => c.type === 'output_text')
+  const annotations = (tb?.annotations || [])
+    .filter(a => a?.url)
+    .map(a => ({ title: a.title || '', url: a.url || '', snippet: '', source: 'xai-api' }))
+  return {
+    backend: 'xai-api',
+    model: useModel,
+    query,
+    answer: (tb?.text || '').trim(),
+    citations: annotations.slice(0, maxResults),
+    durationMs: Date.now() - t0,
+    usage: j?.usage || null,
+    warnings,
+  }
+}

package/src/search/lib/cache.mjs

@@ -0,0 +1,131 @@
+import crypto from 'crypto'
+import { CACHE_PATH, readJson, writeJson } from './config.mjs'
+
+const DEFAULT_CACHE_STATE = {
+  entries: {},
+}
+
+const FLUSH_DELAY_MS = 5000
+
+let cacheDirty = false
+let cacheFlushTimer = null
+let activeCacheState = null
+let lastCacheFlushWarnAt = 0
+
+function nowMs() {
+  return Date.now()
+}
+
+// 5s debounce so a single search invocation that touches the cache multiple
+// times (lookup + insert + prune) coalesces into one writeJson roundtrip.
+// Without this, callers like crawl/batch that don't explicitly flush would
+// either spam fsync (immediate write per mutation) or silently drop dirty
+// state on crash (bare dirty flag).
+function scheduleCacheFlush(state) {
+  cacheDirty = true
+  activeCacheState = state
+  if (cacheFlushTimer) return
+  cacheFlushTimer = setTimeout(() => {
+    cacheFlushTimer = null
+    flushCacheState()
+  }, FLUSH_DELAY_MS)
+  if (cacheFlushTimer.unref) cacheFlushTimer.unref()
+}
+
+function flushCacheState() {
+  if (cacheFlushTimer) {
+    clearTimeout(cacheFlushTimer)
+    cacheFlushTimer = null
+  }
+  if (cacheDirty && activeCacheState) {
+    try {
+      writeJson(CACHE_PATH, activeCacheState)
+      cacheDirty = false
+    } catch (err) {
+      // Cache state is best-effort. Windows AV/indexer can hold the
+      // destination open. Keep the dirty state and retry quietly.
+      const now = Date.now()
+      if (now - lastCacheFlushWarnAt > 60000) {
+        lastCacheFlushWarnAt = now
+        process.stderr.write(`[search-cache] flushCacheState delayed: ${err?.code || err?.message || err}\n`)
+      }
+      if (!cacheFlushTimer) {
+        cacheFlushTimer = setTimeout(() => {
+          cacheFlushTimer = null
+          flushCacheState()
+        }, FLUSH_DELAY_MS * 2)
+        if (cacheFlushTimer.unref) cacheFlushTimer.unref()
+      }
+    }
+  }
+}
+
+process.on('exit', flushCacheState)
+
+export { flushCacheState }
+
+let _instance = null
+
+export function loadCacheState() {
+  if (_instance) return _instance
+  const state = readJson(CACHE_PATH, DEFAULT_CACHE_STATE)
+  if (!state.entries || typeof state.entries !== 'object') {
+    state.entries = {}
+  }
+  _instance = state
+  activeCacheState = state
+  pruneExpiredEntries(state)
+  return state
+}
+
+export function buildCacheKey(namespace, payload) {
+  const hash = crypto
+    .createHash('sha256')
+    .update(JSON.stringify(payload))
+    .digest('hex')
+  return `${namespace}:${hash}`
+}
+
+export function getCachedEntry(state, key) {
+  const entry = state.entries[key]
+  if (!entry) return null
+  if (entry.expiresAt && entry.expiresAt <= nowMs()) {
+    delete state.entries[key]
+    scheduleCacheFlush(state)
+    return null
+  }
+  return entry
+}
+
+export function setCachedEntry(state, key, payload, ttlMs) {
+  const cachedAt = nowMs()
+  state.entries[key] = {
+    cachedAt,
+    expiresAt: cachedAt + ttlMs,
+    payload,
+  }
+  scheduleCacheFlush(state)
+  return state.entries[key]
+}
+
+export function buildCacheMeta(entry, hit) {
+  return {
+    hit,
+    cachedAt: entry ? new Date(entry.cachedAt).toISOString() : null,
+    expiresAt: entry ? new Date(entry.expiresAt).toISOString() : null,
+  }
+}
+
+function pruneExpiredEntries(state) {
+  const current = nowMs()
+  let dirty = false
+  for (const [key, entry] of Object.entries(state.entries)) {
+    if (entry?.expiresAt && entry.expiresAt <= current) {
+      delete state.entries[key]
+      dirty = true
+    }
+  }
+  if (dirty) {
+    scheduleCacheFlush(state)
+  }
+}

package/src/search/lib/config.mjs

@@ -0,0 +1,192 @@
+import fs from 'fs'
+import path from 'path'
+import { fileURLToPath } from 'url'
+import { resolvePluginData } from '../../shared/plugin-paths.mjs'
+import { renameWithRetrySync, writeJsonAtomicSync } from '../../shared/atomic-file.mjs'
+import { readSection, updateSection, stripGeneratedMarker, CONFIG_PATH as MIXDOG_CONFIG_PATH, getSearchApiKey } from '../../shared/config.mjs'
+
+const currentDir = path.dirname(fileURLToPath(import.meta.url))
+// src/search/lib/config.mjs → plugin root is three levels up (src/search/lib → src/search → src → plugin root).
+export const PLUGIN_ROOT = process.env.CLAUDE_PLUGIN_ROOT || path.resolve(currentDir, '..', '..', '..')
+
+// Unified mode: search shares the plugin data dir with the rest of mixdog.
+const SHARED_DATA_DIR = resolvePluginData()
+if (!SHARED_DATA_DIR) throw new Error('[search-config] resolvePluginData() returned falsy — plugin data dir not configured')
+export const DATA_DIR = SHARED_DATA_DIR
+export const USAGE_PATH = path.join(DATA_DIR, 'usage.local.json')
+export const CACHE_PATH = path.join(DATA_DIR, 'cache.local.json')
+
+// Per-provider default models. Single source of truth for any site that
+// needs a fallback when the user config lacks an explicit override.
+export const DEFAULT_MODELS = {
+  openai: 'gpt-5.5',
+  gemini: 'gemini-3-flash-preview',
+  xai:    'grok-4.3',
+}
+
+export const DEFAULT_CONFIG = {
+  // Single active search backend. Switching here changes which backend the
+  // `search` MCP tool calls — does not require credentials for the unused
+  // backends. No priority chain / fallback — the selected provider is the
+  // only one tried; on failure the call throws.
+  provider: 'anthropic-oauth',
+  // Per-provider model override (only used by providers that take a model arg).
+  // Anthropic OAuth is haiku-fixed (third-party policy + practical quota).
+  models: {},
+  rawSearch: {
+    maxResults: 10,
+    credentials: {
+      firecrawl: {
+        apiKey: '',
+      },
+      tavily: {
+        apiKey: '',
+      },
+      exa: {
+        apiKey: '',
+      },
+    },
+  },
+  requestTimeoutMs: 120000,
+  crawl: {
+    maxPages: 10,
+    maxDepth: 2,
+    sameDomainOnly: true,
+  },
+}
+
+export function ensureDir(dirPath) {
+  fs.mkdirSync(dirPath, { recursive: true })
+}
+
+export function ensureDataDir() {
+  ensureDir(DATA_DIR)
+}
+
+export function readJson(filePath, fallback) {
+  try {
+    const raw = fs.readFileSync(filePath, 'utf8')
+    try {
+      return JSON.parse(raw)
+    } catch (parseErr) {
+      try { renameWithRetrySync(filePath, filePath + '.corrupt.' + Date.now()) } catch {}
+      process.stderr.write(`[search-config] corrupt JSON backed up: ${filePath}\n`)
+      throw parseErr
+    }
+  } catch (err) {
+    if (err.code === 'ENOENT') return fallback
+    throw err
+  }
+}
+
+export function writeJson(filePath, value) {
+  ensureDir(path.dirname(filePath))
+  writeJsonAtomicSync(filePath, value, { lock: true, fsyncDir: true })
+}
+
+function hasKeys(value) {
+  return !!value && typeof value === 'object' && !Array.isArray(value) && Object.keys(value).length > 0
+}
+
+export function saveConfig(config) {
+  updateSection('search', () => stripGeneratedMarker(config) || {})
+}
+
+function finiteInt(value, { min, max, def }) {
+  const n = typeof value === 'number' ? value : Number(value)
+  if (!Number.isFinite(n)) return def
+  const i = Math.trunc(n)
+  if (i < min) return min
+  if (i > max) return max
+  return i
+}
+
+function strictBool(value, def) {
+  if (value === true) return true
+  if (value === false) return false
+  return def
+}
+
+export function loadConfig() {
+  ensureDataDir()
+  let config = readSection('search')
+  if (!hasKeys(config)) {
+    saveConfig(DEFAULT_CONFIG)
+    config = DEFAULT_CONFIG
+    process.stderr.write(
+      `mixdog-search: default config created in ${MIXDOG_CONFIG_PATH} (section: search)\n` +
+      '  use /setup to change provider priority and crawl defaults.\n',
+    )
+  }
+  const merged = {
+    ...DEFAULT_CONFIG,
+    ...config,
+    rawSearch: {
+      ...DEFAULT_CONFIG.rawSearch,
+      ...(config?.rawSearch || {}),
+      credentials: {
+        ...DEFAULT_CONFIG.rawSearch.credentials,
+        ...(config?.rawSearch?.credentials || {}),
+      },
+    },
+    crawl: {
+      ...DEFAULT_CONFIG.crawl,
+      ...(config?.crawl || {}),
+    },
+  }
+  merged.requestTimeoutMs = finiteInt(merged.requestTimeoutMs, {
+    min: 1000,
+    max: 300000,
+    def: DEFAULT_CONFIG.requestTimeoutMs,
+  })
+  merged.crawl.maxPages = finiteInt(merged.crawl.maxPages, {
+    min: 1,
+    max: 200,
+    def: DEFAULT_CONFIG.crawl.maxPages,
+  })
+  merged.crawl.maxDepth = finiteInt(merged.crawl.maxDepth, {
+    min: 0,
+    max: 5,
+    def: DEFAULT_CONFIG.crawl.maxDepth,
+  })
+  merged.crawl.sameDomainOnly = strictBool(merged.crawl.sameDomainOnly, true)
+  return merged
+}
+
+export function getRawSearchMaxResults(config) {
+  return config.rawSearch?.maxResults || DEFAULT_CONFIG.rawSearch.maxResults
+}
+
+export function getRawProviderApiKey(_config, provider) {
+  return getSearchApiKey(provider)
+}
+
+export function getRawProviderCredentialSource(config, provider, env = process.env) {
+  if (getRawProviderApiKey(config, provider)) {
+    return 'config'
+  }
+
+  const envKeyByProvider = {
+    firecrawl: 'FIRECRAWL_API_KEY',
+    tavily: 'TAVILY_API_KEY',
+    'xai-api': ['XAI_API_KEY', 'GROK_API_KEY'],
+  }
+
+  const envKey = envKeyByProvider[provider]
+  if (envKey) {
+    const keys = Array.isArray(envKey) ? envKey : [envKey]
+    if (keys.some(k => env?.[k])) {
+      return 'env'
+    }
+  }
+
+  return null
+}
+
+export function getRequestTimeoutMs(config) {
+  return config.requestTimeoutMs || DEFAULT_CONFIG.requestTimeoutMs
+}
+
+export function getFirecrawlApiKey(config) {
+  return getRawProviderApiKey(config, 'firecrawl') || ''
+}