mixdog 0.7.1

package/src/search/lib/formatter.mjs

@@ -0,0 +1,115 @@
+/**
+ * Response formatter — strips metadata, returns human-readable text.
+ */
+
+function formatSearchResults(data) {
+  // data may be the full jsonText payload: { tool, providers, response, cache, ... }
+  // response.results is the array we care about
+  const response = data.response || data
+  const results = response.results || []
+  const answer = String(response.answer || '').trim()
+  const warnings = Array.isArray(response.warnings) ? response.warnings : []
+
+  if (!results.length && !answer) {
+    return '(no search results)'
+  }
+
+  // Explicit display caps — callers that need full payload should use raw JSON output.
+  const TITLE_CAP = 200
+  const SNIPPET_CAP = 600
+  const ANSWER_CAP = 4000
+  const clip = (text, cap) => text.length > cap ? `${text.slice(0, cap)}…` : text
+  const blocks = []
+  if (warnings.length) blocks.push(`Warnings: ${warnings.join('; ')}`)
+  if (answer) blocks.push(clip(answer, ANSWER_CAP))
+  if (!results.length) return blocks.join('\n\n')
+  blocks.push(results
+    .map((r, i) => {
+      const num = i + 1
+      const title = clip(r.title || '(no title)', TITLE_CAP)
+      const url = r.url || ''
+      const date = r.publishedDate || ''
+      const snippet = clip((r.snippet || '').trim(), SNIPPET_CAP)
+
+      const urlPart = [url, date].filter(Boolean).join(' — ')
+      const lines = [`${num}. ${title}`]
+      if (urlPart) lines.push(`   ${urlPart}`)
+      if (snippet) lines.push(`   ${snippet}`)
+      return lines.join('\n')
+    })
+    .join('\n\n'))
+  return blocks.join('\n\n')
+}
+
+function formatCrawl(data) {
+  // data: { tool, pages: [{ url, depth, title, excerpt, extractor } | { url, depth, error }] }
+  const pages = data.pages || []
+
+  if (!pages.length) {
+    return '(no crawl results)'
+  }
+
+  return pages
+    .map(page => {
+      const url = page.url || ''
+      const title = page.title || ''
+      const excerpt = (page.excerpt || '').trim()
+      const error = page.error
+
+      if (error) {
+        return `[${url}]\n(failed)`
+      }
+
+      const header = title ? `[${title}] ${url}` : `[${url}]`
+      return `${header}\n${excerpt || '(no content)'}`
+    })
+    .join('\n\n---\n\n')
+}
+
+function formatFetch(data) {
+  const results = data.results || []
+  if (!results.length) return '(no fetch results)'
+  const cappedNote = data.urlsTruncated ? `[fetched first ${results.length} of ${data.urlsTruncated} URLs; raise FETCH_URL_CAP for more]\n\n` : ''
+
+  return cappedNote + results
+    .map(item => {
+      const url = item.url || ''
+      if (item.status === 'error' || item.error) {
+        return `[${url}]\n(error: ${item.error || 'unknown error'})`
+      }
+      const meta = []
+      if (Number.isFinite(item.bytes)) meta.push(`${item.bytes} bytes`)
+      if (Number.isFinite(item.totalLength) && item.range) {
+        meta.push(`range=${item.range.startIndex}..${item.range.endIndex}/${item.totalLength}`)
+      }
+      if (item.hasMore && item.nextStartIndex != null) {
+        meta.push(`next startIndex=${item.nextStartIndex}`)
+      }
+      if (Number.isFinite(item.durationMs)) meta.push(`${item.durationMs}ms`)
+      const header = `${url}${meta.length ? ` (${meta.join(', ')})` : ''}`
+      const titleRaw = String(item.title || '').replace(/\s+/g, ' ').trim()
+      const titleLine = titleRaw ? `title: ${titleRaw}` : ''
+      const body = String(item.content || '').trim() || '(no content)'
+      return titleLine ? `${header}\n${titleLine}\n${body}` : `${header}\n${body}`
+    })
+    .join('\n\n---\n\n')
+}
+
+/**
+ * Format a tool response into human-readable text.
+ * @param {string} tool - Tool name (search, fetch, crawl)
+ * @param {object} rawResult - The raw result object that was previously passed to jsonText()
+ * @returns {string} Formatted text
+ */
+export function formatResponse(tool, rawResult) {
+  switch (tool) {
+    case 'search':
+      return formatSearchResults(rawResult)
+    case 'crawl':
+      return formatCrawl(rawResult)
+    case 'fetch':
+      return formatFetch(rawResult)
+    default:
+      return JSON.stringify(rawResult, null, 2)
+  }
+}

package/src/search/lib/provider-usage.mjs

@@ -0,0 +1,67 @@
+async function fetchJson(url, apiKey) {
+  const response = await fetch(url, {
+    signal: AbortSignal.timeout(10_000),
+    headers: {
+      Authorization: `Bearer ${apiKey}`,
+    },
+  })
+
+  if (!response.ok) {
+    // Drain/cancel the body before throwing so the underlying socket isn't
+    // held open until GC.
+    try { await response.body?.cancel() } catch {}
+    throw new Error(`Usage request failed: ${response.status}`)
+  }
+
+  return response.json()
+}
+
+async function fetchFirecrawlUsage(apiKey) {
+  if (!apiKey) return null
+  const payload = await fetchJson('https://api.firecrawl.dev/v2/team/credit-usage', apiKey)
+  const data = payload?.data
+  if (!data) return null
+
+  return {
+    remaining: typeof data.remainingCredits === 'number' ? data.remainingCredits : null,
+    limit:
+      typeof data.planCredits === 'number' && data.planCredits > 0
+        ? data.planCredits
+        : null,
+    resetAt: data.billingPeriodEnd || null,
+  }
+}
+
+async function fetchTavilyUsage(apiKey) {
+  if (!apiKey) return null
+  const payload = await fetchJson('https://api.tavily.com/usage', apiKey)
+  const key = payload?.key
+  if (!key) return null
+
+  const usage = typeof key.usage === 'number' ? key.usage : null
+  const limit =
+    typeof key.limit === 'number' && key.limit > 0
+      ? key.limit
+      : null
+
+  return {
+    remaining: usage !== null && limit !== null ? Math.max(limit - usage, 0) : null,
+    limit,
+    resetAt: null,
+  }
+}
+
+export async function fetchProviderUsageSnapshot(provider, env = process.env) {
+  switch (provider) {
+    case 'firecrawl':
+      return fetchFirecrawlUsage(env.FIRECRAWL_API_KEY)
+    case 'tavily':
+      return fetchTavilyUsage(env.TAVILY_API_KEY)
+    default:
+      // No telemetry endpoint just means "no usage patch to apply" — it does
+      // NOT mean the provider is unavailable. Returning {available:false}
+      // here would spread over the {available:true} baseline in
+      // writeStartupSnapshot and mark a configured provider as down.
+      return null
+  }
+}

package/src/search/lib/providers.mjs

@@ -0,0 +1,47 @@
+export const PROVIDER_OVERRIDE_ENUM = [
+  'anthropic-oauth',
+  'openai-oauth',
+  'openai-api',
+  'gemini-api',
+  'xai-api',
+  'grok-oauth',
+  'tavily',
+  'firecrawl',
+  'exa',
+]
+
+export const RAW_PROVIDER_CAPABILITIES = {
+  firecrawl: {
+    usageSupport: {
+      available: true,
+      timestamps: true,
+      cost: false,
+      quota: true,
+    },
+  },
+  tavily: {
+    usageSupport: {
+      available: true,
+      timestamps: true,
+      cost: false,
+      quota: true,
+    },
+  },
+  'xai-api': {
+    usageSupport: {
+      available: true,
+      timestamps: true,
+      cost: true,
+      quota: false,
+    },
+  },
+}
+
+export function getProvidersWithApiKeys(env = process.env) {
+  const providers = []
+  if (env.FIRECRAWL_API_KEY) providers.push('firecrawl')
+  if (env.TAVILY_API_KEY) providers.push('tavily')
+  if (env.EXA_API_KEY) providers.push('exa')
+  if (env.XAI_API_KEY || env.GROK_API_KEY) providers.push('xai-api')
+  return providers
+}

package/src/search/lib/search-intent.mjs

@@ -0,0 +1,109 @@
+const SEARCH_TYPES = Object.freeze(['web', 'news', 'images'])
+const SEARCH_CONTEXT_SIZES = Object.freeze(['low', 'medium', 'high'])
+
+function normalizeLocale(value) {
+  if (!value) return null
+  if (typeof value === 'object' && !Array.isArray(value)) {
+    const locale = {
+      country: value.country ? String(value.country).trim().toUpperCase() : undefined,
+      language: value.language ? String(value.language).trim().toLowerCase() : undefined,
+      region: value.region ? String(value.region).trim() : undefined,
+      city: value.city ? String(value.city).trim() : undefined,
+      timezone: value.timezone ? String(value.timezone).trim() : undefined,
+      explicit: true,
+    }
+    return Object.fromEntries(Object.entries(locale).filter(([, v]) => v !== undefined && v !== ''))
+  }
+  const raw = String(value || '').trim().replace(/_/g, '-')
+  if (!raw) return null
+  const parts = raw.split('-').filter(Boolean)
+  if (parts.length >= 2) {
+    return { language: parts[0].toLowerCase(), country: parts[1].toUpperCase(), explicit: true }
+  }
+  const single = parts[0] || raw
+  if (/^[A-Z]{2}$/.test(single)) return { country: single, explicit: true }
+  if (/^[a-z]{2}$/.test(single)) return { language: single, explicit: true }
+  return { country: single.toUpperCase(), explicit: true }
+}
+
+function clampMaxResults(value, fallback = 5, max = 20) {
+  const n = Number(value)
+  const base = Number.isFinite(n) ? Math.floor(n) : Number(fallback)
+  const safe = Number.isFinite(base) && base > 0 ? base : 5
+  return Math.max(1, Math.min(max, safe))
+}
+
+function normalizeContextSize(value, fallback = 'low') {
+  const raw = String(value || '').trim().toLowerCase()
+  if (SEARCH_CONTEXT_SIZES.includes(raw)) return raw
+  const fb = String(fallback || '').trim().toLowerCase()
+  return SEARCH_CONTEXT_SIZES.includes(fb) ? fb : 'low'
+}
+
+function normalizeSite(site) {
+  const raw = String(site || '').trim()
+  if (!raw) return ''
+  const withoutPrefix = raw.replace(/^site:/i, '').trim()
+  try {
+    const parsed = new URL(withoutPrefix.includes('://') ? withoutPrefix : `https://${withoutPrefix}`)
+    return parsed.hostname.replace(/^www\./i, '').toLowerCase()
+  } catch {
+    return withoutPrefix
+      .replace(/^https?:\/\//i, '')
+      .replace(/^www\./i, '')
+      .replace(/\/.*$/, '')
+      .trim()
+      .toLowerCase()
+  }
+}
+
+export function normalizeSearchIntent(raw = {}, { caps = {}, defaultMaxResults = 5 } = {}) {
+  const rawQuery = String(raw.query ?? raw.keywords ?? '').trim()
+  if (!rawQuery) throw new Error('query is required')
+
+  const supportedTypes = Array.isArray(caps.searchTypes) && caps.searchTypes.length
+    ? caps.searchTypes
+    : ['web']
+  const requestedType = SEARCH_TYPES.includes(String(raw.type || '').trim())
+    ? String(raw.type).trim()
+    : 'web'
+  const type = supportedTypes.includes(requestedType) ? requestedType : 'web'
+  const site = normalizeSite(raw.site)
+  const queryWithSite = site && !/\bsite:/i.test(rawQuery)
+    ? `${rawQuery} site:${site}`
+    : rawQuery
+  const warnings = []
+  if (requestedType !== type) {
+    warnings.push(`search type "${requestedType}" is not supported by this provider; used "${type}"`)
+  }
+  const locale = normalizeLocale(raw.locale)
+  if (locale && caps.localeMode === 'none') {
+    warnings.push('locale is not supported by this provider; ignored')
+  }
+
+  return {
+    query: rawQuery,
+    queryWithSite,
+    rawQuery,
+    site,
+    type,
+    requestedType,
+    maxResults: clampMaxResults(raw.maxResults, defaultMaxResults),
+    contextSize: normalizeContextSize(raw.contextSize ?? raw.searchContextSize, raw.defaultContextSize || 'low'),
+    locale,
+    warnings,
+  }
+}
+
+export function tavilyCountryName(country) {
+  const code = String(country || 'US').toUpperCase()
+  const names = {
+    CN: 'china',
+    JP: 'japan',
+    KR: 'south korea',
+    RU: 'russia',
+    SA: 'saudi arabia',
+    US: 'united states',
+  }
+  return names[code] || 'united states'
+}

package/src/search/lib/setup-handler.mjs

@@ -0,0 +1,261 @@
+import { loadConfig, saveConfig } from './config.mjs'
+import { getSearchApiKey, saveSecret, SECRET_ACCOUNTS } from '../../shared/config.mjs'
+import { DEFAULT_MODELS } from './config.mjs'
+import { PROVIDER_OVERRIDE_ENUM } from './providers.mjs'
+
+const PROVIDER_CHOICES = PROVIDER_OVERRIDE_ENUM
+
+const SEARCH_KEY_PROVIDERS = ['firecrawl', 'tavily', 'exa']
+
+function mask(key) {
+  if (!key) return '  not set'
+  return '  ****' + key.slice(-4)
+}
+
+function icon(key) {
+  return key ? '●' : '○'
+}
+
+function statusBlock(config) {
+  const activeProvider = config.provider || 'anthropic-oauth'
+  const models = config.models || {}
+
+  const lines = [
+    '',
+    '  ╭───────────────────────────────────────╮',
+    '  │  mixdog-search config                   │',
+    '  ╰───────────────────────────────────────╯',
+    '',
+    '  Active Backend',
+    '  ────────────────────────────────────────',
+    `    provider     ${activeProvider}`,
+  ]
+  if (activeProvider === 'openai-oauth' || activeProvider === 'openai-api') {
+    lines.push(`    openai model ${models.openai || 'gpt-5.5'}`)
+  } else if (activeProvider === 'gemini-api') {
+    lines.push(`    gemini model ${models.gemini || DEFAULT_MODELS.gemini}`)
+  } else if (activeProvider === 'xai-api') {
+    lines.push(`    xai model    ${models.xai || 'grok-4.3'}`)
+  } else if (activeProvider === 'anthropic-oauth') {
+    lines.push(`    model        claude-haiku-4-5 (fixed)`)
+  }
+
+  lines.push('')
+  lines.push('  Search-Specific Keys')
+  lines.push('  ────────────────────────────────────────')
+  for (const p of SEARCH_KEY_PROVIDERS) {
+    const key = getSearchApiKey(p)
+    lines.push(`    ${icon(key)} ${p.padEnd(12)}${mask(key)}`)
+  }
+  lines.push('')
+  lines.push('  Options')
+  lines.push('  ────────────────────────────────────────')
+  lines.push(`    max results ${config.rawSearch?.maxResults || 10}`)
+  lines.push(`    crawl       ${config.crawl?.maxPages || 10} pages / depth ${config.crawl?.maxDepth || 1}`)
+  lines.push('')
+  return lines.join('\n')
+}
+
+function sectionHeader(config) {
+  const activeProvider = config.provider || 'anthropic-oauth'
+  const total = SEARCH_KEY_PROVIDERS.filter(p => !!getSearchApiKey(p)).length
+  return [
+    '  ╭───────────────────────────────────────╮',
+    '  │  mixdog-search setup                    │',
+    '  ╰───────────────────────────────────────╯',
+    '',
+    `    active provider: ${activeProvider}`,
+    `    ${total > 0 ? '●' : '○'} ${total} search-specific key(s) configured`,
+    '',
+  ].join('\n')
+}
+
+function keysHeader(title, entries) {
+  const lines = [
+    '  ╭───────────────────────────────────────╮',
+    `  │  ${title.padEnd(37)}│`,
+    '  ╰───────────────────────────────────────╯',
+    '',
+    '    empty = keep current / "clear" = remove',
+    '',
+  ]
+  for (const [name, key] of entries) {
+    lines.push(`    ${icon(key)} ${name.padEnd(12)}${mask(key)}`)
+  }
+  return lines.join('\n')
+}
+
+function applyKeys(config, data) {
+  for (const [provider, value] of Object.entries(data)) {
+    if (value == null) continue
+    const trimmed = String(value).trim()
+    if (!trimmed) continue
+    const isClear = trimmed.toLowerCase() === 'clear'
+    if (isClear) {
+      saveSecret(SECRET_ACCOUNTS.searchApiKey(provider), '')
+    } else {
+      // Validate: API keys must be non-empty printable ASCII, no whitespace
+      if (!/^[\x21-\x7E]+$/.test(trimmed)) {
+        throw new Error(`[setup] invalid API key for ${provider}: must be printable ASCII with no whitespace`)
+      }
+      saveSecret(SECRET_ACCOUNTS.searchApiKey(provider), trimmed)
+    }
+    // Remove plaintext copy from config object before save
+    if (config.rawSearch?.credentials?.[provider]?.apiKey != null) {
+      delete config.rawSearch.credentials[provider].apiKey
+    }
+  }
+}
+
+function save(config) {
+  saveConfig(config)
+}
+
+export async function handleSetup(server) {
+  const config = loadConfig()
+
+  const step1 = await server.elicitInput({
+    message: sectionHeader(config),
+    requestedSchema: {
+      type: 'object',
+      properties: {
+        section: {
+          type: 'string',
+          title: 'Section',
+          enum: ['provider', 'search-keys', 'options', 'status'],
+        },
+      },
+      required: ['section'],
+    },
+  })
+
+  if (step1.action !== 'accept') {
+    return { content: [{ type: 'text', text: statusBlock(config) }] }
+  }
+
+  const section = step1.content.section
+
+  if (section === 'status') {
+    return { content: [{ type: 'text', text: statusBlock(config) }] }
+  }
+
+  if (section === 'provider') {
+    const result = await server.elicitInput({
+      message: [
+        '  ╭───────────────────────────────────────╮',
+        '  │  Active Search Provider               │',
+        '  ╰───────────────────────────────────────╯',
+        '',
+        `    current: ${config.provider || 'anthropic-oauth'}`,
+        '',
+        '    OAuth backends reuse the matching agent.providers',
+        '    OAuth tokens; search-specific backends (firecrawl,',
+        '    tavily, exa) use the keys set in `search-keys`.',
+        '',
+      ].join('\n'),
+      requestedSchema: {
+        type: 'object',
+        properties: {
+          provider: {
+            type: 'string',
+            title: 'Search Provider',
+            enum: PROVIDER_CHOICES,
+          },
+          openaiModel: {
+            type: 'string',
+            title: 'OpenAI model preset (used when provider=openai-oauth/openai-api)',
+            enum: ['gpt-5.5', 'gpt-5.4', 'gpt-5.4-mini', 'gpt-4o', 'gpt-4.1'],
+          },
+          geminiModel: {
+            type: 'string',
+            title: 'Gemini model preset (used when provider=gemini-api)',
+            enum: ['gemini-3-flash-preview', 'gemini-3.1-flash', 'gemini-3-flash', 'gemini-2.5-flash', 'gemini-2.5-pro'],
+          },
+          xaiModel: {
+            type: 'string',
+            title: 'xAI model preset (used when provider=xai-api)',
+            enum: ['grok-4.3', 'grok-4.20-fast', 'grok-composer-2.5-fast'],
+          },
+        },
+      },
+    })
+    if (result.action === 'accept' && result.content) {
+      const d = result.content
+      if (typeof d.provider === 'string' && PROVIDER_CHOICES.includes(d.provider)) {
+        config.provider = d.provider
+      }
+      if (!config.models) config.models = {}
+      if (typeof d.openaiModel === 'string' && d.openaiModel.trim()) config.models.openai = d.openaiModel.trim()
+      if (typeof d.geminiModel === 'string' && d.geminiModel.trim()) config.models.gemini = d.geminiModel.trim()
+      if (typeof d.xaiModel === 'string' && d.xaiModel.trim()) config.models.xai = d.xaiModel.trim()
+      save(config)
+      return { content: [{ type: 'text', text: '  ✓ Provider saved.\n' + statusBlock(loadConfig()) }] }
+    }
+    return { content: [{ type: 'text', text: '  ⏎ Cancelled.' }] }
+  }
+
+  if (section === 'search-keys') {
+    const result = await server.elicitInput({
+      message: keysHeader('Search-Specific Keys', [
+        ['firecrawl', getSearchApiKey('firecrawl')],
+        ['tavily', getSearchApiKey('tavily')],
+        ['exa', getSearchApiKey('exa')],
+      ]),
+      requestedSchema: {
+        type: 'object',
+        properties: {
+          firecrawl: { type: 'string', title: 'Firecrawl' },
+          tavily: { type: 'string', title: 'Tavily' },
+          exa: { type: 'string', title: 'Exa' },
+        },
+      },
+    })
+
+    if (result.action === 'accept' && result.content) {
+      try {
+        applyKeys(config, result.content)
+      } catch (err) {
+        const message = err instanceof Error ? err.message : String(err)
+        return { content: [{ type: 'text', text: `  ✗ Search keys not saved: ${message}` }] }
+      }
+      save(config)
+      return { content: [{ type: 'text', text: '  ✓ Search keys saved.\n' + statusBlock(loadConfig()) }] }
+    }
+    return { content: [{ type: 'text', text: '  ⏎ Cancelled.' }] }
+  }
+
+  if (section === 'options') {
+    const result = await server.elicitInput({
+      message: [
+        '  ╭───────────────────────────────────────╮',
+        '  │  Search Options                       │',
+        '  ╰───────────────────────────────────────╯',
+        '',
+        `    max results  ${config.rawSearch?.maxResults || 10}`,
+        `    crawl pages  ${config.crawl?.maxPages || 10}`,
+        `    crawl depth  ${config.crawl?.maxDepth || 1}`,
+        `    same domain  ${config.crawl?.sameDomainOnly ?? true}`,
+      ].join('\n'),
+      requestedSchema: {
+        type: 'object',
+        properties: {
+          maxResults: { type: 'integer', title: 'Max search results' },
+          crawlMaxPages: { type: 'integer', title: 'Crawl max pages' },
+          crawlMaxDepth: { type: 'integer', title: 'Crawl max depth' },
+          sameDomainOnly: { type: 'boolean', title: 'Same domain only' },
+        },
+      },
+    })
+
+    if (result.action === 'accept' && result.content) {
+      const d = result.content
+      if (d.maxResults != null) { if (!config.rawSearch) config.rawSearch = {}; config.rawSearch.maxResults = d.maxResults }
+      if (d.crawlMaxPages != null) { if (!config.crawl) config.crawl = {}; config.crawl.maxPages = d.crawlMaxPages }
+      if (d.crawlMaxDepth != null) { if (!config.crawl) config.crawl = {}; config.crawl.maxDepth = d.crawlMaxDepth }
+      if (d.sameDomainOnly != null) { if (!config.crawl) config.crawl = {}; config.crawl.sameDomainOnly = d.sameDomainOnly }
+      save(config)
+      return { content: [{ type: 'text', text: '  ✓ Options saved.\n' + statusBlock(loadConfig()) }] }
+    }
+    return { content: [{ type: 'text', text: '  ⏎ Cancelled.' }] }
+  }
+}