@coldiq/mcp 0.3.9 → 0.3.11

package/src/utils/compact-companies.ts

@@ -0,0 +1,208 @@
+// Normalize search_companies responses into a small, predictable shape.
+//
+// The search_companies waterfall fans across many providers, each with a
+// different payload shape (CompanyEnrich nests at data.items[], Apollo at
+// data.organizations[] + pagination, FullEnrich at data.companies[], others at
+// data.data[]/results[]). The raw provider object per company is heavy
+// (~3-4KB: full funding-round history, technologies, ~30-item keyword arrays,
+// NAICS codes, every social URL, SEO descriptions, logos). Compact mode keeps
+// only the fields agents act on downstream — company identity (name, domain,
+// website, linkedin_url, used by find_people/find_emails) plus the firmographic
+// signals a user filters on (employees, revenue, industry, country, city,
+// founded_year, categories). It changes how many descriptive fields ride along,
+// never which companies are returned. Verbose mode keeps the raw passthrough.
+
+export interface CompactCompany {
+  name?: string
+  domain?: string
+  website?: string
+  linkedin_url?: string
+  employees?: number | string
+  revenue?: number | string
+  industry?: string
+  country?: string
+  city?: string
+  founded_year?: number | string
+  categories?: string[]
+}
+
+function getPath(obj: unknown, path: string): unknown {
+  if (obj == null) return undefined
+  let cur: unknown = obj
+  for (const part of path.split('.')) {
+    if (cur == null) return undefined
+    if (Array.isArray(cur)) {
+      const idx = Number(part)
+      if (!Number.isInteger(idx)) return undefined
+      cur = cur[idx]
+    } else if (typeof cur === 'object') {
+      cur = (cur as Record<string, unknown>)[part]
+    } else {
+      return undefined
+    }
+  }
+  return cur
+}
+
+function pick(obj: unknown, paths: string[]): unknown {
+  for (const p of paths) {
+    const v = getPath(obj, p)
+    if (v !== undefined && v !== null && v !== '') return v
+  }
+  return undefined
+}
+
+function asString(v: unknown): string | undefined {
+  if (typeof v === 'string') return v.length > 0 ? v : undefined
+  if (typeof v === 'number') return String(v)
+  return undefined
+}
+
+function asStringOrNumber(v: unknown): number | string | undefined {
+  if (typeof v === 'number') return v
+  if (typeof v === 'string') return v.length > 0 ? v : undefined
+  return undefined
+}
+
+function isLinkedInUrl(v: unknown): v is string {
+  if (typeof v !== 'string' || v.length === 0) return false
+  return v.toLowerCase().includes('linkedin.com/')
+}
+
+function pickLinkedIn(obj: unknown, paths: string[]): string | undefined {
+  for (const p of paths) {
+    const v = getPath(obj, p)
+    if (isLinkedInUrl(v)) return v
+  }
+  return undefined
+}
+
+// Probe the keys providers use to hold the companies array. Covers the flat
+// shapes (CompanyEnrich `items`, FullEnrich/BlitzAPI `companies`, Apollo
+// `organizations`, generic `data`/`results`, AI-Ark `content`) and the one-level
+// nested shape LinkUp uses (`data.companies[]`). Keys are kept in sync with the
+// `hasResult` checks in registry.ts so compact never returns [] for a provider
+// that actually matched.
+export function extractCompaniesArray(data: unknown): unknown[] {
+  if (data == null) return []
+  if (Array.isArray(data)) return data
+  if (typeof data !== 'object') return []
+  const d = data as Record<string, unknown>
+  for (const key of ['items', 'companies', 'organizations', 'data', 'results', 'content']) {
+    const v = d[key]
+    if (Array.isArray(v)) return v
+  }
+  // LinkUp (search / fundraising / hiring) wraps results under data.companies[].
+  const inner = d.data
+  if (inner && typeof inner === 'object' && !Array.isArray(inner)) {
+    const di = inner as Record<string, unknown>
+    for (const key of ['companies', 'organizations', 'results', 'items', 'content']) {
+      const v = di[key]
+      if (Array.isArray(v)) return v
+    }
+  }
+  return []
+}
+
+export function normalizeCompany(record: unknown): CompactCompany | null {
+  if (!record || typeof record !== 'object' || Array.isArray(record)) return null
+  const r = record as Record<string, unknown>
+  const out: CompactCompany = {}
+
+  const name = asString(pick(r, ['name', 'company_name', 'organization.name', 'companyName']))
+  if (name) out.name = name
+
+  const domain = asString(pick(r, ['domain', 'primary_domain', 'website_domain', 'organization.primary_domain']))
+  if (domain) out.domain = domain
+
+  const website = asString(pick(r, ['website', 'website_url', 'organization.website_url', 'url']))
+  if (website) out.website = website
+
+  // LinkedIn URL is strict-validated: a wrong value silently breaks downstream
+  // find_people LinkedIn-driven lookups.
+  const linkedin = pickLinkedIn(r, [
+    'socials.linkedin_url',
+    'linkedin_url',
+    'organization.linkedin_url',
+    'social_profiles.linkedin.url',
+  ])
+  if (linkedin) out.linkedin_url = linkedin
+
+  const employees = asStringOrNumber(pick(r, [
+    'employees',
+    'employee_count',
+    'employees_on_linkedin', // BlitzAPI
+    'estimated_num_employees',
+    'num_employees',
+    'organization.estimated_num_employees',
+    'size',
+    'headcount',
+  ]))
+  if (employees !== undefined) out.employees = employees
+
+  const revenue = asStringOrNumber(pick(r, [
+    'revenue',
+    'estimated_revenue',
+    'annual_revenue',
+    'organization.annual_revenue',
+  ]))
+  if (revenue !== undefined) out.revenue = revenue
+
+  const industry = asString(pick(r, ['industry', 'organization.industry']))
+  if (industry) out.industry = industry
+
+  const country = asString(pick(r, [
+    'location.country.name',
+    'location.country.code',
+    'country',
+    'organization.country',
+    'location_country',
+  ]))
+  if (country) out.country = country
+
+  const city = asString(pick(r, [
+    'location.city.name',
+    'city',
+    'organization.city',
+    'location_city',
+  ]))
+  if (city) out.city = city
+
+  const foundedYear = asStringOrNumber(pick(r, ['founded_year', 'organization.founded_year', 'foundation_date']))
+  if (foundedYear !== undefined) out.founded_year = foundedYear
+
+  const categories = getPath(r, 'categories')
+  if (Array.isArray(categories)) {
+    const cats = categories.filter((c): c is string => typeof c === 'string' && c.length > 0)
+    if (cats.length > 0) out.categories = cats
+  }
+
+  // Drop records that resolved to nothing meaningful.
+  if (Object.keys(out).length === 0) return null
+  return out
+}
+
+export interface CompactCompaniesPayload {
+  companies: CompactCompany[]
+  total?: number
+}
+
+export function compactCompaniesPayload(data: unknown): CompactCompaniesPayload {
+  const arr = extractCompaniesArray(data)
+  const companies: CompactCompany[] = []
+  for (const r of arr) {
+    const c = normalizeCompany(r)
+    if (c) companies.push(c)
+  }
+
+  const out: CompactCompaniesPayload = { companies }
+
+  // Preserve the total-addressable-market count when the provider exposes one
+  // (CompanyEnrich: data.totalItems; Apollo: data.pagination.total_entries).
+  if (data && typeof data === 'object' && !Array.isArray(data)) {
+    const total = pick(data, ['totalItems', 'total', 'pagination.total_entries', 'pagination.total'])
+    if (typeof total === 'number') out.total = total
+  }
+
+  return out
+}

package/tests/registry-polling.test.ts

@@ -47,7 +47,7 @@ describe('MCP polling interval schedules', () => {
     expect(30_000 / 1500).toBeGreaterThanOrEqual(20)
   })
 
-  it('LeadsFactory find-people: shorter front, ≥7 polls in 5 minutes', () => {
+  it('LeadsFactory find-people: fast front probes, then flat 12s ceiling', () => {
     const providers = getProviders('find_people')
     const lf = providers.find((p) => p.id === 'leadsfactory')!
     expect(lf?.async).toBeDefined()
@@ -55,7 +55,13 @@ describe('MCP polling interval schedules', () => {
     expect(fn(1)).toBe(2000) // faster first probe vs old 3000
     expect(fn(2)).toBe(5000)
     expect(fn(3)).toBe(12000)
-    // Validates ≥7 polls still fit in 5 min (guard against overly aggressive growth)
-    expect(evalPollSchedule(fn, 300_000).length).toBeGreaterThanOrEqual(7)
+    // Flat ceiling: never grows past 12s, so completion-detection lag is bounded
+    // to ≤12s (the old +8s/attempt growth reached 44–72s gaps).
+    expect(fn(7)).toBe(12000)
+    expect(fn(20)).toBe(12000)
+    const schedule = evalPollSchedule(fn, 300_000)
+    expect(Math.max(...schedule)).toBe(12000) // never exceeds the ceiling
+    // Validates ≥7 polls still fit in 5 min
+    expect(schedule.length).toBeGreaterThanOrEqual(7)
   })
 })

package/tests/registry.test.ts

@@ -551,20 +551,22 @@ describe('registry', () => {
       expect((result.body as Record<string, unknown>).company_linkedin_urls).toBeUndefined()
     })
 
-    it('LeadsFactory has async config with continuous backoff schedule', () => {
+    it('LeadsFactory has async config with capped backoff schedule', () => {
       const providers = getProviders('find_people')
       const lf = providers.find((p) => p.id === 'leadsfactory')!
       expect(lf.async).toBeDefined()
       expect(lf.async!.timeoutMs).toBe(300_000)
-      // Function-form pollIntervalMs: fast first probe, then continuously growing backoff.
+      // Function-form pollIntervalMs: fast first probes, then a flat 12s ceiling
+      // so completion-detection lag stays bounded (≤12s) instead of growing
+      // unbounded into 44–72s gaps.
       const sched = lf.async!.pollIntervalMs
       expect(typeof sched).toBe('function')
       const fn = sched as (attempt: number) => number
       expect(fn(1)).toBe(2000)
       expect(fn(2)).toBe(5000)
       expect(fn(3)).toBe(12000)
-      expect(fn(4)).toBe(20000)
-      expect(fn(8)).toBe(52000)
+      expect(fn(4)).toBe(12000)
+      expect(fn(8)).toBe(12000)
       // At least 7 polls must fit inside the 5-minute timeout — guards against
       // future tuning that would make the ramp so steep we only get 1–2 polls.
       let cumulative = 0

package/tests/tools/find-emails.test.ts

@@ -522,6 +522,48 @@ describe('find_emails handler — resilience', () => {
     expect(parsed.data.results[0]).toMatchObject({ id: 'p1', email: 'alice@example.com', provider: 'findymail' })
   })
 
+  it('FullEnrich poll bails after 3 consecutive errors instead of polling to the 45s deadline', async () => {
+    // Regression: a 502-ing FullEnrich poll used to `continue` every 2s for the full
+    // 45s deadline (~22 polls) — pure latency on the chunk's critical path with zero
+    // emails. It must now stop after COLDIQ_MAX_POLL_ERRORS (default 3) consecutive errors.
+    vi.useFakeTimers()
+
+    let fePollCount = 0
+    globalThis.fetch = vi.fn(async (url: string | URL | Request) => {
+      const u = url.toString()
+      if (u.includes('/prospeo/bulk-enrich-person')) {
+        return new Response(
+          JSON.stringify({ error: false, results: [{ identifier: 'p1', error: true, person: null }], total_cost: 0 }),
+          { status: 200 },
+        )
+      }
+      // FullEnrich poll (has the id segment) — always 502.
+      if (u.includes('/fullenrich/contact/enrich/bulk/abc-fe-err')) {
+        fePollCount++
+        return new Response(JSON.stringify({ error: 'bad gateway' }), { status: 502 })
+      }
+      // FullEnrich create.
+      if (u.includes('/fullenrich/contact/enrich/bulk')) {
+        return new Response(JSON.stringify({ enrichment_id: 'abc-fe-err' }), { status: 200 })
+      }
+      // FindyMail / IcyPeas / single-fallback all miss cleanly (no email, no failure).
+      return new Response(JSON.stringify({ email: null }), { status: 200 })
+    }) as typeof fetch
+
+    const handlerPromise = findEmailsHandler({
+      people: [{ id: 'p1', first_name: 'Alice', last_name: 'Smith', domain: 'example.com' }],
+    })
+
+    await vi.runAllTimersAsync()
+    const result = await handlerPromise
+    vi.useRealTimers()
+
+    // Bailed at the 3-error cap, not the ~22 polls that 45000/2000 would have produced.
+    expect(fePollCount).toBe(3)
+    const parsed = JSON.parse(result.content[0].text)
+    expect(parsed._meta.providers.fullenrich).toEqual({ attempted: 1, failed: 1 })
+  })
+
   it('falls back to single find_email waterfall when bulk providers all miss (covers limadata/blitzapi/linkupapi gap)', async () => {
     // Bulk path: Prospeo + FullEnrich + Findymail + Icypeas all miss.
     // Single-only providers (limadata-work-email, blitzapi, limadata-work-email-linkedin, linkupapi)

package/tests/tools/search-companies.test.ts

@@ -13,7 +13,7 @@ describe('search_companies handler', () => {
     globalThis.fetch = originalFetch
   })
 
-  it('returns results from first successful provider', async () => {
+  it('returns results from first successful provider (compact by default)', async () => {
     globalThis.fetch = vi.fn(async () =>
       new Response(JSON.stringify({ data: [{ name: 'ColdIQ', domain: 'coldiq.com' }] }), { status: 200 })
     ) as typeof fetch
@@ -23,7 +23,83 @@ describe('search_companies handler', () => {
     expect(result.isError).toBeUndefined()
     const parsed = JSON.parse(result.content[0].text)
     expect(parsed._meta.provider).toBe('companyenrich')
-    expect(parsed.data.data).toHaveLength(1)
+    // Compact-by-default: the raw provider array is normalized into companies[].
+    expect(parsed.data.companies).toHaveLength(1)
+    expect(parsed.data.companies[0]).toMatchObject({ name: 'ColdIQ', domain: 'coldiq.com' })
+  })
+
+  it('compact (default) keeps identity + firmographic fields and drops heavy ones', async () => {
+    // Mirror the real CompanyEnrich shape: companies under data.items[], each a
+    // heavy object with funding/technologies/keywords/socials/seo.
+    globalThis.fetch = vi.fn(async () =>
+      new Response(JSON.stringify({
+        items: [{
+          id: '0192562e-7008-702b-97a2-0b2c489d339b',
+          name: 'Hugging Face',
+          domain: 'huggingface.co',
+          website: 'https://huggingface.co',
+          industry: 'Business Services',
+          categories: ['b2b', 'b2c', 'saas'],
+          employees: '201-500',
+          revenue: '10m-50m',
+          founded_year: 2016,
+          description: 'A very long company description that should be dropped in compact mode...',
+          keywords: Array.from({ length: 28 }, (_, i) => `kw${i}`),
+          technologies: ['Stripe', 'Nginx', 'Node-Js'],
+          naics_codes: ['511210', '518210'],
+          socials: { linkedin_url: 'https://www.linkedin.com/company/huggingface', twitter_url: 'https://twitter.com/huggingface' },
+          financial: { total_funding: 395200000, funding: [{ date: '2024-08-01', type: 'Venture Round' }] },
+          location: { country: { code: 'FR', name: 'France' }, city: { name: 'Paris' } },
+        }],
+        page: 1,
+        totalItems: 10000,
+      }), { status: 200 })
+    ) as typeof fetch
+
+    const result = await searchCompaniesHandler({ keywords: ['SaaS'], countries: ['FR'], limit: 5 })
+
+    expect(result.isError).toBeUndefined()
+    const parsed = JSON.parse(result.content[0].text)
+    expect(parsed.data.total).toBe(10000)
+    expect(parsed.data.companies).toHaveLength(1)
+    const c = parsed.data.companies[0]
+    // Kept: identity + firmographics + downstream-critical linkedin_url.
+    expect(c).toEqual({
+      name: 'Hugging Face',
+      domain: 'huggingface.co',
+      website: 'https://huggingface.co',
+      linkedin_url: 'https://www.linkedin.com/company/huggingface',
+      employees: '201-500',
+      revenue: '10m-50m',
+      industry: 'Business Services',
+      country: 'France',
+      city: 'Paris',
+      founded_year: 2016,
+      categories: ['b2b', 'b2c', 'saas'],
+    })
+    // Dropped heavy fields.
+    expect(c.description).toBeUndefined()
+    expect(c.keywords).toBeUndefined()
+    expect(c.technologies).toBeUndefined()
+    expect(c.naics_codes).toBeUndefined()
+    expect(c.financial).toBeUndefined()
+    expect(c.id).toBeUndefined()
+  })
+
+  it('fields=verbose returns the raw provider passthrough unchanged', async () => {
+    globalThis.fetch = vi.fn(async () =>
+      new Response(JSON.stringify({ items: [{ name: 'Hugging Face', keywords: ['ai'], technologies: ['Stripe'] }], totalItems: 10000 }), { status: 200 })
+    ) as typeof fetch
+
+    const result = await searchCompaniesHandler({ keywords: ['SaaS'], limit: 5, fields: 'verbose' })
+
+    expect(result.isError).toBeUndefined()
+    const parsed = JSON.parse(result.content[0].text)
+    // Raw shape preserved: items[] with heavy fields, no companies[] normalization.
+    expect(parsed.data.items).toHaveLength(1)
+    expect(parsed.data.items[0].keywords).toEqual(['ai'])
+    expect(parsed.data.items[0].technologies).toEqual(['Stripe'])
+    expect(parsed.data.companies).toBeUndefined()
   })
 
   it('falls back to FullEnrich (priority 2) on CompanyEnrich failure', async () => {

package/tests/utils/compact-companies.test.ts

@@ -0,0 +1,146 @@
+import { describe, it, expect } from 'vitest'
+import {
+  compactCompaniesPayload,
+  extractCompaniesArray,
+  normalizeCompany,
+} from '../../src/utils/compact-companies.js'
+
+describe('extractCompaniesArray', () => {
+  it('finds the array under each provider key', () => {
+    expect(extractCompaniesArray({ items: [{ name: 'A' }] })).toHaveLength(1) // CompanyEnrich
+    expect(extractCompaniesArray({ companies: [{ name: 'A' }] })).toHaveLength(1) // FullEnrich
+    expect(extractCompaniesArray({ organizations: [{ name: 'A' }, { name: 'B' }] })).toHaveLength(2) // Apollo
+    expect(extractCompaniesArray({ data: [{ name: 'A' }] })).toHaveLength(1) // generic
+    expect(extractCompaniesArray({ results: [{ name: 'A' }] })).toHaveLength(1)
+  })
+
+  it('handles a bare top-level array', () => {
+    expect(extractCompaniesArray([{ name: 'A' }])).toHaveLength(1)
+  })
+
+  it('finds AI-Ark content[] and LinkUp nested data.companies[]', () => {
+    // AI-Ark companies live under content[] (registry hasResult: isNonEmptyArray(d.content)).
+    expect(extractCompaniesArray({ content: [{ name: 'A' }, { name: 'B' }] })).toHaveLength(2)
+    // LinkUp (search/fundraising/hiring) nests under data.companies[]
+    // (registry hasResult: isNonEmptyArray(d.data.companies)).
+    expect(extractCompaniesArray({ data: { companies: [{ name: 'A' }] } })).toHaveLength(1)
+    expect(extractCompaniesArray({ data: { organizations: [{ name: 'A' }, { name: 'B' }] } })).toHaveLength(2)
+  })
+
+  it('prefers a direct array under data over descending into a nested object', () => {
+    // PDL/Prospeo/TheirStack shape: { data: [...] } must stay flat, not be skipped.
+    expect(extractCompaniesArray({ data: [{ name: 'A' }] })).toHaveLength(1)
+  })
+
+  it('returns [] for null / non-object / no recognized key', () => {
+    expect(extractCompaniesArray(null)).toEqual([])
+    expect(extractCompaniesArray('nope')).toEqual([])
+    expect(extractCompaniesArray({ totalItems: 10 })).toEqual([])
+  })
+})
+
+describe('normalizeCompany', () => {
+  it('maps the CompanyEnrich nested shape to compact fields', () => {
+    const c = normalizeCompany({
+      id: 'uuid-123',
+      name: 'Mistral AI',
+      domain: 'mistral.ai',
+      website: 'https://mistral.ai',
+      industry: 'Software',
+      categories: ['b2b', 'saas'],
+      employees: '201-500',
+      revenue: '1m-10m',
+      founded_year: 2023,
+      description: 'long blurb',
+      keywords: ['a', 'b'],
+      technologies: ['Gmail'],
+      socials: { linkedin_url: 'https://www.linkedin.com/company/mistralai' },
+      location: { country: { code: 'FR', name: 'France' }, city: { name: 'Paris' } },
+    })
+    expect(c).toEqual({
+      name: 'Mistral AI',
+      domain: 'mistral.ai',
+      website: 'https://mistral.ai',
+      linkedin_url: 'https://www.linkedin.com/company/mistralai',
+      employees: '201-500',
+      revenue: '1m-10m',
+      industry: 'Software',
+      country: 'France',
+      city: 'Paris',
+      founded_year: 2023,
+      categories: ['b2b', 'saas'],
+    })
+  })
+
+  it('maps the Apollo organization shape (numeric employees, flat fields)', () => {
+    const c = normalizeCompany({
+      name: 'Apple',
+      primary_domain: 'apple.com',
+      website_url: 'https://apple.com',
+      linkedin_url: 'https://www.linkedin.com/company/apple',
+      estimated_num_employees: 164000,
+      industry: 'consumer electronics',
+      founded_year: 1976,
+    })
+    expect(c).toMatchObject({
+      name: 'Apple',
+      domain: 'apple.com',
+      website: 'https://apple.com',
+      linkedin_url: 'https://www.linkedin.com/company/apple',
+      employees: 164000,
+      industry: 'consumer electronics',
+      founded_year: 1976,
+    })
+  })
+
+  it('maps BlitzAPI employees_on_linkedin into employees', () => {
+    const c = normalizeCompany({ name: 'Blitz Co', domain: 'blitz.co', employees_on_linkedin: 320 })
+    expect(c).toMatchObject({ name: 'Blitz Co', domain: 'blitz.co', employees: 320 })
+  })
+
+  it('rejects a non-LinkedIn url in the linkedin slot', () => {
+    const c = normalizeCompany({ name: 'X', socials: { linkedin_url: 'https://twitter.com/x' } })
+    expect(c?.linkedin_url).toBeUndefined()
+    expect(c?.name).toBe('X')
+  })
+
+  it('returns null for empty / non-object records', () => {
+    expect(normalizeCompany(null)).toBeNull()
+    expect(normalizeCompany({})).toBeNull()
+    expect(normalizeCompany([{ name: 'A' }])).toBeNull()
+    expect(normalizeCompany('x')).toBeNull()
+  })
+
+  it('omits fields that are not present', () => {
+    const c = normalizeCompany({ name: 'Solo' })
+    expect(c).toEqual({ name: 'Solo' })
+  })
+})
+
+describe('compactCompaniesPayload', () => {
+  it('normalizes items[] and surfaces totalItems', () => {
+    const out = compactCompaniesPayload({
+      items: [{ name: 'A', domain: 'a.com' }, { name: 'B', domain: 'b.com' }],
+      totalItems: 10000,
+    })
+    expect(out.companies).toHaveLength(2)
+    expect(out.total).toBe(10000)
+  })
+
+  it('reads Apollo pagination.total_entries as total', () => {
+    const out = compactCompaniesPayload({ organizations: [{ name: 'A' }], pagination: { total_entries: 8371 } })
+    expect(out.companies).toHaveLength(1)
+    expect(out.total).toBe(8371)
+  })
+
+  it('drops records that normalize to nothing but keeps real ones', () => {
+    const out = compactCompaniesPayload({ items: [{ name: 'A' }, {}, { foo: 'bar' }] })
+    expect(out.companies).toHaveLength(1)
+    expect(out.companies[0].name).toBe('A')
+  })
+
+  it('returns an empty companies array (no total) when there is nothing to extract', () => {
+    const out = compactCompaniesPayload({ unexpected: true })
+    expect(out).toEqual({ companies: [] })
+  })
+})