@coldiq/mcp 0.5.2 → 0.5.4

package/src/tools/extract-post-engagement.ts

@@ -6,7 +6,8 @@ export const extractPostEngagementName = 'extract_post_engagement'
 export const extractPostEngagementDescription =
   'Extract the people who engaged with a LinkedIn post — commenters and/or reactors — as a deduplicated list of contacts (name, profile URL, headline). ' +
   'Use this for social-signal prospecting: pull everyone who engaged with a viral post, then chain the results into enrich_person / find_email to get roles and work emails. ' +
-  'Runs an async extraction job (typically ~30–120s) and returns once the people are ready. Pricing is returned in the response credit headers.'
+  'Runs an async extraction job (typically 1–3 minutes; longer for posts with 1000+ interactions) and returns every extracted person once ready. ' +
+  'If the primary source fails or its account is out of credits, automatically retries on a second provider and says so in `_meta.provider` — those records carry name/profile URL/headline only, so enrich them before use. Pricing is returned in the response credit headers.'
 
 export const extractPostEngagementSchema = {
   post_url: z
@@ -17,16 +18,133 @@ export const extractPostEngagementSchema = {
     .enum(['comments', 'reactions', 'both'])
     .default('both')
     .describe('Which engagement to extract: "comments" (people who commented), "reactions" (people who reacted), or "both" (default — deduplicated across both).'),
-  include_replies: z
-    .boolean()
-    .optional()
-    .describe('When extracting comments, also include people who replied to comments. Defaults to true upstream.'),
+  // `include_replies` was removed: this tool returns a DEDUPLICATED contact list,
+  // where a person is one entry regardless of whether they commented or replied,
+  // so the flag never changed the result. Use GET /v1/jungler/workbooks/{id}/comments
+  // to work with individual comments and their reply metadata.
+}
+
+// One page covers most posts; the loop below still walks the rest when it doesn't.
+const CONTACTS_PAGE_SIZE = 500
+// Backstop so a mis-paginating upstream can't spin this tool forever.
+const MAX_CONTACT_PAGES = 20
+
+/**
+ * True unless the contact demonstrably engaged with nothing.
+ *
+ * The workbook always collects the post itself, so the post's AUTHOR comes back
+ * as a contact with `stats: {comments: 0, reactions: 0}` even when nobody
+ * engaged. This tool promises the people who engaged, so a zero/zero contact is
+ * not one of them. Anything without usable stats is kept — never drop a contact
+ * on ambiguity.
+ */
+function engaged(contact: unknown): boolean {
+  if (!contact || typeof contact !== 'object') return true
+  const stats = (contact as { stats?: unknown }).stats
+  if (!stats || typeof stats !== 'object') return true
+  const { comments, reactions } = stats as { comments?: unknown; reactions?: unknown }
+  if (typeof comments !== 'number' || typeof reactions !== 'number') return true
+  return comments > 0 || reactions > 0
+}
+
+/** Read one page of the contacts envelope, tolerating a bare-array response. */
+function readContactsPage(data: unknown): { items: unknown[]; total?: number; pages?: number } {
+  if (Array.isArray(data)) return { items: data, pages: 1 }
+  if (data && typeof data === 'object') {
+    const obj = data as { items?: unknown; total?: unknown; pages?: unknown }
+    if (Array.isArray(obj.items)) {
+      return {
+        items: obj.items,
+        total: typeof obj.total === 'number' ? obj.total : undefined,
+        pages: typeof obj.pages === 'number' ? obj.pages : undefined,
+      }
+    }
+  }
+  return { items: [], pages: 1 }
 }
 
 function sleep(ms: number): Promise<void> {
   return new Promise((resolve) => setTimeout(resolve, ms))
 }
 
+// ── Fallback provider ───────────────────────────────────────────────────────
+//
+// Jungler is the primary: it charges a flat rate per post regardless of size and
+// returns enriched contacts (work email, company domain, title, seniority). Lima
+// Data costs the same per post but returns profile URLs only, so it needs a
+// separate enrichment pass — hence second, not first.
+//
+// We reach for it ONLY when Jungler failed for a reason that is ours, not the
+// post's: a failed run, or an upstream account out of credits. A genuinely empty
+// post must NOT fall through — Lima would return empty too, for another charge.
+// Same env knobs as the primary poll, so a caller (or a test) can tighten both.
+const fallbackPollMs = () => parseInt(process.env.COLDIQ_ENGAGEMENT_POLL_MS ?? '5000', 10)
+const fallbackTimeoutMs = () => parseInt(process.env.COLDIQ_ENGAGEMENT_FALLBACK_TIMEOUT_MS ?? '240000', 10)
+
+/** Flatten Lima's `{commenters:{comments:[{actor}]}, reactors:{reactions:[{actor}]}}` into contacts. */
+function limaEngagersToPeople(data: unknown, type: 'comments' | 'reactions' | 'both'): unknown[] {
+  if (!data || typeof data !== 'object') return []
+  const d = data as {
+    commenters?: { comments?: unknown[] }
+    reactors?: { reactions?: unknown[] }
+  }
+  const rows: Array<{ actor?: unknown; engagement: string }> = []
+  if (type !== 'reactions') {
+    for (const c of d.commenters?.comments ?? []) rows.push({ ...(c as object), engagement: 'COMMENT' } as never)
+  }
+  if (type !== 'comments') {
+    for (const r of d.reactors?.reactions ?? []) rows.push({ ...(r as object), engagement: 'REACTION' } as never)
+  }
+  // Deduplicate by profile URL: someone who both commented and reacted is one person.
+  const byProfile = new Map<string, Record<string, unknown>>()
+  for (const row of rows) {
+    const actor = (row.actor ?? {}) as Record<string, unknown>
+    const key = String(actor.profile_url ?? actor.name ?? Math.random())
+    const existing = byProfile.get(key)
+    if (existing) {
+      const seen = existing.engagement_types as string[]
+      if (!seen.includes(row.engagement)) seen.push(row.engagement)
+      continue
+    }
+    byProfile.set(key, {
+      name: actor.name ?? null,
+      profile_url: actor.profile_url ?? null,
+      description: actor.headline ?? null,
+      profile_image_url: actor.image_url ?? null,
+      engagement_types: [row.engagement],
+    })
+  }
+  return [...byProfile.values()]
+}
+
+async function extractViaLimaData(postUrl: string, type: 'comments' | 'reactions' | 'both') {
+  const submit = await callApi('POST', '/limadata/batch/post-engagements', { urls: [postUrl] })
+  if (!submit.ok) {
+    return { ok: false as const, error: extractErrorMessage(submit.data) ?? `Fallback provider failed to start (status ${submit.status})` }
+  }
+  const batchId = (submit.data as { batch_id?: number | string }).batch_id
+  if (batchId === undefined) return { ok: false as const, error: 'Fallback provider did not return a batch id' }
+
+  const creditsCharged = Number(submit.headers['x-coldiq-credits-charged'])
+  const timeoutMs = fallbackTimeoutMs()
+  const deadline = Date.now() + timeoutMs
+  while (Date.now() < deadline) {
+    await sleep(fallbackPollMs())
+    const poll = await callApi('POST', '/limadata/batch/results', { batch_id: String(batchId) })
+    if (!poll.ok) continue
+    const body = poll.data as { items?: Array<{ status?: string; data?: unknown }> }
+    const item = body.items?.[0]
+    if (!item || String(item.status).toLowerCase() !== 'completed') continue
+    return {
+      ok: true as const,
+      people: limaEngagersToPeople(item.data, type),
+      creditsCharged: Number.isFinite(creditsCharged) ? creditsCharged : undefined,
+      batchId,
+    }
+  }
+  return { ok: false as const, error: `Fallback provider did not finish within ${Math.round(timeoutMs / 1000)}s`, batchId }
+}
+
 function errorResult(error: string, extra?: Record<string, unknown>) {
   return {
     content: [{ type: 'text' as const, text: JSON.stringify({ error, ...extra }) }],
@@ -42,11 +160,50 @@ function extractErrorMessage(data: unknown): string | undefined {
   return undefined
 }
 
+/**
+ * Try the fallback provider after the primary failed; if it also fails, report
+ * the ORIGINAL failure (that is the one the caller can act on) with the
+ * fallback's outcome attached.
+ *
+ * Set COLDIQ_ENGAGEMENT_FALLBACK=0 to disable — the fallback is a second billable
+ * provider, so anyone running to a strict budget must be able to opt out.
+ */
+async function fallbackOr(
+  postUrl: string,
+  type: 'comments' | 'reactions' | 'both',
+  primaryError: string,
+  extra?: Record<string, unknown>,
+) {
+  if (process.env.COLDIQ_ENGAGEMENT_FALLBACK === '0') {
+    return errorResult(primaryError, { post_url: postUrl, ...extra })
+  }
+
+  const fb = await extractViaLimaData(postUrl, type)
+  if (!fb.ok) {
+    return errorResult(primaryError, { post_url: postUrl, ...extra, fallback_error: fb.error })
+  }
+
+  const meta: Record<string, unknown> = { provider: 'limadata', primary_error: primaryError }
+  if (fb.creditsCharged !== undefined) meta.credits_charged = fb.creditsCharged
+  return {
+    content: [{
+      type: 'text' as const,
+      text: JSON.stringify({
+        data: { post_url: postUrl, type, people: fb.people, count: fb.people.length },
+        // Say plainly that these records are thinner than the primary's, so the
+        // caller knows to run enrichment before treating them as contacts.
+        _meta: {
+          ...meta,
+          note: 'Served by the fallback provider after the primary failed. These records carry name, profile URL and headline only — no work email or company data. Run enrich_person / find_email on them if you need contact details.',
+        },
+      }),
+    }],
+  }
+}
+
 export async function extractPostEngagementHandler(input: Record<string, unknown>) {
   const postUrl = input.post_url as string
   const type = (input.type as 'comments' | 'reactions' | 'both' | undefined) ?? 'both'
-  const includeReplies = input.include_replies as boolean | undefined
-
   const dataTypes =
     type === 'comments' ? ['comment'] : type === 'reactions' ? ['reaction'] : ['comment', 'reaction']
 
@@ -56,21 +213,30 @@ export async function extractPostEngagementHandler(input: Record<string, unknown
     data_types: dataTypes,
   })
   if (!createRes.ok) {
-    return errorResult(extractErrorMessage(createRes.data) ?? `Failed to start extraction (status ${createRes.status})`)
+    return fallbackOr(
+      postUrl, type,
+      extractErrorMessage(createRes.data) ?? `Failed to start extraction (status ${createRes.status})`,
+    )
   }
-  const createData = createRes.data as { task_id?: string }
+  const createData = createRes.data as { task_id?: string; workbook_id?: string }
   const taskId = createData.task_id
   if (!taskId) {
-    return errorResult('Extraction job did not return a task id')
+    return fallbackOr(postUrl, type, 'Extraction job did not return a task id')
   }
+  // Kept so a timeout can still hand the caller a way to collect what they paid
+  // for — the workbook stays readable upstream for 12 hours.
+  const createWorkbookId = createData.workbook_id
   // Credit headers are emitted on the create call (the only billed step).
   const creditsCharged = Number(createRes.headers['x-coldiq-credits-charged'])
   const creditsRemaining = Number(createRes.headers['x-coldiq-credits-remaining'])
 
   // Step 2 — poll task status until it resolves. The status endpoint is free, so
   // polling does not bill; only the create call above charged credits.
-  const pollIntervalMs = parseInt(process.env.COLDIQ_ENGAGEMENT_POLL_MS ?? '2000', 10)
-  const timeoutMs = parseInt(process.env.COLDIQ_ENGAGEMENT_TIMEOUT_MS ?? '180000', 10)
+  // Upstream quotes 1-3 minutes and explicitly warns that large posts (1000+
+  // interactions) take longer — which is exactly the viral post this tool is for.
+  // A 180s deadline cut those off mid-extraction and reported a bogus timeout.
+  const pollIntervalMs = parseInt(process.env.COLDIQ_ENGAGEMENT_POLL_MS ?? '5000', 10)
+  const timeoutMs = parseInt(process.env.COLDIQ_ENGAGEMENT_TIMEOUT_MS ?? '600000', 10)
   const maxPollErrors = 3
   const deadline = Date.now() + timeoutMs
 
@@ -88,46 +254,82 @@ export async function extractPostEngagementHandler(input: Record<string, unknown
       continue
     }
     consecutivePollErrors = 0
-    const status = (statusRes.data as { status?: string; workbook_id?: string }).status
+    const statusData = statusRes.data as { status?: string; workbook_id?: string; credits_exhausted?: boolean }
+    const status = statusData.status
     if (status === 'success') {
-      workbookId = (statusRes.data as { workbook_id?: string }).workbook_id
+      // A credits-exhausted run is terminal but collected little or nothing
+      // because OUR provider account ran dry — the caller's request is sound, so
+      // route it to the fallback rather than handing back a hollow result. This
+      // charge is refunded server-side.
+      if (statusData.credits_exhausted) {
+        return fallbackOr(
+          postUrl, type,
+          'Engagement extraction stopped early — the data provider account is out of credits (this request was refunded)',
+          { task_id: taskId },
+        )
+      }
+      workbookId = statusData.workbook_id
       break
     }
     if (status === 'failure') {
-      return errorResult('Engagement extraction failed upstream — the post may be private, deleted, or have no engagement', { post_url: postUrl })
+      // The status route refunds the create charge when a run reaches failure,
+      // so the user is made whole — say so rather than leaving them to wonder.
+      return fallbackOr(
+        postUrl, type,
+        'Engagement extraction failed upstream — the post may be private, deleted, or have no engagement. The credits for this task have been refunded.',
+        { task_id: taskId },
+      )
     }
   }
 
   if (!workbookId) {
-    return errorResult(`Engagement extraction did not complete within ${Math.round(timeoutMs / 1000)}s — try again shortly`, { post_url: postUrl })
+    // Giving up here does NOT cancel the run, and the create call was already
+    // billed. Never drop that on the floor: hand back the ids so the caller can
+    // collect the results (upstream keeps the workbook for 12 hours) instead of
+    // paying again for a re-run that is probably still in flight.
+    return errorResult(
+      `Engagement extraction is still running after ${Math.round(timeoutMs / 1000)}s. It has NOT been cancelled and no re-run is needed — poll GET /v1/jungler/tasks/${taskId}/status until it reports success, then fetch the people from GET /v1/jungler/workbooks/${createWorkbookId ?? '{workbook_id}'}/contacts. Results stay available for 12 hours after the task was created.`,
+      { post_url: postUrl, task_id: taskId, workbook_id: createWorkbookId, status: 'running' },
+    )
   }
 
   // Step 3 — fetch the deduplicated people. activity_filter narrows to commenters
   // or reactors; omitted for "both" so the upstream returns all unique contacts.
-  const queryParams: Record<string, string> = {}
+  // Contacts are paginated, and a viral post — the case this tool exists for —
+  // routinely exceeds one page, so walk every page rather than truncating to the
+  // first. Reads are free, so pagination costs the caller nothing.
+  const queryParams: Record<string, string> = { page_size: String(CONTACTS_PAGE_SIZE) }
   if (type === 'comments') queryParams.activity_filter = 'commenters'
   else if (type === 'reactions') queryParams.activity_filter = 'reactors'
-  if (includeReplies !== undefined) queryParams.include_replies = String(includeReplies)
 
-  const contactsRes = await callApi(
-    'GET',
-    `/jungler/workbooks/${workbookId}/contacts`,
-    undefined,
-    Object.keys(queryParams).length > 0 ? queryParams : undefined,
-  )
-  if (!contactsRes.ok) {
-    return errorResult(extractErrorMessage(contactsRes.data) ?? 'Failed to fetch extracted people', { post_url: postUrl })
+  const people: unknown[] = []
+  let totalContacts: number | undefined
+  for (let page = 1; page <= MAX_CONTACT_PAGES; page++) {
+    const contactsRes = await callApi(
+      'GET',
+      `/jungler/workbooks/${workbookId}/contacts`,
+      undefined,
+      { ...queryParams, page: String(page) },
+    )
+    if (!contactsRes.ok) {
+      return errorResult(extractErrorMessage(contactsRes.data) ?? 'Failed to fetch extracted people', { post_url: postUrl, task_id: taskId, workbook_id: workbookId })
+    }
+    const { items, total, pages } = readContactsPage(contactsRes.data)
+    people.push(...items.filter(engaged))
+    totalContacts = total
+    if (items.length === 0 || (pages !== undefined && page >= pages)) break
   }
 
   const meta: Record<string, unknown> = {}
   if (Number.isFinite(creditsCharged)) meta.credits_charged = creditsCharged
   if (Number.isFinite(creditsRemaining)) meta.credits_remaining = creditsRemaining
+  if (totalContacts !== undefined && totalContacts > people.length) meta.truncated_from = totalContacts
 
   return {
     content: [{
       type: 'text' as const,
       text: JSON.stringify({
-        data: { post_url: postUrl, type, people: contactsRes.data },
+        data: { post_url: postUrl, type, people, count: people.length },
         _meta: meta,
       }),
     }],

package/src/tools/get-endpoint-details.ts

@@ -11,10 +11,12 @@ export const getEndpointDetailsDescription =
 export const getEndpointDetailsSchema = {
   api: z.string().describe('Provider key from search_endpoints, e.g. "coldiq" (Lima Data), "apollo", or "exa".'),
   path: z.string().describe('Endpoint path without the /v1 prefix, e.g. "/limadata/prospect/people/filter".'),
+  method: z.enum(['GET', 'POST', 'PUT', 'PATCH', 'DELETE']).optional()
+    .describe('HTTP method from search_endpoints. Set this when one path supports more than one method.'),
 }
 
 export async function getEndpointDetailsHandler(input: Record<string, unknown>) {
-  const res = await callApi('POST', '/endpoints/details', { api: input.api, path: input.path })
+  const res = await callApi('POST', '/endpoints/details', { api: input.api, path: input.path, method: input.method })
   if (!res.ok) {
     return {
       content: [{ type: 'text' as const, text: JSON.stringify({ error: 'Endpoint not found or details unavailable', status: res.status, data: res.data }) }],

package/src/tools/search-web.ts

@@ -5,7 +5,7 @@ import { getProvidersForCapability } from '../utils/provider-resolver.js'
 export const searchWebName = 'search_web'
 
 export const searchWebDescription =
-  'Search the web for market research, company news, or general lookups. Supports Google search (default) and neural/semantic search via Exa. ' +
+  'Search the web for market research, company news, or general lookups. Supports Google search (default), Exa auto through the neural compatibility alias, and Exa deep modes. ' +
   'NEVER use this to find, verify, or look up people, contacts, LinkedIn profiles, or executives — even as a fallback when find_people returns uncertain results. ' +
   'find_people is the only tool for people lookups; do not supplement or replace it with web search. ' +
   'ColdIQ automatically picks the best provider — pass use_providers only if you need a specific tool.'
@@ -14,7 +14,7 @@ export const searchWebSchema = {
   query: z.string().describe('Search query'),
   num_results: z.number().min(1).max(100).default(10).describe('Number of results (default: 10, max: 100)'),
   country: z.string().optional().describe('Country code for geo-targeting (e.g. "us", "fr")'),
-  search_type: z.enum(['general', 'neural']).default('general').describe('"general" for Google search (default), "neural" for semantic search via Exa'),
+  search_type: z.enum(['general', 'neural', 'deep', 'deep-reasoning']).default('general').describe('"general" for the default web search, "neural" for Exa auto compatibility, or an Exa deep mode'),
   use_providers: z.array(z.string()).optional().describe(`Optional ordered list of providers to use. Leave empty to let ColdIQ automatically pick the best tool for your inputs — recommended for most use cases. Available providers: ${getProvidersForCapability('search_web').join(', ')}. Provider names are matched fuzzily, so minor typos are tolerated.`),
 }
 

package/tests/registry-enrich-person.test.ts

@@ -390,20 +390,22 @@ describe('icypeas-reverse-email-lookup', () => {
     })
   })
 
-  it('hasResult: true when non-empty profiles array', () => {
-    expect(p().hasResult({ profiles: [{ name: 'Michel Lieben' }] })).toBe(true)
-  })
-
-  it('hasResult: true when success + data', () => {
-    expect(p().hasResult({ success: true, data: { name: 'Michel Lieben' } })).toBe(true)
-  })
-
-  it('hasResult: false when profiles is empty array', () => {
-    expect(p().hasResult({ profiles: [] })).toBe(false)
-  })
-
-  it('hasResult: false on empty object', () => {
-    expect(p().hasResult({})).toBe(false)
+  it('hasResult: true for the documented hit shape', () => {
+    expect(p().hasResult({
+      success: true,
+      status: 'FOUND',
+      results: 'https://www.linkedin.com/in/michel-lieben',
+    })).toBe(true)
+  })
+
+  it.each([
+    { success: true, status: 'NOT_FOUND', results: 'https://www.linkedin.com/in/michel-lieben' },
+    { success: false, validationErrors: ['Invalid email'] },
+    { success: true },
+    { profiles: [{ name: 'Michel Lieben' }] },
+    { success: true, data: { name: 'Michel Lieben' } },
+  ])('hasResult: false for a miss, invalid, or unknown body', (body) => {
+    expect(p().hasResult(body)).toBe(false)
   })
 })
 

package/tests/registry.test.ts

@@ -20,6 +20,13 @@ const ALL_CAPABILITIES: Capability[] = [
 ]
 
 describe('registry', () => {
+  it('maps the neural compatibility alias to Exa auto and preserves deep modes', () => {
+    const exa = getConfiguredProviders('search_web').find((provider) => provider.id === 'exa')!
+    expect(exa.mapParams({ query: 'ColdIQ', search_type: 'neural' }).body).toMatchObject({ type: 'auto' })
+    expect(exa.mapParams({ query: 'ColdIQ', search_type: 'deep' }).body).toMatchObject({ type: 'deep' })
+    expect(exa.mapParams({ query: 'ColdIQ', search_type: 'deep-reasoning' }).body).toMatchObject({ type: 'deep-reasoning' })
+  })
+
   it('every capability has at least one provider', () => {
     for (const cap of ALL_CAPABILITIES) {
       const providers = getProviders(cap)

package/tests/search-web-exa-compat.test.ts

@@ -0,0 +1,11 @@
+import { describe, expect, it } from 'vitest'
+import { z } from 'zod'
+import { searchWebSchema } from '../src/tools/search-web.js'
+
+const schema = z.object(searchWebSchema)
+
+describe('search_web Exa compatibility', () => {
+  it.each(['general', 'neural', 'deep', 'deep-reasoning'])('accepts %s', (search_type) => {
+    expect(schema.safeParse({ query: 'ColdIQ', search_type }).success).toBe(true)
+  })
+})

package/tests/tools/call-endpoint.test.ts

@@ -38,4 +38,20 @@ describe('call_endpoint handler', () => {
     expect(parsed.status).toBe(404)
     expect(parsed.error).toBe('Endpoint call failed')
   })
+
+  it('forwards the HTTP method for a shared endpoint path', async () => {
+    let capturedBody: unknown
+    globalThis.fetch = vi.fn(async (_url: string | URL | Request, opts?: RequestInit) => {
+      capturedBody = opts?.body ? JSON.parse(opts.body as string) : undefined
+      return new Response(JSON.stringify({ ok: true, status: 201, data: { id: 'run_1' } }), { status: 200 })
+    }) as typeof fetch
+
+    await callEndpointHandler({
+      api: 'exa', path: '/exa/agent/runs', method: 'POST', params: { query: 'Research ColdIQ' },
+    })
+
+    expect(capturedBody).toEqual({
+      api: 'exa', path: '/exa/agent/runs', method: 'POST', params: { query: 'Research ColdIQ' },
+    })
+  })
 })