@coldiq/mcp 0.5.3 → 5.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/provider-availability.d.ts.map +1 -1
- package/dist/provider-availability.js +3 -0
- package/dist/provider-availability.js.map +1 -1
- package/dist/tools/extract-post-engagement.d.ts +0 -7
- package/dist/tools/extract-post-engagement.d.ts.map +1 -1
- package/dist/tools/extract-post-engagement.js +205 -21
- package/dist/tools/extract-post-engagement.js.map +1 -1
- package/package.json +1 -1
- package/src/provider-availability.ts +3 -0
- package/src/tools/extract-post-engagement.ts +229 -27
- package/tests/provider-availability.test.ts +23 -4
- package/tests/registry-search-companies.test.ts +2 -2
- package/tests/registry.test.ts +13 -13
- package/tests/tools/extract-post-engagement.test.ts +167 -3
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"provider-availability.d.ts","sourceRoot":"","sources":["../src/provider-availability.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"provider-availability.d.ts","sourceRoot":"","sources":["../src/provider-availability.ts"],"names":[],"mappings":"AAwBA,8EAA8E;AAC9E,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAI3D"}
|
|
@@ -6,9 +6,12 @@
|
|
|
6
6
|
* configured and testable while public routing and tool schemas omit them.
|
|
7
7
|
*/
|
|
8
8
|
const DISABLED_PROVIDERS = new Set([
|
|
9
|
+
'companyenrich',
|
|
9
10
|
'people-data-labs',
|
|
10
11
|
]);
|
|
11
12
|
const PROVIDER_ALIASES = {
|
|
13
|
+
'company-enrich': 'companyenrich',
|
|
14
|
+
'companyenrich-props': 'companyenrich',
|
|
12
15
|
pdl: 'people-data-labs',
|
|
13
16
|
'pdl-person-enrich': 'people-data-labs',
|
|
14
17
|
'pdl-person-identify': 'people-data-labs',
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"provider-availability.js","sourceRoot":"","sources":["../src/provider-availability.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAAC;IACjC,kBAAkB;CACnB,CAAC,CAAA;AAEF,MAAM,gBAAgB,GAA2B;IAC/C,GAAG,EAAE,kBAAkB;IACvB,mBAAmB,EAAE,kBAAkB;IACvC,qBAAqB,EAAE,kBAAkB;CAC1C,CAAA;AAED,SAAS,oBAAoB,CAAC,QAAgB;IAC5C,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAA;AAC9D,CAAC;AAED,8EAA8E;AAC9E,MAAM,UAAU,iBAAiB,CAAC,QAAgB;IAChD,MAAM,UAAU,GAAG,oBAAoB,CAAC,QAAQ,CAAC,CAAA;IACjD,MAAM,SAAS,GAAG,gBAAgB,CAAC,UAAU,CAAC,IAAI,UAAU,CAAA;IAC5D,OAAO,CAAC,kBAAkB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;AAC3C,CAAC"}
|
|
1
|
+
{"version":3,"file":"provider-availability.js","sourceRoot":"","sources":["../src/provider-availability.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAAC;IACjC,eAAe;IACf,kBAAkB;CACnB,CAAC,CAAA;AAEF,MAAM,gBAAgB,GAA2B;IAC/C,gBAAgB,EAAE,eAAe;IACjC,qBAAqB,EAAE,eAAe;IACtC,GAAG,EAAE,kBAAkB;IACvB,mBAAmB,EAAE,kBAAkB;IACvC,qBAAqB,EAAE,kBAAkB;CAC1C,CAAA;AAED,SAAS,oBAAoB,CAAC,QAAgB;IAC5C,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAA;AAC9D,CAAC;AAED,8EAA8E;AAC9E,MAAM,UAAU,iBAAiB,CAAC,QAAgB;IAChD,MAAM,UAAU,GAAG,oBAAoB,CAAC,QAAQ,CAAC,CAAA;IACjD,MAAM,SAAS,GAAG,gBAAgB,CAAC,UAAU,CAAC,IAAI,UAAU,CAAA;IAC5D,OAAO,CAAC,kBAAkB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;AAC3C,CAAC"}
|
|
@@ -4,18 +4,11 @@ export declare const extractPostEngagementDescription: string;
|
|
|
4
4
|
export declare const extractPostEngagementSchema: {
|
|
5
5
|
post_url: z.ZodString;
|
|
6
6
|
type: z.ZodDefault<z.ZodEnum<["comments", "reactions", "both"]>>;
|
|
7
|
-
include_replies: z.ZodOptional<z.ZodBoolean>;
|
|
8
7
|
};
|
|
9
8
|
export declare function extractPostEngagementHandler(input: Record<string, unknown>): Promise<{
|
|
10
9
|
content: {
|
|
11
10
|
type: "text";
|
|
12
11
|
text: string;
|
|
13
12
|
}[];
|
|
14
|
-
isError: boolean;
|
|
15
|
-
} | {
|
|
16
|
-
content: {
|
|
17
|
-
type: "text";
|
|
18
|
-
text: string;
|
|
19
|
-
}[];
|
|
20
13
|
}>;
|
|
21
14
|
//# sourceMappingURL=extract-post-engagement.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extract-post-engagement.d.ts","sourceRoot":"","sources":["../../src/tools/extract-post-engagement.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAGvB,eAAO,MAAM,yBAAyB,4BAA4B,CAAA;AAElE,eAAO,MAAM,gCAAgC,
|
|
1
|
+
{"version":3,"file":"extract-post-engagement.d.ts","sourceRoot":"","sources":["../../src/tools/extract-post-engagement.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAGvB,eAAO,MAAM,yBAAyB,4BAA4B,CAAA;AAElE,eAAO,MAAM,gCAAgC,QAIoO,CAAA;AAEjR,eAAO,MAAM,2BAA2B;;;CAavC,CAAA;AAmLD,wBAAsB,4BAA4B,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;;;;;GAqIhF"}
|
|
@@ -3,7 +3,8 @@ import { callApi } from '../client.js';
|
|
|
3
3
|
export const extractPostEngagementName = 'extract_post_engagement';
|
|
4
4
|
export const extractPostEngagementDescription = 'Extract the people who engaged with a LinkedIn post — commenters and/or reactors — as a deduplicated list of contacts (name, profile URL, headline). ' +
|
|
5
5
|
'Use this for social-signal prospecting: pull everyone who engaged with a viral post, then chain the results into enrich_person / find_email to get roles and work emails. ' +
|
|
6
|
-
'Runs an async extraction job (typically
|
|
6
|
+
'Runs an async extraction job (typically 1–3 minutes; longer for posts with 1000+ interactions) and returns every extracted person once ready. ' +
|
|
7
|
+
'If the primary source fails or its account is out of credits, automatically retries on a second provider and says so in `_meta.provider` — those records carry name/profile URL/headline only, so enrich them before use. Pricing is returned in the response credit headers.';
|
|
7
8
|
export const extractPostEngagementSchema = {
|
|
8
9
|
post_url: z
|
|
9
10
|
.string()
|
|
@@ -13,14 +14,132 @@ export const extractPostEngagementSchema = {
|
|
|
13
14
|
.enum(['comments', 'reactions', 'both'])
|
|
14
15
|
.default('both')
|
|
15
16
|
.describe('Which engagement to extract: "comments" (people who commented), "reactions" (people who reacted), or "both" (default — deduplicated across both).'),
|
|
16
|
-
include_replies:
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
17
|
+
// `include_replies` was removed: this tool returns a DEDUPLICATED contact list,
|
|
18
|
+
// where a person is one entry regardless of whether they commented or replied,
|
|
19
|
+
// so the flag never changed the result. Use GET /v1/jungler/workbooks/{id}/comments
|
|
20
|
+
// to work with individual comments and their reply metadata.
|
|
20
21
|
};
|
|
22
|
+
// One page covers most posts; the loop below still walks the rest when it doesn't.
|
|
23
|
+
const CONTACTS_PAGE_SIZE = 500;
|
|
24
|
+
// Backstop so a mis-paginating upstream can't spin this tool forever.
|
|
25
|
+
const MAX_CONTACT_PAGES = 20;
|
|
26
|
+
/**
|
|
27
|
+
* True unless the contact demonstrably engaged with nothing.
|
|
28
|
+
*
|
|
29
|
+
* The workbook always collects the post itself, so the post's AUTHOR comes back
|
|
30
|
+
* as a contact with `stats: {comments: 0, reactions: 0}` even when nobody
|
|
31
|
+
* engaged. This tool promises the people who engaged, so a zero/zero contact is
|
|
32
|
+
* not one of them. Anything without usable stats is kept — never drop a contact
|
|
33
|
+
* on ambiguity.
|
|
34
|
+
*/
|
|
35
|
+
function engaged(contact) {
|
|
36
|
+
if (!contact || typeof contact !== 'object')
|
|
37
|
+
return true;
|
|
38
|
+
const stats = contact.stats;
|
|
39
|
+
if (!stats || typeof stats !== 'object')
|
|
40
|
+
return true;
|
|
41
|
+
const { comments, reactions } = stats;
|
|
42
|
+
if (typeof comments !== 'number' || typeof reactions !== 'number')
|
|
43
|
+
return true;
|
|
44
|
+
return comments > 0 || reactions > 0;
|
|
45
|
+
}
|
|
46
|
+
/** Read one page of the contacts envelope, tolerating a bare-array response. */
|
|
47
|
+
function readContactsPage(data) {
|
|
48
|
+
if (Array.isArray(data))
|
|
49
|
+
return { items: data, pages: 1 };
|
|
50
|
+
if (data && typeof data === 'object') {
|
|
51
|
+
const obj = data;
|
|
52
|
+
if (Array.isArray(obj.items)) {
|
|
53
|
+
return {
|
|
54
|
+
items: obj.items,
|
|
55
|
+
total: typeof obj.total === 'number' ? obj.total : undefined,
|
|
56
|
+
pages: typeof obj.pages === 'number' ? obj.pages : undefined,
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
return { items: [], pages: 1 };
|
|
61
|
+
}
|
|
21
62
|
function sleep(ms) {
|
|
22
63
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
23
64
|
}
|
|
65
|
+
// ── Fallback provider ───────────────────────────────────────────────────────
|
|
66
|
+
//
|
|
67
|
+
// Jungler is the primary: it charges a flat rate per post regardless of size and
|
|
68
|
+
// returns enriched contacts (work email, company domain, title, seniority). Lima
|
|
69
|
+
// Data costs the same per post but returns profile URLs only, so it needs a
|
|
70
|
+
// separate enrichment pass — hence second, not first.
|
|
71
|
+
//
|
|
72
|
+
// We reach for it ONLY when Jungler failed for a reason that is ours, not the
|
|
73
|
+
// post's: a failed run, or an upstream account out of credits. A genuinely empty
|
|
74
|
+
// post must NOT fall through — Lima would return empty too, for another charge.
|
|
75
|
+
// Same env knobs as the primary poll, so a caller (or a test) can tighten both.
|
|
76
|
+
const fallbackPollMs = () => parseInt(process.env.COLDIQ_ENGAGEMENT_POLL_MS ?? '5000', 10);
|
|
77
|
+
const fallbackTimeoutMs = () => parseInt(process.env.COLDIQ_ENGAGEMENT_FALLBACK_TIMEOUT_MS ?? '240000', 10);
|
|
78
|
+
/** Flatten Lima's `{commenters:{comments:[{actor}]}, reactors:{reactions:[{actor}]}}` into contacts. */
|
|
79
|
+
function limaEngagersToPeople(data, type) {
|
|
80
|
+
if (!data || typeof data !== 'object')
|
|
81
|
+
return [];
|
|
82
|
+
const d = data;
|
|
83
|
+
const rows = [];
|
|
84
|
+
if (type !== 'reactions') {
|
|
85
|
+
for (const c of d.commenters?.comments ?? [])
|
|
86
|
+
rows.push({ ...c, engagement: 'COMMENT' });
|
|
87
|
+
}
|
|
88
|
+
if (type !== 'comments') {
|
|
89
|
+
for (const r of d.reactors?.reactions ?? [])
|
|
90
|
+
rows.push({ ...r, engagement: 'REACTION' });
|
|
91
|
+
}
|
|
92
|
+
// Deduplicate by profile URL: someone who both commented and reacted is one person.
|
|
93
|
+
const byProfile = new Map();
|
|
94
|
+
for (const row of rows) {
|
|
95
|
+
const actor = (row.actor ?? {});
|
|
96
|
+
const key = String(actor.profile_url ?? actor.name ?? Math.random());
|
|
97
|
+
const existing = byProfile.get(key);
|
|
98
|
+
if (existing) {
|
|
99
|
+
const seen = existing.engagement_types;
|
|
100
|
+
if (!seen.includes(row.engagement))
|
|
101
|
+
seen.push(row.engagement);
|
|
102
|
+
continue;
|
|
103
|
+
}
|
|
104
|
+
byProfile.set(key, {
|
|
105
|
+
name: actor.name ?? null,
|
|
106
|
+
profile_url: actor.profile_url ?? null,
|
|
107
|
+
description: actor.headline ?? null,
|
|
108
|
+
profile_image_url: actor.image_url ?? null,
|
|
109
|
+
engagement_types: [row.engagement],
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
return [...byProfile.values()];
|
|
113
|
+
}
|
|
114
|
+
async function extractViaLimaData(postUrl, type) {
|
|
115
|
+
const submit = await callApi('POST', '/limadata/batch/post-engagements', { urls: [postUrl] });
|
|
116
|
+
if (!submit.ok) {
|
|
117
|
+
return { ok: false, error: extractErrorMessage(submit.data) ?? `Fallback provider failed to start (status ${submit.status})` };
|
|
118
|
+
}
|
|
119
|
+
const batchId = submit.data.batch_id;
|
|
120
|
+
if (batchId === undefined)
|
|
121
|
+
return { ok: false, error: 'Fallback provider did not return a batch id' };
|
|
122
|
+
const creditsCharged = Number(submit.headers['x-coldiq-credits-charged']);
|
|
123
|
+
const timeoutMs = fallbackTimeoutMs();
|
|
124
|
+
const deadline = Date.now() + timeoutMs;
|
|
125
|
+
while (Date.now() < deadline) {
|
|
126
|
+
await sleep(fallbackPollMs());
|
|
127
|
+
const poll = await callApi('POST', '/limadata/batch/results', { batch_id: String(batchId) });
|
|
128
|
+
if (!poll.ok)
|
|
129
|
+
continue;
|
|
130
|
+
const body = poll.data;
|
|
131
|
+
const item = body.items?.[0];
|
|
132
|
+
if (!item || String(item.status).toLowerCase() !== 'completed')
|
|
133
|
+
continue;
|
|
134
|
+
return {
|
|
135
|
+
ok: true,
|
|
136
|
+
people: limaEngagersToPeople(item.data, type),
|
|
137
|
+
creditsCharged: Number.isFinite(creditsCharged) ? creditsCharged : undefined,
|
|
138
|
+
batchId,
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
return { ok: false, error: `Fallback provider did not finish within ${Math.round(timeoutMs / 1000)}s`, batchId };
|
|
142
|
+
}
|
|
24
143
|
function errorResult(error, extra) {
|
|
25
144
|
return {
|
|
26
145
|
content: [{ type: 'text', text: JSON.stringify({ error, ...extra }) }],
|
|
@@ -34,10 +153,43 @@ function extractErrorMessage(data) {
|
|
|
34
153
|
}
|
|
35
154
|
return undefined;
|
|
36
155
|
}
|
|
156
|
+
/**
|
|
157
|
+
* Try the fallback provider after the primary failed; if it also fails, report
|
|
158
|
+
* the ORIGINAL failure (that is the one the caller can act on) with the
|
|
159
|
+
* fallback's outcome attached.
|
|
160
|
+
*
|
|
161
|
+
* Set COLDIQ_ENGAGEMENT_FALLBACK=0 to disable — the fallback is a second billable
|
|
162
|
+
* provider, so anyone running to a strict budget must be able to opt out.
|
|
163
|
+
*/
|
|
164
|
+
async function fallbackOr(postUrl, type, primaryError, extra) {
|
|
165
|
+
if (process.env.COLDIQ_ENGAGEMENT_FALLBACK === '0') {
|
|
166
|
+
return errorResult(primaryError, { post_url: postUrl, ...extra });
|
|
167
|
+
}
|
|
168
|
+
const fb = await extractViaLimaData(postUrl, type);
|
|
169
|
+
if (!fb.ok) {
|
|
170
|
+
return errorResult(primaryError, { post_url: postUrl, ...extra, fallback_error: fb.error });
|
|
171
|
+
}
|
|
172
|
+
const meta = { provider: 'limadata', primary_error: primaryError };
|
|
173
|
+
if (fb.creditsCharged !== undefined)
|
|
174
|
+
meta.credits_charged = fb.creditsCharged;
|
|
175
|
+
return {
|
|
176
|
+
content: [{
|
|
177
|
+
type: 'text',
|
|
178
|
+
text: JSON.stringify({
|
|
179
|
+
data: { post_url: postUrl, type, people: fb.people, count: fb.people.length },
|
|
180
|
+
// Say plainly that these records are thinner than the primary's, so the
|
|
181
|
+
// caller knows to run enrichment before treating them as contacts.
|
|
182
|
+
_meta: {
|
|
183
|
+
...meta,
|
|
184
|
+
note: 'Served by the fallback provider after the primary failed. These records carry name, profile URL and headline only — no work email or company data. Run enrich_person / find_email on them if you need contact details.',
|
|
185
|
+
},
|
|
186
|
+
}),
|
|
187
|
+
}],
|
|
188
|
+
};
|
|
189
|
+
}
|
|
37
190
|
export async function extractPostEngagementHandler(input) {
|
|
38
191
|
const postUrl = input.post_url;
|
|
39
192
|
const type = input.type ?? 'both';
|
|
40
|
-
const includeReplies = input.include_replies;
|
|
41
193
|
const dataTypes = type === 'comments' ? ['comment'] : type === 'reactions' ? ['reaction'] : ['comment', 'reaction'];
|
|
42
194
|
// Step 1 — create the extraction task. Billing happens here.
|
|
43
195
|
const createRes = await callApi('POST', '/jungler/workbooks', {
|
|
@@ -45,20 +197,26 @@ export async function extractPostEngagementHandler(input) {
|
|
|
45
197
|
data_types: dataTypes,
|
|
46
198
|
});
|
|
47
199
|
if (!createRes.ok) {
|
|
48
|
-
return
|
|
200
|
+
return fallbackOr(postUrl, type, extractErrorMessage(createRes.data) ?? `Failed to start extraction (status ${createRes.status})`);
|
|
49
201
|
}
|
|
50
202
|
const createData = createRes.data;
|
|
51
203
|
const taskId = createData.task_id;
|
|
52
204
|
if (!taskId) {
|
|
53
|
-
return
|
|
205
|
+
return fallbackOr(postUrl, type, 'Extraction job did not return a task id');
|
|
54
206
|
}
|
|
207
|
+
// Kept so a timeout can still hand the caller a way to collect what they paid
|
|
208
|
+
// for — the workbook stays readable upstream for 12 hours.
|
|
209
|
+
const createWorkbookId = createData.workbook_id;
|
|
55
210
|
// Credit headers are emitted on the create call (the only billed step).
|
|
56
211
|
const creditsCharged = Number(createRes.headers['x-coldiq-credits-charged']);
|
|
57
212
|
const creditsRemaining = Number(createRes.headers['x-coldiq-credits-remaining']);
|
|
58
213
|
// Step 2 — poll task status until it resolves. The status endpoint is free, so
|
|
59
214
|
// polling does not bill; only the create call above charged credits.
|
|
60
|
-
|
|
61
|
-
|
|
215
|
+
// Upstream quotes 1-3 minutes and explicitly warns that large posts (1000+
|
|
216
|
+
// interactions) take longer — which is exactly the viral post this tool is for.
|
|
217
|
+
// A 180s deadline cut those off mid-extraction and reported a bogus timeout.
|
|
218
|
+
const pollIntervalMs = parseInt(process.env.COLDIQ_ENGAGEMENT_POLL_MS ?? '5000', 10);
|
|
219
|
+
const timeoutMs = parseInt(process.env.COLDIQ_ENGAGEMENT_TIMEOUT_MS ?? '600000', 10);
|
|
62
220
|
const maxPollErrors = 3;
|
|
63
221
|
const deadline = Date.now() + timeoutMs;
|
|
64
222
|
let workbookId;
|
|
@@ -74,41 +232,67 @@ export async function extractPostEngagementHandler(input) {
|
|
|
74
232
|
continue;
|
|
75
233
|
}
|
|
76
234
|
consecutivePollErrors = 0;
|
|
77
|
-
const
|
|
235
|
+
const statusData = statusRes.data;
|
|
236
|
+
const status = statusData.status;
|
|
78
237
|
if (status === 'success') {
|
|
79
|
-
|
|
238
|
+
// A credits-exhausted run is terminal but collected little or nothing
|
|
239
|
+
// because OUR provider account ran dry — the caller's request is sound, so
|
|
240
|
+
// route it to the fallback rather than handing back a hollow result. This
|
|
241
|
+
// charge is refunded server-side.
|
|
242
|
+
if (statusData.credits_exhausted) {
|
|
243
|
+
return fallbackOr(postUrl, type, 'Engagement extraction stopped early — the data provider account is out of credits (this request was refunded)', { task_id: taskId });
|
|
244
|
+
}
|
|
245
|
+
workbookId = statusData.workbook_id;
|
|
80
246
|
break;
|
|
81
247
|
}
|
|
82
248
|
if (status === 'failure') {
|
|
83
|
-
|
|
249
|
+
// The status route refunds the create charge when a run reaches failure,
|
|
250
|
+
// so the user is made whole — say so rather than leaving them to wonder.
|
|
251
|
+
return fallbackOr(postUrl, type, 'Engagement extraction failed upstream — the post may be private, deleted, or have no engagement. The credits for this task have been refunded.', { task_id: taskId });
|
|
84
252
|
}
|
|
85
253
|
}
|
|
86
254
|
if (!workbookId) {
|
|
87
|
-
|
|
255
|
+
// Giving up here does NOT cancel the run, and the create call was already
|
|
256
|
+
// billed. Never drop that on the floor: hand back the ids so the caller can
|
|
257
|
+
// collect the results (upstream keeps the workbook for 12 hours) instead of
|
|
258
|
+
// paying again for a re-run that is probably still in flight.
|
|
259
|
+
return errorResult(`Engagement extraction is still running after ${Math.round(timeoutMs / 1000)}s. It has NOT been cancelled and no re-run is needed — poll GET /v1/jungler/tasks/${taskId}/status until it reports success, then fetch the people from GET /v1/jungler/workbooks/${createWorkbookId ?? '{workbook_id}'}/contacts. Results stay available for 12 hours after the task was created.`, { post_url: postUrl, task_id: taskId, workbook_id: createWorkbookId, status: 'running' });
|
|
88
260
|
}
|
|
89
261
|
// Step 3 — fetch the deduplicated people. activity_filter narrows to commenters
|
|
90
262
|
// or reactors; omitted for "both" so the upstream returns all unique contacts.
|
|
91
|
-
|
|
263
|
+
// Contacts are paginated, and a viral post — the case this tool exists for —
|
|
264
|
+
// routinely exceeds one page, so walk every page rather than truncating to the
|
|
265
|
+
// first. Reads are free, so pagination costs the caller nothing.
|
|
266
|
+
const queryParams = { page_size: String(CONTACTS_PAGE_SIZE) };
|
|
92
267
|
if (type === 'comments')
|
|
93
268
|
queryParams.activity_filter = 'commenters';
|
|
94
269
|
else if (type === 'reactions')
|
|
95
270
|
queryParams.activity_filter = 'reactors';
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
271
|
+
const people = [];
|
|
272
|
+
let totalContacts;
|
|
273
|
+
for (let page = 1; page <= MAX_CONTACT_PAGES; page++) {
|
|
274
|
+
const contactsRes = await callApi('GET', `/jungler/workbooks/${workbookId}/contacts`, undefined, { ...queryParams, page: String(page) });
|
|
275
|
+
if (!contactsRes.ok) {
|
|
276
|
+
return errorResult(extractErrorMessage(contactsRes.data) ?? 'Failed to fetch extracted people', { post_url: postUrl, task_id: taskId, workbook_id: workbookId });
|
|
277
|
+
}
|
|
278
|
+
const { items, total, pages } = readContactsPage(contactsRes.data);
|
|
279
|
+
people.push(...items.filter(engaged));
|
|
280
|
+
totalContacts = total;
|
|
281
|
+
if (items.length === 0 || (pages !== undefined && page >= pages))
|
|
282
|
+
break;
|
|
101
283
|
}
|
|
102
284
|
const meta = {};
|
|
103
285
|
if (Number.isFinite(creditsCharged))
|
|
104
286
|
meta.credits_charged = creditsCharged;
|
|
105
287
|
if (Number.isFinite(creditsRemaining))
|
|
106
288
|
meta.credits_remaining = creditsRemaining;
|
|
289
|
+
if (totalContacts !== undefined && totalContacts > people.length)
|
|
290
|
+
meta.truncated_from = totalContacts;
|
|
107
291
|
return {
|
|
108
292
|
content: [{
|
|
109
293
|
type: 'text',
|
|
110
294
|
text: JSON.stringify({
|
|
111
|
-
data: { post_url: postUrl, type, people:
|
|
295
|
+
data: { post_url: postUrl, type, people, count: people.length },
|
|
112
296
|
_meta: meta,
|
|
113
297
|
}),
|
|
114
298
|
}],
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extract-post-engagement.js","sourceRoot":"","sources":["../../src/tools/extract-post-engagement.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AACvB,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAA;AAEtC,MAAM,CAAC,MAAM,yBAAyB,GAAG,yBAAyB,CAAA;AAElE,MAAM,CAAC,MAAM,gCAAgC,GAC3C,uJAAuJ;IACvJ,4KAA4K;IAC5K,8IAA8I,CAAA;AAEhJ,MAAM,CAAC,MAAM,2BAA2B,GAAG;IACzC,QAAQ,EAAE,CAAC;SACR,MAAM,EAAE;SACR,GAAG,EAAE;SACL,QAAQ,CAAC,wJAAwJ,CAAC;IACrK,IAAI,EAAE,CAAC;SACJ,IAAI,CAAC,CAAC,UAAU,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;SACvC,OAAO,CAAC,MAAM,CAAC;SACf,QAAQ,CAAC,mJAAmJ,CAAC;IAChK,eAAe,EAAE,CAAC;SACf,OAAO,EAAE;SACT,QAAQ,EAAE;SACV,QAAQ,CAAC,mGAAmG,CAAC;CACjH,CAAA;AAED,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAA;AAC1D,CAAC;AAED,SAAS,WAAW,CAAC,KAAa,EAAE,KAA+B;IACjE,OAAO;QACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,GAAG,KAAK,EAAE,CAAC,EAAE,CAAC;QAC/E,OAAO,EAAE,IAAI;KACd,CAAA;AACH,CAAC;AAED,SAAS,mBAAmB,CAAC,IAAa;IACxC,IAAI,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;QACxD,MAAM,CAAC,GAAI,IAAgC,CAAC,KAAK,CAAA;QACjD,OAAO,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAA;IACtD,CAAC;IACD,OAAO,SAAS,CAAA;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,4BAA4B,CAAC,KAA8B;IAC/E,MAAM,OAAO,GAAG,KAAK,CAAC,QAAkB,CAAA;IACxC,MAAM,IAAI,GAAI,KAAK,CAAC,IAAsD,IAAI,MAAM,CAAA;IACpF,MAAM,cAAc,GAAG,KAAK,CAAC,eAAsC,CAAA;IAEnE,MAAM,SAAS,GACb,IAAI,KAAK,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,UAAU,CAAC,CAAA;IAEnG,6DAA6D;IAC7D,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,MAAM,EAAE,oBAAoB,EAAE;QAC5D,QAAQ,EAAE,OAAO;QACjB,UAAU,EAAE,SAAS;KACtB,CAAC,CAAA;IACF,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,CAAC;QAClB,OAAO,WAAW,CAAC,mBAAmB,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,sCAAsC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAA;IACtH,CAAC;IACD,MAAM,UAAU,GAAG,SAAS,CAAC,IAA4B,CAAA;IACzD,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,CAAA;IACjC,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,WAAW,CAAC,yCAAyC,CAAC,CAAA;IAC/D,CAAC;IACD,wEAAwE;IACxE,MAAM,cAAc,GAAG,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,0BAA0B,CAAC,CAAC,CAAA;IAC5E,MAAM,gBAAgB,GAAG,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,4BAA4B,CAAC,CAAC,CAAA;IAEhF,+EAA+E;IAC/E,qEAAqE;IACrE,MAAM,cAAc,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,yBAAyB,IAAI,MAAM,EAAE,EAAE,CAAC,CAAA;IACpF,MAAM,SAAS,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,4BAA4B,IAAI,QAAQ,EAAE,EAAE,CAAC,CAAA;IACpF,MAAM,aAAa,GAAG,CAAC,CAAA;IACvB,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAA;IAEvC,IAAI,UAA8B,CAAA;IAClC,IAAI,qBAAqB,GAAG,CAAC,CAAA;IAE7B,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,EAAE,CAAC;QAC7B,MAAM,KAAK,CAAC,cAAc,CAAC,CAAA;QAC3B,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,KAAK,EAAE,kBAAkB,MAAM,SAAS,CAAC,CAAA;QACzE,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,CAAC;YAClB,qBAAqB,EAAE,CAAA;YACvB,IAAI,qBAAqB,IAAI,aAAa,EAAE,CAAC;gBAC3C,OAAO,WAAW,CAAC,iDAAiD,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAA;YAC9F,CAAC;YACD,SAAQ;QACV,CAAC;QACD,qBAAqB,GAAG,CAAC,CAAA;QACzB,MAAM,MAAM,GAAI,SAAS,CAAC,IAAkD,CAAC,MAAM,CAAA;QACnF,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,UAAU,GAAI,SAAS,CAAC,IAAiC,CAAC,WAAW,CAAA;YACrE,MAAK;QACP,CAAC;QACD,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,OAAO,WAAW,CAAC,iGAAiG,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAA;QAC9I,CAAC;IACH,CAAC;IAED,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,OAAO,WAAW,CAAC,iDAAiD,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,uBAAuB,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAA;IACjJ,CAAC;IAED,gFAAgF;IAChF,+EAA+E;IAC/E,MAAM,WAAW,GAA2B,EAAE,CAAA;IAC9C,IAAI,IAAI,KAAK,UAAU;QAAE,WAAW,CAAC,eAAe,GAAG,YAAY,CAAA;SAC9D,IAAI,IAAI,KAAK,WAAW;QAAE,WAAW,CAAC,eAAe,GAAG,UAAU,CAAA;IACvE,IAAI,cAAc,KAAK,SAAS;QAAE,WAAW,CAAC,eAAe,GAAG,MAAM,CAAC,cAAc,CAAC,CAAA;IAEtF,MAAM,WAAW,GAAG,MAAM,OAAO,CAC/B,KAAK,EACL,sBAAsB,UAAU,WAAW,EAC3C,SAAS,EACT,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAC9D,CAAA;IACD,IAAI,CAAC,WAAW,CAAC,EAAE,EAAE,CAAC;QACpB,OAAO,WAAW,CAAC,mBAAmB,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,kCAAkC,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAA;IACxH,CAAC;IAED,MAAM,IAAI,GAA4B,EAAE,CAAA;IACxC,IAAI,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAC;QAAE,IAAI,CAAC,eAAe,GAAG,cAAc,CAAA;IAC1E,IAAI,MAAM,CAAC,QAAQ,CAAC,gBAAgB,CAAC;QAAE,IAAI,CAAC,iBAAiB,GAAG,gBAAgB,CAAA;IAEhF,OAAO;QACL,OAAO,EAAE,CAAC;gBACR,IAAI,EAAE,MAAe;gBACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,IAAI,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,CAAC,IAAI,EAAE;oBAC3D,KAAK,EAAE,IAAI;iBACZ,CAAC;aACH,CAAC;KACH,CAAA;AACH,CAAC"}
|
|
1
|
+
{"version":3,"file":"extract-post-engagement.js","sourceRoot":"","sources":["../../src/tools/extract-post-engagement.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AACvB,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAA;AAEtC,MAAM,CAAC,MAAM,yBAAyB,GAAG,yBAAyB,CAAA;AAElE,MAAM,CAAC,MAAM,gCAAgC,GAC3C,uJAAuJ;IACvJ,4KAA4K;IAC5K,gJAAgJ;IAChJ,+QAA+Q,CAAA;AAEjR,MAAM,CAAC,MAAM,2BAA2B,GAAG;IACzC,QAAQ,EAAE,CAAC;SACR,MAAM,EAAE;SACR,GAAG,EAAE;SACL,QAAQ,CAAC,wJAAwJ,CAAC;IACrK,IAAI,EAAE,CAAC;SACJ,IAAI,CAAC,CAAC,UAAU,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;SACvC,OAAO,CAAC,MAAM,CAAC;SACf,QAAQ,CAAC,mJAAmJ,CAAC;IAChK,gFAAgF;IAChF,+EAA+E;IAC/E,oFAAoF;IACpF,6DAA6D;CAC9D,CAAA;AAED,mFAAmF;AACnF,MAAM,kBAAkB,GAAG,GAAG,CAAA;AAC9B,sEAAsE;AACtE,MAAM,iBAAiB,GAAG,EAAE,CAAA;AAE5B;;;;;;;;GAQG;AACH,SAAS,OAAO,CAAC,OAAgB;IAC/B,IAAI,CAAC,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAA;IACxD,MAAM,KAAK,GAAI,OAA+B,CAAC,KAAK,CAAA;IACpD,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAA;IACpD,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,KAAoD,CAAA;IACpF,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,OAAO,SAAS,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAA;IAC9E,OAAO,QAAQ,GAAG,CAAC,IAAI,SAAS,GAAG,CAAC,CAAA;AACtC,CAAC;AAED,gFAAgF;AAChF,SAAS,gBAAgB,CAAC,IAAa;IACrC,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;QAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAA;IACzD,IAAI,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;QACrC,MAAM,GAAG,GAAG,IAA6D,CAAA;QACzE,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YAC7B,OAAO;gBACL,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,KAAK,EAAE,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;gBAC5D,KAAK,EAAE,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;aAC7D,CAAA;QACH,CAAC;IACH,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAA;AAChC,CAAC;AAED,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAA;AAC1D,CAAC;AAED,+EAA+E;AAC/E,EAAE;AACF,iFAAiF;AACjF,iFAAiF;AACjF,4EAA4E;AAC5E,sDAAsD;AACtD,EAAE;AACF,8EAA8E;AAC9E,iFAAiF;AACjF,gFAAgF;AAChF,gFAAgF;AAChF,MAAM,cAAc,GAAG,GAAG,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,yBAAyB,IAAI,MAAM,EAAE,EAAE,CAAC,CAAA;AAC1F,MAAM,iBAAiB,GAAG,GAAG,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,qCAAqC,IAAI,QAAQ,EAAE,EAAE,CAAC,CAAA;AAE3G,wGAAwG;AACxG,SAAS,oBAAoB,CAAC,IAAa,EAAE,IAAuC;IAClF,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ;QAAE,OAAO,EAAE,CAAA;IAChD,MAAM,CAAC,GAAG,IAGT,CAAA;IACD,MAAM,IAAI,GAAmD,EAAE,CAAA;IAC/D,IAAI,IAAI,KAAK,WAAW,EAAE,CAAC;QACzB,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,QAAQ,IAAI,EAAE;YAAE,IAAI,CAAC,IAAI,CAAC,EAAE,GAAI,CAAY,EAAE,UAAU,EAAE,SAAS,EAAW,CAAC,CAAA;IAC/G,CAAC;IACD,IAAI,IAAI,KAAK,UAAU,EAAE,CAAC;QACxB,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,SAAS,IAAI,EAAE;YAAE,IAAI,CAAC,IAAI,CAAC,EAAE,GAAI,CAAY,EAAE,UAAU,EAAE,UAAU,EAAW,CAAC,CAAA;IAC/G,CAAC;IACD,oFAAoF;IACpF,MAAM,SAAS,GAAG,IAAI,GAAG,EAAmC,CAAA;IAC5D,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,KAAK,GAAG,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE,CAA4B,CAAA;QAC1D,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,CAAA;QACpE,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QACnC,IAAI,QAAQ,EAAE,CAAC;YACb,MAAM,IAAI,GAAG,QAAQ,CAAC,gBAA4B,CAAA;YAClD,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC;gBAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CAAA;YAC7D,SAAQ;QACV,CAAC;QACD,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE;YACjB,IAAI,EAAE,KAAK,CAAC,IAAI,IAAI,IAAI;YACxB,WAAW,EAAE,KAAK,CAAC,WAAW,IAAI,IAAI;YACtC,WAAW,EAAE,KAAK,CAAC,QAAQ,IAAI,IAAI;YACnC,iBAAiB,EAAE,KAAK,CAAC,SAAS,IAAI,IAAI;YAC1C,gBAAgB,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC;SACnC,CAAC,CAAA;IACJ,CAAC;IACD,OAAO,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,CAAA;AAChC,CAAC;AAED,KAAK,UAAU,kBAAkB,CAAC,OAAe,EAAE,IAAuC;IACxF,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,MAAM,EAAE,kCAAkC,EAAE,EAAE,IAAI,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;IAC7F,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;QACf,OAAO,EAAE,EAAE,EAAE,KAAc,EAAE,KAAK,EAAE,mBAAmB,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,6CAA6C,MAAM,CAAC,MAAM,GAAG,EAAE,CAAA;IACzI,CAAC;IACD,MAAM,OAAO,GAAI,MAAM,CAAC,IAAuC,CAAC,QAAQ,CAAA;IACxE,IAAI,OAAO,KAAK,SAAS;QAAE,OAAO,EAAE,EAAE,EAAE,KAAc,EAAE,KAAK,EAAE,6CAA6C,EAAE,CAAA;IAE9G,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,0BAA0B,CAAC,CAAC,CAAA;IACzE,MAAM,SAAS,GAAG,iBAAiB,EAAE,CAAA;IACrC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAA;IACvC,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,EAAE,CAAC;QAC7B,MAAM,KAAK,CAAC,cAAc,EAAE,CAAC,CAAA;QAC7B,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,MAAM,EAAE,yBAAyB,EAAE,EAAE,QAAQ,EAAE,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;QAC5F,IAAI,CAAC,IAAI,CAAC,EAAE;YAAE,SAAQ;QACtB,MAAM,IAAI,GAAG,IAAI,CAAC,IAA8D,CAAA;QAChF,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAA;QAC5B,IAAI,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,KAAK,WAAW;YAAE,SAAQ;QACxE,OAAO;YACL,EAAE,EAAE,IAAa;YACjB,MAAM,EAAE,oBAAoB,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC;YAC7C,cAAc,EAAE,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,SAAS;YAC5E,OAAO;SACR,CAAA;IACH,CAAC;IACD,OAAO,EAAE,EAAE,EAAE,KAAc,EAAE,KAAK,EAAE,2CAA2C,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE,CAAA;AAC3H,CAAC;AAED,SAAS,WAAW,CAAC,KAAa,EAAE,KAA+B;IACjE,OAAO;QACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,GAAG,KAAK,EAAE,CAAC,EAAE,CAAC;QAC/E,OAAO,EAAE,IAAI;KACd,CAAA;AACH,CAAC;AAED,SAAS,mBAAmB,CAAC,IAAa;IACxC,IAAI,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;QACxD,MAAM,CAAC,GAAI,IAAgC,CAAC,KAAK,CAAA;QACjD,OAAO,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAA;IACtD,CAAC;IACD,OAAO,SAAS,CAAA;AAClB,CAAC;AAED;;;;;;;GAOG;AACH,KAAK,UAAU,UAAU,CACvB,OAAe,EACf,IAAuC,EACvC,YAAoB,EACpB,KAA+B;IAE/B,IAAI,OAAO,CAAC,GAAG,CAAC,0BAA0B,KAAK,GAAG,EAAE,CAAC;QACnD,OAAO,WAAW,CAAC,YAAY,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,KAAK,EAAE,CAAC,CAAA;IACnE,CAAC;IAED,MAAM,EAAE,GAAG,MAAM,kBAAkB,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;IAClD,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACX,OAAO,WAAW,CAAC,YAAY,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,KAAK,EAAE,cAAc,EAAE,EAAE,CAAC,KAAK,EAAE,CAAC,CAAA;IAC7F,CAAC;IAED,MAAM,IAAI,GAA4B,EAAE,QAAQ,EAAE,UAAU,EAAE,aAAa,EAAE,YAAY,EAAE,CAAA;IAC3F,IAAI,EAAE,CAAC,cAAc,KAAK,SAAS;QAAE,IAAI,CAAC,eAAe,GAAG,EAAE,CAAC,cAAc,CAAA;IAC7E,OAAO;QACL,OAAO,EAAE,CAAC;gBACR,IAAI,EAAE,MAAe;gBACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,IAAI,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,EAAE;oBAC7E,wEAAwE;oBACxE,mEAAmE;oBACnE,KAAK,EAAE;wBACL,GAAG,IAAI;wBACP,IAAI,EAAE,wNAAwN;qBAC/N;iBACF,CAAC;aACH,CAAC;KACH,CAAA;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,4BAA4B,CAAC,KAA8B;IAC/E,MAAM,OAAO,GAAG,KAAK,CAAC,QAAkB,CAAA;IACxC,MAAM,IAAI,GAAI,KAAK,CAAC,IAAsD,IAAI,MAAM,CAAA;IACpF,MAAM,SAAS,GACb,IAAI,KAAK,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,UAAU,CAAC,CAAA;IAEnG,6DAA6D;IAC7D,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,MAAM,EAAE,oBAAoB,EAAE;QAC5D,QAAQ,EAAE,OAAO;QACjB,UAAU,EAAE,SAAS;KACtB,CAAC,CAAA;IACF,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,CAAC;QAClB,OAAO,UAAU,CACf,OAAO,EAAE,IAAI,EACb,mBAAmB,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,sCAAsC,SAAS,CAAC,MAAM,GAAG,CACjG,CAAA;IACH,CAAC;IACD,MAAM,UAAU,GAAG,SAAS,CAAC,IAAkD,CAAA;IAC/E,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,CAAA;IACjC,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,UAAU,CAAC,OAAO,EAAE,IAAI,EAAE,yCAAyC,CAAC,CAAA;IAC7E,CAAC;IACD,8EAA8E;IAC9E,2DAA2D;IAC3D,MAAM,gBAAgB,GAAG,UAAU,CAAC,WAAW,CAAA;IAC/C,wEAAwE;IACxE,MAAM,cAAc,GAAG,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,0BAA0B,CAAC,CAAC,CAAA;IAC5E,MAAM,gBAAgB,GAAG,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,4BAA4B,CAAC,CAAC,CAAA;IAEhF,+EAA+E;IAC/E,qEAAqE;IACrE,2EAA2E;IAC3E,gFAAgF;IAChF,6EAA6E;IAC7E,MAAM,cAAc,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,yBAAyB,IAAI,MAAM,EAAE,EAAE,CAAC,CAAA;IACpF,MAAM,SAAS,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,4BAA4B,IAAI,QAAQ,EAAE,EAAE,CAAC,CAAA;IACpF,MAAM,aAAa,GAAG,CAAC,CAAA;IACvB,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAA;IAEvC,IAAI,UAA8B,CAAA;IAClC,IAAI,qBAAqB,GAAG,CAAC,CAAA;IAE7B,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,EAAE,CAAC;QAC7B,MAAM,KAAK,CAAC,cAAc,CAAC,CAAA;QAC3B,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,KAAK,EAAE,kBAAkB,MAAM,SAAS,CAAC,CAAA;QACzE,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,CAAC;YAClB,qBAAqB,EAAE,CAAA;YACvB,IAAI,qBAAqB,IAAI,aAAa,EAAE,CAAC;gBAC3C,OAAO,WAAW,CAAC,iDAAiD,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAA;YAC9F,CAAC;YACD,SAAQ;QACV,CAAC;QACD,qBAAqB,GAAG,CAAC,CAAA;QACzB,MAAM,UAAU,GAAG,SAAS,CAAC,IAA8E,CAAA;QAC3G,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,CAAA;QAChC,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,sEAAsE;YACtE,2EAA2E;YAC3E,0EAA0E;YAC1E,kCAAkC;YAClC,IAAI,UAAU,CAAC,iBAAiB,EAAE,CAAC;gBACjC,OAAO,UAAU,CACf,OAAO,EAAE,IAAI,EACb,+GAA+G,EAC/G,EAAE,OAAO,EAAE,MAAM,EAAE,CACpB,CAAA;YACH,CAAC;YACD,UAAU,GAAG,UAAU,CAAC,WAAW,CAAA;YACnC,MAAK;QACP,CAAC;QACD,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,yEAAyE;YACzE,yEAAyE;YACzE,OAAO,UAAU,CACf,OAAO,EAAE,IAAI,EACb,gJAAgJ,EAChJ,EAAE,OAAO,EAAE,MAAM,EAAE,CACpB,CAAA;QACH,CAAC;IACH,CAAC;IAED,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,0EAA0E;QAC1E,4EAA4E;QAC5E,4EAA4E;QAC5E,8DAA8D;QAC9D,OAAO,WAAW,CAChB,gDAAgD,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,qFAAqF,MAAM,0FAA0F,gBAAgB,IAAI,eAAe,4EAA4E,EAChX,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,EAAE,SAAS,EAAE,CACzF,CAAA;IACH,CAAC;IAED,gFAAgF;IAChF,+EAA+E;IAC/E,6EAA6E;IAC7E,+EAA+E;IAC/E,iEAAiE;IACjE,MAAM,WAAW,GAA2B,EAAE,SAAS,EAAE,MAAM,CAAC,kBAAkB,CAAC,EAAE,CAAA;IACrF,IAAI,IAAI,KAAK,UAAU;QAAE,WAAW,CAAC,eAAe,GAAG,YAAY,CAAA;SAC9D,IAAI,IAAI,KAAK,WAAW;QAAE,WAAW,CAAC,eAAe,GAAG,UAAU,CAAA;IAEvE,MAAM,MAAM,GAAc,EAAE,CAAA;IAC5B,IAAI,aAAiC,CAAA;IACrC,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,IAAI,iBAAiB,EAAE,IAAI,EAAE,EAAE,CAAC;QACrD,MAAM,WAAW,GAAG,MAAM,OAAO,CAC/B,KAAK,EACL,sBAAsB,UAAU,WAAW,EAC3C,SAAS,EACT,EAAE,GAAG,WAAW,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,CACvC,CAAA;QACD,IAAI,CAAC,WAAW,CAAC,EAAE,EAAE,CAAC;YACpB,OAAO,WAAW,CAAC,mBAAmB,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,kCAAkC,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,CAAC,CAAA;QAClK,CAAC;QACD,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,gBAAgB,CAAC,WAAW,CAAC,IAAI,CAAC,CAAA;QAClE,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAA;QACrC,aAAa,GAAG,KAAK,CAAA;QACrB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,SAAS,IAAI,IAAI,IAAI,KAAK,CAAC;YAAE,MAAK;IACzE,CAAC;IAED,MAAM,IAAI,GAA4B,EAAE,CAAA;IACxC,IAAI,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAC;QAAE,IAAI,CAAC,eAAe,GAAG,cAAc,CAAA;IAC1E,IAAI,MAAM,CAAC,QAAQ,CAAC,gBAAgB,CAAC;QAAE,IAAI,CAAC,iBAAiB,GAAG,gBAAgB,CAAA;IAChF,IAAI,aAAa,KAAK,SAAS,IAAI,aAAa,GAAG,MAAM,CAAC,MAAM;QAAE,IAAI,CAAC,cAAc,GAAG,aAAa,CAAA;IAErG,OAAO;QACL,OAAO,EAAE,CAAC;gBACR,IAAI,EAAE,MAAe;gBACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,IAAI,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE;oBAC/D,KAAK,EAAE,IAAI;iBACZ,CAAC;aACH,CAAC;KACH,CAAA;AACH,CAAC"}
|
package/package.json
CHANGED
|
@@ -6,10 +6,13 @@
|
|
|
6
6
|
* configured and testable while public routing and tool schemas omit them.
|
|
7
7
|
*/
|
|
8
8
|
const DISABLED_PROVIDERS = new Set([
|
|
9
|
+
'companyenrich',
|
|
9
10
|
'people-data-labs',
|
|
10
11
|
])
|
|
11
12
|
|
|
12
13
|
const PROVIDER_ALIASES: Record<string, string> = {
|
|
14
|
+
'company-enrich': 'companyenrich',
|
|
15
|
+
'companyenrich-props': 'companyenrich',
|
|
13
16
|
pdl: 'people-data-labs',
|
|
14
17
|
'pdl-person-enrich': 'people-data-labs',
|
|
15
18
|
'pdl-person-identify': 'people-data-labs',
|
|
@@ -6,7 +6,8 @@ export const extractPostEngagementName = 'extract_post_engagement'
|
|
|
6
6
|
export const extractPostEngagementDescription =
|
|
7
7
|
'Extract the people who engaged with a LinkedIn post — commenters and/or reactors — as a deduplicated list of contacts (name, profile URL, headline). ' +
|
|
8
8
|
'Use this for social-signal prospecting: pull everyone who engaged with a viral post, then chain the results into enrich_person / find_email to get roles and work emails. ' +
|
|
9
|
-
'Runs an async extraction job (typically
|
|
9
|
+
'Runs an async extraction job (typically 1–3 minutes; longer for posts with 1000+ interactions) and returns every extracted person once ready. ' +
|
|
10
|
+
'If the primary source fails or its account is out of credits, automatically retries on a second provider and says so in `_meta.provider` — those records carry name/profile URL/headline only, so enrich them before use. Pricing is returned in the response credit headers.'
|
|
10
11
|
|
|
11
12
|
export const extractPostEngagementSchema = {
|
|
12
13
|
post_url: z
|
|
@@ -17,16 +18,133 @@ export const extractPostEngagementSchema = {
|
|
|
17
18
|
.enum(['comments', 'reactions', 'both'])
|
|
18
19
|
.default('both')
|
|
19
20
|
.describe('Which engagement to extract: "comments" (people who commented), "reactions" (people who reacted), or "both" (default — deduplicated across both).'),
|
|
20
|
-
include_replies:
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
21
|
+
// `include_replies` was removed: this tool returns a DEDUPLICATED contact list,
|
|
22
|
+
// where a person is one entry regardless of whether they commented or replied,
|
|
23
|
+
// so the flag never changed the result. Use GET /v1/jungler/workbooks/{id}/comments
|
|
24
|
+
// to work with individual comments and their reply metadata.
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
// One page covers most posts; the loop below still walks the rest when it doesn't.
|
|
28
|
+
const CONTACTS_PAGE_SIZE = 500
|
|
29
|
+
// Backstop so a mis-paginating upstream can't spin this tool forever.
|
|
30
|
+
const MAX_CONTACT_PAGES = 20
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* True unless the contact demonstrably engaged with nothing.
|
|
34
|
+
*
|
|
35
|
+
* The workbook always collects the post itself, so the post's AUTHOR comes back
|
|
36
|
+
* as a contact with `stats: {comments: 0, reactions: 0}` even when nobody
|
|
37
|
+
* engaged. This tool promises the people who engaged, so a zero/zero contact is
|
|
38
|
+
* not one of them. Anything without usable stats is kept — never drop a contact
|
|
39
|
+
* on ambiguity.
|
|
40
|
+
*/
|
|
41
|
+
function engaged(contact: unknown): boolean {
|
|
42
|
+
if (!contact || typeof contact !== 'object') return true
|
|
43
|
+
const stats = (contact as { stats?: unknown }).stats
|
|
44
|
+
if (!stats || typeof stats !== 'object') return true
|
|
45
|
+
const { comments, reactions } = stats as { comments?: unknown; reactions?: unknown }
|
|
46
|
+
if (typeof comments !== 'number' || typeof reactions !== 'number') return true
|
|
47
|
+
return comments > 0 || reactions > 0
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/** Read one page of the contacts envelope, tolerating a bare-array response. */
|
|
51
|
+
function readContactsPage(data: unknown): { items: unknown[]; total?: number; pages?: number } {
|
|
52
|
+
if (Array.isArray(data)) return { items: data, pages: 1 }
|
|
53
|
+
if (data && typeof data === 'object') {
|
|
54
|
+
const obj = data as { items?: unknown; total?: unknown; pages?: unknown }
|
|
55
|
+
if (Array.isArray(obj.items)) {
|
|
56
|
+
return {
|
|
57
|
+
items: obj.items,
|
|
58
|
+
total: typeof obj.total === 'number' ? obj.total : undefined,
|
|
59
|
+
pages: typeof obj.pages === 'number' ? obj.pages : undefined,
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
return { items: [], pages: 1 }
|
|
24
64
|
}
|
|
25
65
|
|
|
26
66
|
function sleep(ms: number): Promise<void> {
|
|
27
67
|
return new Promise((resolve) => setTimeout(resolve, ms))
|
|
28
68
|
}
|
|
29
69
|
|
|
70
|
+
// ── Fallback provider ───────────────────────────────────────────────────────
|
|
71
|
+
//
|
|
72
|
+
// Jungler is the primary: it charges a flat rate per post regardless of size and
|
|
73
|
+
// returns enriched contacts (work email, company domain, title, seniority). Lima
|
|
74
|
+
// Data costs the same per post but returns profile URLs only, so it needs a
|
|
75
|
+
// separate enrichment pass — hence second, not first.
|
|
76
|
+
//
|
|
77
|
+
// We reach for it ONLY when Jungler failed for a reason that is ours, not the
|
|
78
|
+
// post's: a failed run, or an upstream account out of credits. A genuinely empty
|
|
79
|
+
// post must NOT fall through — Lima would return empty too, for another charge.
|
|
80
|
+
// Same env knobs as the primary poll, so a caller (or a test) can tighten both.
|
|
81
|
+
const fallbackPollMs = () => parseInt(process.env.COLDIQ_ENGAGEMENT_POLL_MS ?? '5000', 10)
|
|
82
|
+
const fallbackTimeoutMs = () => parseInt(process.env.COLDIQ_ENGAGEMENT_FALLBACK_TIMEOUT_MS ?? '240000', 10)
|
|
83
|
+
|
|
84
|
+
/** Flatten Lima's `{commenters:{comments:[{actor}]}, reactors:{reactions:[{actor}]}}` into contacts. */
|
|
85
|
+
function limaEngagersToPeople(data: unknown, type: 'comments' | 'reactions' | 'both'): unknown[] {
|
|
86
|
+
if (!data || typeof data !== 'object') return []
|
|
87
|
+
const d = data as {
|
|
88
|
+
commenters?: { comments?: unknown[] }
|
|
89
|
+
reactors?: { reactions?: unknown[] }
|
|
90
|
+
}
|
|
91
|
+
const rows: Array<{ actor?: unknown; engagement: string }> = []
|
|
92
|
+
if (type !== 'reactions') {
|
|
93
|
+
for (const c of d.commenters?.comments ?? []) rows.push({ ...(c as object), engagement: 'COMMENT' } as never)
|
|
94
|
+
}
|
|
95
|
+
if (type !== 'comments') {
|
|
96
|
+
for (const r of d.reactors?.reactions ?? []) rows.push({ ...(r as object), engagement: 'REACTION' } as never)
|
|
97
|
+
}
|
|
98
|
+
// Deduplicate by profile URL: someone who both commented and reacted is one person.
|
|
99
|
+
const byProfile = new Map<string, Record<string, unknown>>()
|
|
100
|
+
for (const row of rows) {
|
|
101
|
+
const actor = (row.actor ?? {}) as Record<string, unknown>
|
|
102
|
+
const key = String(actor.profile_url ?? actor.name ?? Math.random())
|
|
103
|
+
const existing = byProfile.get(key)
|
|
104
|
+
if (existing) {
|
|
105
|
+
const seen = existing.engagement_types as string[]
|
|
106
|
+
if (!seen.includes(row.engagement)) seen.push(row.engagement)
|
|
107
|
+
continue
|
|
108
|
+
}
|
|
109
|
+
byProfile.set(key, {
|
|
110
|
+
name: actor.name ?? null,
|
|
111
|
+
profile_url: actor.profile_url ?? null,
|
|
112
|
+
description: actor.headline ?? null,
|
|
113
|
+
profile_image_url: actor.image_url ?? null,
|
|
114
|
+
engagement_types: [row.engagement],
|
|
115
|
+
})
|
|
116
|
+
}
|
|
117
|
+
return [...byProfile.values()]
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
async function extractViaLimaData(postUrl: string, type: 'comments' | 'reactions' | 'both') {
|
|
121
|
+
const submit = await callApi('POST', '/limadata/batch/post-engagements', { urls: [postUrl] })
|
|
122
|
+
if (!submit.ok) {
|
|
123
|
+
return { ok: false as const, error: extractErrorMessage(submit.data) ?? `Fallback provider failed to start (status ${submit.status})` }
|
|
124
|
+
}
|
|
125
|
+
const batchId = (submit.data as { batch_id?: number | string }).batch_id
|
|
126
|
+
if (batchId === undefined) return { ok: false as const, error: 'Fallback provider did not return a batch id' }
|
|
127
|
+
|
|
128
|
+
const creditsCharged = Number(submit.headers['x-coldiq-credits-charged'])
|
|
129
|
+
const timeoutMs = fallbackTimeoutMs()
|
|
130
|
+
const deadline = Date.now() + timeoutMs
|
|
131
|
+
while (Date.now() < deadline) {
|
|
132
|
+
await sleep(fallbackPollMs())
|
|
133
|
+
const poll = await callApi('POST', '/limadata/batch/results', { batch_id: String(batchId) })
|
|
134
|
+
if (!poll.ok) continue
|
|
135
|
+
const body = poll.data as { items?: Array<{ status?: string; data?: unknown }> }
|
|
136
|
+
const item = body.items?.[0]
|
|
137
|
+
if (!item || String(item.status).toLowerCase() !== 'completed') continue
|
|
138
|
+
return {
|
|
139
|
+
ok: true as const,
|
|
140
|
+
people: limaEngagersToPeople(item.data, type),
|
|
141
|
+
creditsCharged: Number.isFinite(creditsCharged) ? creditsCharged : undefined,
|
|
142
|
+
batchId,
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
return { ok: false as const, error: `Fallback provider did not finish within ${Math.round(timeoutMs / 1000)}s`, batchId }
|
|
146
|
+
}
|
|
147
|
+
|
|
30
148
|
function errorResult(error: string, extra?: Record<string, unknown>) {
|
|
31
149
|
return {
|
|
32
150
|
content: [{ type: 'text' as const, text: JSON.stringify({ error, ...extra }) }],
|
|
@@ -42,11 +160,50 @@ function extractErrorMessage(data: unknown): string | undefined {
|
|
|
42
160
|
return undefined
|
|
43
161
|
}
|
|
44
162
|
|
|
163
|
+
/**
|
|
164
|
+
* Try the fallback provider after the primary failed; if it also fails, report
|
|
165
|
+
* the ORIGINAL failure (that is the one the caller can act on) with the
|
|
166
|
+
* fallback's outcome attached.
|
|
167
|
+
*
|
|
168
|
+
* Set COLDIQ_ENGAGEMENT_FALLBACK=0 to disable — the fallback is a second billable
|
|
169
|
+
* provider, so anyone running to a strict budget must be able to opt out.
|
|
170
|
+
*/
|
|
171
|
+
async function fallbackOr(
|
|
172
|
+
postUrl: string,
|
|
173
|
+
type: 'comments' | 'reactions' | 'both',
|
|
174
|
+
primaryError: string,
|
|
175
|
+
extra?: Record<string, unknown>,
|
|
176
|
+
) {
|
|
177
|
+
if (process.env.COLDIQ_ENGAGEMENT_FALLBACK === '0') {
|
|
178
|
+
return errorResult(primaryError, { post_url: postUrl, ...extra })
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
const fb = await extractViaLimaData(postUrl, type)
|
|
182
|
+
if (!fb.ok) {
|
|
183
|
+
return errorResult(primaryError, { post_url: postUrl, ...extra, fallback_error: fb.error })
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
const meta: Record<string, unknown> = { provider: 'limadata', primary_error: primaryError }
|
|
187
|
+
if (fb.creditsCharged !== undefined) meta.credits_charged = fb.creditsCharged
|
|
188
|
+
return {
|
|
189
|
+
content: [{
|
|
190
|
+
type: 'text' as const,
|
|
191
|
+
text: JSON.stringify({
|
|
192
|
+
data: { post_url: postUrl, type, people: fb.people, count: fb.people.length },
|
|
193
|
+
// Say plainly that these records are thinner than the primary's, so the
|
|
194
|
+
// caller knows to run enrichment before treating them as contacts.
|
|
195
|
+
_meta: {
|
|
196
|
+
...meta,
|
|
197
|
+
note: 'Served by the fallback provider after the primary failed. These records carry name, profile URL and headline only — no work email or company data. Run enrich_person / find_email on them if you need contact details.',
|
|
198
|
+
},
|
|
199
|
+
}),
|
|
200
|
+
}],
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
|
|
45
204
|
export async function extractPostEngagementHandler(input: Record<string, unknown>) {
|
|
46
205
|
const postUrl = input.post_url as string
|
|
47
206
|
const type = (input.type as 'comments' | 'reactions' | 'both' | undefined) ?? 'both'
|
|
48
|
-
const includeReplies = input.include_replies as boolean | undefined
|
|
49
|
-
|
|
50
207
|
const dataTypes =
|
|
51
208
|
type === 'comments' ? ['comment'] : type === 'reactions' ? ['reaction'] : ['comment', 'reaction']
|
|
52
209
|
|
|
@@ -56,21 +213,30 @@ export async function extractPostEngagementHandler(input: Record<string, unknown
|
|
|
56
213
|
data_types: dataTypes,
|
|
57
214
|
})
|
|
58
215
|
if (!createRes.ok) {
|
|
59
|
-
return
|
|
216
|
+
return fallbackOr(
|
|
217
|
+
postUrl, type,
|
|
218
|
+
extractErrorMessage(createRes.data) ?? `Failed to start extraction (status ${createRes.status})`,
|
|
219
|
+
)
|
|
60
220
|
}
|
|
61
|
-
const createData = createRes.data as { task_id?: string }
|
|
221
|
+
const createData = createRes.data as { task_id?: string; workbook_id?: string }
|
|
62
222
|
const taskId = createData.task_id
|
|
63
223
|
if (!taskId) {
|
|
64
|
-
return
|
|
224
|
+
return fallbackOr(postUrl, type, 'Extraction job did not return a task id')
|
|
65
225
|
}
|
|
226
|
+
// Kept so a timeout can still hand the caller a way to collect what they paid
|
|
227
|
+
// for — the workbook stays readable upstream for 12 hours.
|
|
228
|
+
const createWorkbookId = createData.workbook_id
|
|
66
229
|
// Credit headers are emitted on the create call (the only billed step).
|
|
67
230
|
const creditsCharged = Number(createRes.headers['x-coldiq-credits-charged'])
|
|
68
231
|
const creditsRemaining = Number(createRes.headers['x-coldiq-credits-remaining'])
|
|
69
232
|
|
|
70
233
|
// Step 2 — poll task status until it resolves. The status endpoint is free, so
|
|
71
234
|
// polling does not bill; only the create call above charged credits.
|
|
72
|
-
|
|
73
|
-
|
|
235
|
+
// Upstream quotes 1-3 minutes and explicitly warns that large posts (1000+
|
|
236
|
+
// interactions) take longer — which is exactly the viral post this tool is for.
|
|
237
|
+
// A 180s deadline cut those off mid-extraction and reported a bogus timeout.
|
|
238
|
+
const pollIntervalMs = parseInt(process.env.COLDIQ_ENGAGEMENT_POLL_MS ?? '5000', 10)
|
|
239
|
+
const timeoutMs = parseInt(process.env.COLDIQ_ENGAGEMENT_TIMEOUT_MS ?? '600000', 10)
|
|
74
240
|
const maxPollErrors = 3
|
|
75
241
|
const deadline = Date.now() + timeoutMs
|
|
76
242
|
|
|
@@ -88,46 +254,82 @@ export async function extractPostEngagementHandler(input: Record<string, unknown
|
|
|
88
254
|
continue
|
|
89
255
|
}
|
|
90
256
|
consecutivePollErrors = 0
|
|
91
|
-
const
|
|
257
|
+
const statusData = statusRes.data as { status?: string; workbook_id?: string; credits_exhausted?: boolean }
|
|
258
|
+
const status = statusData.status
|
|
92
259
|
if (status === 'success') {
|
|
93
|
-
|
|
260
|
+
// A credits-exhausted run is terminal but collected little or nothing
|
|
261
|
+
// because OUR provider account ran dry — the caller's request is sound, so
|
|
262
|
+
// route it to the fallback rather than handing back a hollow result. This
|
|
263
|
+
// charge is refunded server-side.
|
|
264
|
+
if (statusData.credits_exhausted) {
|
|
265
|
+
return fallbackOr(
|
|
266
|
+
postUrl, type,
|
|
267
|
+
'Engagement extraction stopped early — the data provider account is out of credits (this request was refunded)',
|
|
268
|
+
{ task_id: taskId },
|
|
269
|
+
)
|
|
270
|
+
}
|
|
271
|
+
workbookId = statusData.workbook_id
|
|
94
272
|
break
|
|
95
273
|
}
|
|
96
274
|
if (status === 'failure') {
|
|
97
|
-
|
|
275
|
+
// The status route refunds the create charge when a run reaches failure,
|
|
276
|
+
// so the user is made whole — say so rather than leaving them to wonder.
|
|
277
|
+
return fallbackOr(
|
|
278
|
+
postUrl, type,
|
|
279
|
+
'Engagement extraction failed upstream — the post may be private, deleted, or have no engagement. The credits for this task have been refunded.',
|
|
280
|
+
{ task_id: taskId },
|
|
281
|
+
)
|
|
98
282
|
}
|
|
99
283
|
}
|
|
100
284
|
|
|
101
285
|
if (!workbookId) {
|
|
102
|
-
|
|
286
|
+
// Giving up here does NOT cancel the run, and the create call was already
|
|
287
|
+
// billed. Never drop that on the floor: hand back the ids so the caller can
|
|
288
|
+
// collect the results (upstream keeps the workbook for 12 hours) instead of
|
|
289
|
+
// paying again for a re-run that is probably still in flight.
|
|
290
|
+
return errorResult(
|
|
291
|
+
`Engagement extraction is still running after ${Math.round(timeoutMs / 1000)}s. It has NOT been cancelled and no re-run is needed — poll GET /v1/jungler/tasks/${taskId}/status until it reports success, then fetch the people from GET /v1/jungler/workbooks/${createWorkbookId ?? '{workbook_id}'}/contacts. Results stay available for 12 hours after the task was created.`,
|
|
292
|
+
{ post_url: postUrl, task_id: taskId, workbook_id: createWorkbookId, status: 'running' },
|
|
293
|
+
)
|
|
103
294
|
}
|
|
104
295
|
|
|
105
296
|
// Step 3 — fetch the deduplicated people. activity_filter narrows to commenters
|
|
106
297
|
// or reactors; omitted for "both" so the upstream returns all unique contacts.
|
|
107
|
-
|
|
298
|
+
// Contacts are paginated, and a viral post — the case this tool exists for —
|
|
299
|
+
// routinely exceeds one page, so walk every page rather than truncating to the
|
|
300
|
+
// first. Reads are free, so pagination costs the caller nothing.
|
|
301
|
+
const queryParams: Record<string, string> = { page_size: String(CONTACTS_PAGE_SIZE) }
|
|
108
302
|
if (type === 'comments') queryParams.activity_filter = 'commenters'
|
|
109
303
|
else if (type === 'reactions') queryParams.activity_filter = 'reactors'
|
|
110
|
-
if (includeReplies !== undefined) queryParams.include_replies = String(includeReplies)
|
|
111
304
|
|
|
112
|
-
const
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
305
|
+
const people: unknown[] = []
|
|
306
|
+
let totalContacts: number | undefined
|
|
307
|
+
for (let page = 1; page <= MAX_CONTACT_PAGES; page++) {
|
|
308
|
+
const contactsRes = await callApi(
|
|
309
|
+
'GET',
|
|
310
|
+
`/jungler/workbooks/${workbookId}/contacts`,
|
|
311
|
+
undefined,
|
|
312
|
+
{ ...queryParams, page: String(page) },
|
|
313
|
+
)
|
|
314
|
+
if (!contactsRes.ok) {
|
|
315
|
+
return errorResult(extractErrorMessage(contactsRes.data) ?? 'Failed to fetch extracted people', { post_url: postUrl, task_id: taskId, workbook_id: workbookId })
|
|
316
|
+
}
|
|
317
|
+
const { items, total, pages } = readContactsPage(contactsRes.data)
|
|
318
|
+
people.push(...items.filter(engaged))
|
|
319
|
+
totalContacts = total
|
|
320
|
+
if (items.length === 0 || (pages !== undefined && page >= pages)) break
|
|
120
321
|
}
|
|
121
322
|
|
|
122
323
|
const meta: Record<string, unknown> = {}
|
|
123
324
|
if (Number.isFinite(creditsCharged)) meta.credits_charged = creditsCharged
|
|
124
325
|
if (Number.isFinite(creditsRemaining)) meta.credits_remaining = creditsRemaining
|
|
326
|
+
if (totalContacts !== undefined && totalContacts > people.length) meta.truncated_from = totalContacts
|
|
125
327
|
|
|
126
328
|
return {
|
|
127
329
|
content: [{
|
|
128
330
|
type: 'text' as const,
|
|
129
331
|
text: JSON.stringify({
|
|
130
|
-
data: { post_url: postUrl, type, people:
|
|
332
|
+
data: { post_url: postUrl, type, people, count: people.length },
|
|
131
333
|
_meta: meta,
|
|
132
334
|
}),
|
|
133
335
|
}],
|
|
@@ -34,8 +34,18 @@ const CAPABILITIES: Capability[] = [
|
|
|
34
34
|
]
|
|
35
35
|
|
|
36
36
|
describe('standalone MCP provider availability', () => {
|
|
37
|
-
it('mirrors every
|
|
38
|
-
for (const alias of [
|
|
37
|
+
it('mirrors every disabled-provider alias from the Worker policy', () => {
|
|
38
|
+
for (const alias of [
|
|
39
|
+
'companyenrich',
|
|
40
|
+
'CompanyEnrich',
|
|
41
|
+
'Company Enrich',
|
|
42
|
+
'companyenrich_props',
|
|
43
|
+
'people-data-labs',
|
|
44
|
+
'People Data Labs',
|
|
45
|
+
'pdl',
|
|
46
|
+
'pdl-person-enrich',
|
|
47
|
+
'pdl-person-identify',
|
|
48
|
+
]) {
|
|
39
49
|
expect(isProviderEnabled(alias), alias).toBe(false)
|
|
40
50
|
expect(isProviderEnabled(alias), alias).toBe(isWorkerProviderEnabled(alias))
|
|
41
51
|
}
|
|
@@ -46,6 +56,8 @@ describe('standalone MCP provider availability', () => {
|
|
|
46
56
|
// pdl-person-enrich left find_email entirely (2026-07-24 reorder); the dormant
|
|
47
57
|
// PDL definition still lives in find_people for when the account is re-enabled.
|
|
48
58
|
expect(getConfiguredProviders('find_people').some((provider) => provider.id === 'pdl')).toBe(true)
|
|
59
|
+
expect(getConfiguredProviders('search_companies').some((provider) => provider.id === 'companyenrich')).toBe(true)
|
|
60
|
+
expect(getConfiguredProviders('enrich_company').some((provider) => provider.id === 'companyenrich_props')).toBe(true)
|
|
49
61
|
|
|
50
62
|
for (const capability of CAPABILITIES) {
|
|
51
63
|
expect(getProviders(capability).every((provider) => isProviderEnabled(provider.id))).toBe(true)
|
|
@@ -55,8 +67,13 @@ describe('standalone MCP provider availability', () => {
|
|
|
55
67
|
})
|
|
56
68
|
|
|
57
69
|
it('rejects an explicit disabled-provider pin without advertising it as available', () => {
|
|
58
|
-
for (const capability
|
|
59
|
-
|
|
70
|
+
for (const [capability, provider] of [
|
|
71
|
+
['find_email', 'pdl-person-enrich'],
|
|
72
|
+
['find_emails', 'pdl-person-enrich'],
|
|
73
|
+
['search_companies', 'companyenrich'],
|
|
74
|
+
['enrich_company', 'companyenrich_props'],
|
|
75
|
+
] as const) {
|
|
76
|
+
const result = resolvePreferredProviders(capability, {}, [provider])
|
|
60
77
|
expect(result.ok).toBe(false)
|
|
61
78
|
if (!result.ok) {
|
|
62
79
|
expect(result.error.available_providers.every(isProviderEnabled)).toBe(true)
|
|
@@ -79,5 +96,7 @@ describe('standalone MCP provider availability', () => {
|
|
|
79
96
|
|
|
80
97
|
expect(publicMetadata).not.toContain('people data labs')
|
|
81
98
|
expect(publicMetadata).not.toMatch(/\bpdl\b/)
|
|
99
|
+
expect(publicMetadata).not.toContain('companyenrich')
|
|
100
|
+
expect(publicMetadata).not.toContain('/companyenrich')
|
|
82
101
|
})
|
|
83
102
|
})
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { describe, it, expect } from 'vitest'
|
|
2
|
-
import { getProviders, prospeoHeadcountBuckets } from '../src/registry.js'
|
|
2
|
+
import { getConfiguredProviders, getProviders, prospeoHeadcountBuckets } from '../src/registry.js'
|
|
3
3
|
|
|
4
4
|
const providers = () => getProviders('search_companies')
|
|
5
5
|
const get = (id: string) => {
|
|
6
|
-
const p =
|
|
6
|
+
const p = getConfiguredProviders('search_companies').find((p) => p.id === id)
|
|
7
7
|
if (!p) throw new Error(`Provider '${id}' not found in search_companies`)
|
|
8
8
|
return p
|
|
9
9
|
}
|
package/tests/registry.test.ts
CHANGED
|
@@ -59,7 +59,7 @@ describe('registry', () => {
|
|
|
59
59
|
// `query` matches company NAME + domain, so a theme like "SaaS" returns
|
|
60
60
|
// companies merely NAMED "SaaS …". "SaaS" is a controlled category → routes to
|
|
61
61
|
// the precise `category` filter; "Software" has no category → stays free-text.
|
|
62
|
-
const providers =
|
|
62
|
+
const providers = getConfiguredProviders('search_companies')
|
|
63
63
|
const ce = providers.find((p) => p.id === 'companyenrich')!
|
|
64
64
|
const result = ce.mapParams({
|
|
65
65
|
keywords: ['SaaS'],
|
|
@@ -81,7 +81,7 @@ describe('registry', () => {
|
|
|
81
81
|
})
|
|
82
82
|
|
|
83
83
|
it('CompanyEnrich routes "B2B SaaS" to category=[saas,b2b] with And (the user-intended conjunction)', () => {
|
|
84
|
-
const providers =
|
|
84
|
+
const providers = getConfiguredProviders('search_companies')
|
|
85
85
|
const ce = providers.find((p) => p.id === 'companyenrich')!
|
|
86
86
|
const body = ce.mapParams({ keywords: ['SaaS', 'B2B'], countries: ['FR'] }).body as Record<string, unknown>
|
|
87
87
|
expect(body.category).toEqual(['saas', 'b2b'])
|
|
@@ -94,7 +94,7 @@ describe('registry', () => {
|
|
|
94
94
|
// The unified `keywords` param means topics/business-models/themes — brand-name
|
|
95
95
|
// lookups belong to enrich_company or the `names`/`domains` fields, not here.
|
|
96
96
|
// "fintech" is not a controlled category, so it stays in the keyword tag filter.
|
|
97
|
-
const providers =
|
|
97
|
+
const providers = getConfiguredProviders('search_companies')
|
|
98
98
|
const ce = providers.find((p) => p.id === 'companyenrich')!
|
|
99
99
|
const result = ce.mapParams({ keywords: ['fintech'], limit: 5 })
|
|
100
100
|
const body = result.body as Record<string, unknown>
|
|
@@ -106,7 +106,7 @@ describe('registry', () => {
|
|
|
106
106
|
})
|
|
107
107
|
|
|
108
108
|
it('CompanyEnrich omits both category and keyword filters when no keywords/industries provided', () => {
|
|
109
|
-
const providers =
|
|
109
|
+
const providers = getConfiguredProviders('search_companies')
|
|
110
110
|
const ce = providers.find((p) => p.id === 'companyenrich')!
|
|
111
111
|
const result = ce.mapParams({ countries: ['US'], limit: 5 })
|
|
112
112
|
const body = result.body as Record<string, unknown>
|
|
@@ -469,7 +469,6 @@ describe('registry', () => {
|
|
|
469
469
|
it('search_companies omits disabled providers while preserving priority order', () => {
|
|
470
470
|
const providers = getProviders('search_companies')
|
|
471
471
|
expect(providers.map((p) => p.id)).toEqual([
|
|
472
|
-
'companyenrich',
|
|
473
472
|
'fullenrich',
|
|
474
473
|
'theirstack',
|
|
475
474
|
'signalbase',
|
|
@@ -800,14 +799,14 @@ describe('registry', () => {
|
|
|
800
799
|
it('providers preserve their order with disabled providers omitted', () => {
|
|
801
800
|
const ids = getProviders('enrich_company').map((p) => p.id)
|
|
802
801
|
expect(ids).toEqual([
|
|
803
|
-
'
|
|
804
|
-
'
|
|
802
|
+
'apollo', 'limadata', 'prospeo',
|
|
803
|
+
'blitzapi', 'wiza', 'findymail', 'icypeas',
|
|
805
804
|
'builtwith', 'openmart', 'linkupapi-by-domain', 'linkupapi-by-url', 'discolike',
|
|
806
805
|
])
|
|
807
806
|
})
|
|
808
807
|
|
|
809
808
|
it('CompanyEnrich uses GET with queryParams and is gated on domain', () => {
|
|
810
|
-
const ce =
|
|
809
|
+
const ce = getConfiguredProviders('enrich_company').find((p) => p.id === 'companyenrich')!
|
|
811
810
|
expect(ce.method).toBe('GET')
|
|
812
811
|
expect(ce.mapParams({ domain: 'stripe.com' }).queryParams).toEqual({ domain: 'stripe.com' })
|
|
813
812
|
expect(ce.isApplicable!({ domain: 'stripe.com' })).toBe(true)
|
|
@@ -881,7 +880,7 @@ describe('registry', () => {
|
|
|
881
880
|
})
|
|
882
881
|
|
|
883
882
|
it('CompanyEnrich POST maps name and linkedinUrl (camelCase)', () => {
|
|
884
|
-
const cp =
|
|
883
|
+
const cp = getConfiguredProviders('enrich_company').find((p) => p.id === 'companyenrich_props')!
|
|
885
884
|
expect(cp.method).toBe('POST')
|
|
886
885
|
expect(cp.isApplicable!({ name: 'Stripe' })).toBe(true)
|
|
887
886
|
expect(cp.isApplicable!({ linkedin_url: 'https://linkedin.com/company/stripe' })).toBe(true)
|
|
@@ -1195,10 +1194,11 @@ describe('registry', () => {
|
|
|
1195
1194
|
|
|
1196
1195
|
it('top-tier active providers have no postFilter', () => {
|
|
1197
1196
|
const providers = getProviders('search_companies')
|
|
1198
|
-
for (const id of ['
|
|
1197
|
+
for (const id of ['fullenrich', 'signalbase']) {
|
|
1199
1198
|
const p = providers.find((x) => x.id === id)!
|
|
1200
1199
|
expect(p.postFilter, `${id} should not have postFilter`).toBeUndefined()
|
|
1201
1200
|
}
|
|
1201
|
+
expect(getConfiguredProviders('search_companies').find((provider) => provider.id === 'companyenrich')?.postFilter).toBeUndefined()
|
|
1202
1202
|
expect(getConfiguredProviders('search_companies').find((provider) => provider.id === 'pdl')?.postFilter).toBeUndefined()
|
|
1203
1203
|
})
|
|
1204
1204
|
|
|
@@ -1238,7 +1238,7 @@ describe('registry', () => {
|
|
|
1238
1238
|
|
|
1239
1239
|
describe('search_companies Phase B GTM filters', () => {
|
|
1240
1240
|
it('CompanyEnrich maps technologies, fundingAmount, fundingYear, revenue, workforceGrowth, and exclude', () => {
|
|
1241
|
-
const ce =
|
|
1241
|
+
const ce = getConfiguredProviders('search_companies').find((p) => p.id === 'companyenrich')!
|
|
1242
1242
|
const result = ce.mapParams({
|
|
1243
1243
|
keywords: ['SaaS'],
|
|
1244
1244
|
technologies: ['Salesforce', 'HubSpot'],
|
|
@@ -1274,7 +1274,7 @@ describe('registry', () => {
|
|
|
1274
1274
|
})
|
|
1275
1275
|
|
|
1276
1276
|
it('CompanyEnrich omits optional Phase B fields when not provided', () => {
|
|
1277
|
-
const ce =
|
|
1277
|
+
const ce = getConfiguredProviders('search_companies').find((p) => p.id === 'companyenrich')!
|
|
1278
1278
|
const result = ce.mapParams({ countries: ['US'], limit: 5 })
|
|
1279
1279
|
const body = result.body as Record<string, unknown>
|
|
1280
1280
|
expect(body.technologies).toBeUndefined()
|
|
@@ -1286,7 +1286,7 @@ describe('registry', () => {
|
|
|
1286
1286
|
})
|
|
1287
1287
|
|
|
1288
1288
|
it('CompanyEnrich exclude object only includes keys that were provided', () => {
|
|
1289
|
-
const ce =
|
|
1289
|
+
const ce = getConfiguredProviders('search_companies').find((p) => p.id === 'companyenrich')!
|
|
1290
1290
|
const result = ce.mapParams({ exclude_domains: ['a.com'], limit: 5 })
|
|
1291
1291
|
expect((result.body as Record<string, unknown>).exclude).toEqual({ domains: ['a.com'] })
|
|
1292
1292
|
})
|
|
@@ -26,9 +26,9 @@ describe('extract_post_engagement handler', () => {
|
|
|
26
26
|
const people = [{ name: 'Alex Vacca', profile_url: 'https://www.linkedin.com/in/alexvacca' }]
|
|
27
27
|
globalThis.fetch = vi.fn(async (url: string | URL) => {
|
|
28
28
|
const u = url.toString()
|
|
29
|
-
if (u.includes('/jungler/workbooks/wb_1/contacts')) return res(people)
|
|
29
|
+
if (u.includes('/jungler/workbooks/wb_1/contacts')) return res({ items: people, total: 1, page: 1, pages: 1 })
|
|
30
30
|
if (u.includes('/jungler/tasks/task_1/status')) return res({ task_id: 'task_1', status: 'success', workbook_id: 'wb_1' })
|
|
31
|
-
if (u.includes('/jungler/workbooks')) return res({ task_id: 'task_1', status: 'PENDING' }, 200, { 'x-coldiq-credits-charged': '10', 'x-coldiq-credits-remaining': '90' })
|
|
31
|
+
if (u.includes('/jungler/workbooks')) return res({ task_id: 'task_1', status: 'PENDING', workbook_id: 'wb_1' }, 200, { 'x-coldiq-credits-charged': '10', 'x-coldiq-credits-remaining': '90' })
|
|
32
32
|
throw new Error(`unexpected url ${u}`)
|
|
33
33
|
}) as typeof fetch
|
|
34
34
|
|
|
@@ -36,15 +36,80 @@ describe('extract_post_engagement handler', () => {
|
|
|
36
36
|
const parsed = JSON.parse(result.content[0].text)
|
|
37
37
|
expect(result.isError).toBeUndefined()
|
|
38
38
|
expect(parsed.data.people).toEqual(people)
|
|
39
|
+
expect(parsed.data.count).toBe(1)
|
|
39
40
|
expect(parsed.data.type).toBe('both')
|
|
40
41
|
expect(parsed._meta.credits_charged).toBe(10)
|
|
41
42
|
})
|
|
42
43
|
|
|
44
|
+
it('walks every contacts page — a viral post exceeds one page', async () => {
|
|
45
|
+
const pageOf = (n: number) => Array.from({ length: n }, (_, i) => ({ name: `Person ${i}` }))
|
|
46
|
+
const seenPages: string[] = []
|
|
47
|
+
globalThis.fetch = vi.fn(async (url: string | URL) => {
|
|
48
|
+
const u = url.toString()
|
|
49
|
+
if (u.includes('/contacts')) {
|
|
50
|
+
const page = new URL(u).searchParams.get('page') ?? '1'
|
|
51
|
+
seenPages.push(page)
|
|
52
|
+
return res({ items: pageOf(page === '1' ? 500 : 103), total: 603, page: Number(page), pages: 2 })
|
|
53
|
+
}
|
|
54
|
+
if (u.includes('/jungler/tasks/task_1/status')) return res({ task_id: 'task_1', status: 'success', workbook_id: 'wb_1' })
|
|
55
|
+
if (u.includes('/jungler/workbooks')) return res({ task_id: 'task_1', status: 'PENDING', workbook_id: 'wb_1' })
|
|
56
|
+
throw new Error(`unexpected url ${u}`)
|
|
57
|
+
}) as typeof fetch
|
|
58
|
+
|
|
59
|
+
const result = await extractPostEngagementHandler({ post_url: POST_URL, type: 'both' })
|
|
60
|
+
const parsed = JSON.parse(result.content[0].text)
|
|
61
|
+
expect(seenPages).toEqual(['1', '2'])
|
|
62
|
+
expect(parsed.data.count).toBe(603)
|
|
63
|
+
expect(parsed._meta.truncated_from).toBeUndefined()
|
|
64
|
+
})
|
|
65
|
+
|
|
66
|
+
it('drops the post author, who is collected with zero engagement and never engaged', async () => {
|
|
67
|
+
// The workbook always collects the post itself, so its author comes back as a
|
|
68
|
+
// contact with stats 0/0. This tool promises the people who ENGAGED.
|
|
69
|
+
globalThis.fetch = vi.fn(async (url: string | URL) => {
|
|
70
|
+
const u = url.toString()
|
|
71
|
+
if (u.includes('/contacts')) return res({
|
|
72
|
+
items: [
|
|
73
|
+
{ name: 'Michel Lieben', stats: { comments: 0, reactions: 0 } },
|
|
74
|
+
{ name: 'Jane Smith', stats: { comments: 2, reactions: 1 } },
|
|
75
|
+
{ name: 'No Stats Person' },
|
|
76
|
+
],
|
|
77
|
+
total: 3, page: 1, pages: 1,
|
|
78
|
+
})
|
|
79
|
+
if (u.includes('/jungler/tasks/task_1/status')) return res({ task_id: 'task_1', status: 'success', workbook_id: 'wb_1' })
|
|
80
|
+
if (u.includes('/jungler/workbooks')) return res({ task_id: 'task_1', status: 'PENDING', workbook_id: 'wb_1' })
|
|
81
|
+
throw new Error(`unexpected url ${u}`)
|
|
82
|
+
}) as typeof fetch
|
|
83
|
+
|
|
84
|
+
const parsed = JSON.parse((await extractPostEngagementHandler({ post_url: POST_URL, type: 'both' })).content[0].text)
|
|
85
|
+
// Unknown stats are kept — never drop a contact on ambiguity.
|
|
86
|
+
expect(parsed.data.people.map((p: { name: string }) => p.name)).toEqual(['Jane Smith', 'No Stats Person'])
|
|
87
|
+
})
|
|
88
|
+
|
|
89
|
+
it('reports the task and workbook ids instead of losing the paid run when polling times out', async () => {
|
|
90
|
+
vi.stubEnv('COLDIQ_ENGAGEMENT_TIMEOUT_MS', '30')
|
|
91
|
+
globalThis.fetch = vi.fn(async (url: string | URL) => {
|
|
92
|
+
const u = url.toString()
|
|
93
|
+
// Never terminal — the caller must still be able to collect what they paid for.
|
|
94
|
+
if (u.includes('/jungler/tasks/task_1/status')) return res({ task_id: 'task_1', status: 'started' })
|
|
95
|
+
if (u.includes('/jungler/workbooks')) return res({ task_id: 'task_1', status: 'PENDING', workbook_id: 'wb_1' })
|
|
96
|
+
throw new Error(`unexpected url ${u}`)
|
|
97
|
+
}) as typeof fetch
|
|
98
|
+
|
|
99
|
+
const result = await extractPostEngagementHandler({ post_url: POST_URL, type: 'both' })
|
|
100
|
+
const parsed = JSON.parse(result.content[0].text)
|
|
101
|
+
expect(result.isError).toBe(true)
|
|
102
|
+
expect(parsed.task_id).toBe('task_1')
|
|
103
|
+
expect(parsed.workbook_id).toBe('wb_1')
|
|
104
|
+
expect(parsed.error).toMatch(/still running/i)
|
|
105
|
+
expect(parsed.error).toMatch(/NOT been cancelled/i)
|
|
106
|
+
})
|
|
107
|
+
|
|
43
108
|
it('passes activity_filter=commenters when type=comments', async () => {
|
|
44
109
|
let contactsUrl = ''
|
|
45
110
|
globalThis.fetch = vi.fn(async (url: string | URL) => {
|
|
46
111
|
const u = url.toString()
|
|
47
|
-
if (u.includes('/jungler/workbooks/wb_1/contacts')) { contactsUrl = u; return res([]) }
|
|
112
|
+
if (u.includes('/jungler/workbooks/wb_1/contacts')) { contactsUrl = u; return res({ items: [], total: 0, pages: 0 }) }
|
|
48
113
|
if (u.includes('/jungler/tasks/task_1/status')) return res({ task_id: 'task_1', status: 'success', workbook_id: 'wb_1' })
|
|
49
114
|
if (u.includes('/jungler/workbooks')) return res({ task_id: 'task_1', status: 'PENDING' })
|
|
50
115
|
throw new Error(`unexpected url ${u}`)
|
|
@@ -67,6 +132,105 @@ describe('extract_post_engagement handler', () => {
|
|
|
67
132
|
expect(JSON.parse(result.content[0].text).error).toMatch(/extraction failed/i)
|
|
68
133
|
})
|
|
69
134
|
|
|
135
|
+
// ── Fallback to the second provider ──────────────────────────────────────
|
|
136
|
+
//
|
|
137
|
+
// Only for failures that are OURS (dead run, our provider account out of
|
|
138
|
+
// credits) — never for a post that genuinely has no engagement, which would
|
|
139
|
+
// buy an empty second result.
|
|
140
|
+
const LIMA_PAYLOAD = {
|
|
141
|
+
items: [{
|
|
142
|
+
status: 'Completed',
|
|
143
|
+
data: {
|
|
144
|
+
post_url: POST_URL,
|
|
145
|
+
commenters: { comments: [{ actor: { name: 'Jane Smith', profile_url: 'https://linkedin.com/in/jane', headline: 'VP Eng' } }] },
|
|
146
|
+
reactors: {
|
|
147
|
+
reactions: [
|
|
148
|
+
{ actor: { name: 'Bob Johnson', profile_url: 'https://linkedin.com/in/bob', headline: 'Sales' } },
|
|
149
|
+
// Same person also reacted — must collapse to one contact.
|
|
150
|
+
{ actor: { name: 'Jane Smith', profile_url: 'https://linkedin.com/in/jane', headline: 'VP Eng' } },
|
|
151
|
+
],
|
|
152
|
+
},
|
|
153
|
+
},
|
|
154
|
+
}],
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
function stubExhaustedThenLima(limaOk = true) {
|
|
158
|
+
globalThis.fetch = vi.fn(async (url: string | URL) => {
|
|
159
|
+
const u = url.toString()
|
|
160
|
+
if (u.includes('/limadata/batch/post-engagements')) {
|
|
161
|
+
return limaOk ? res({ batch_id: 99 }, 200, { 'x-coldiq-credits-charged': '35' }) : res({ error: 'lima down' }, 502)
|
|
162
|
+
}
|
|
163
|
+
if (u.includes('/limadata/batch/results')) return res(LIMA_PAYLOAD)
|
|
164
|
+
if (u.includes('/jungler/tasks/task_1/status')) return res({ task_id: 'task_1', status: 'success', credits_exhausted: true })
|
|
165
|
+
if (u.includes('/jungler/workbooks')) return res({ task_id: 'task_1', status: 'PENDING', workbook_id: 'wb_1' })
|
|
166
|
+
throw new Error(`unexpected url ${u}`)
|
|
167
|
+
}) as typeof fetch
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
it('falls back to the second provider when our provider account is out of credits', async () => {
|
|
171
|
+
stubExhaustedThenLima()
|
|
172
|
+
const result = await extractPostEngagementHandler({ post_url: POST_URL, type: 'both' })
|
|
173
|
+
const parsed = JSON.parse(result.content[0].text)
|
|
174
|
+
|
|
175
|
+
expect(result.isError).toBeUndefined()
|
|
176
|
+
expect(parsed._meta.provider).toBe('limadata')
|
|
177
|
+
expect(parsed._meta.credits_charged).toBe(35)
|
|
178
|
+
// Jane commented AND reacted — one contact, both engagement types.
|
|
179
|
+
expect(parsed.data.count).toBe(2)
|
|
180
|
+
const jane = parsed.data.people.find((p: { name: string }) => p.name === 'Jane Smith')
|
|
181
|
+
expect(jane.engagement_types.sort()).toEqual(['COMMENT', 'REACTION'])
|
|
182
|
+
expect(jane.profile_url).toBe('https://linkedin.com/in/jane')
|
|
183
|
+
// The caller must know these records are thinner than the primary's.
|
|
184
|
+
expect(parsed._meta.note).toMatch(/no work email/i)
|
|
185
|
+
})
|
|
186
|
+
|
|
187
|
+
it('honours type=comments when falling back', async () => {
|
|
188
|
+
stubExhaustedThenLima()
|
|
189
|
+
const parsed = JSON.parse((await extractPostEngagementHandler({ post_url: POST_URL, type: 'comments' })).content[0].text)
|
|
190
|
+
expect(parsed.data.people.map((p: { name: string }) => p.name)).toEqual(['Jane Smith'])
|
|
191
|
+
})
|
|
192
|
+
|
|
193
|
+
it('reports the ORIGINAL failure when the fallback also fails', async () => {
|
|
194
|
+
stubExhaustedThenLima(false)
|
|
195
|
+
const result = await extractPostEngagementHandler({ post_url: POST_URL, type: 'both' })
|
|
196
|
+
const parsed = JSON.parse(result.content[0].text)
|
|
197
|
+
expect(result.isError).toBe(true)
|
|
198
|
+
expect(parsed.error).toMatch(/out of credits/i)
|
|
199
|
+
expect(parsed.fallback_error).toBeTruthy()
|
|
200
|
+
})
|
|
201
|
+
|
|
202
|
+
it('does not spend on a fallback when it is switched off', async () => {
|
|
203
|
+
vi.stubEnv('COLDIQ_ENGAGEMENT_FALLBACK', '0')
|
|
204
|
+
let limaCalled = false
|
|
205
|
+
globalThis.fetch = vi.fn(async (url: string | URL) => {
|
|
206
|
+
const u = url.toString()
|
|
207
|
+
if (u.includes('/limadata')) { limaCalled = true; return res({}) }
|
|
208
|
+
if (u.includes('/jungler/tasks/task_1/status')) return res({ task_id: 'task_1', status: 'success', credits_exhausted: true })
|
|
209
|
+
if (u.includes('/jungler/workbooks')) return res({ task_id: 'task_1', status: 'PENDING', workbook_id: 'wb_1' })
|
|
210
|
+
throw new Error(`unexpected url ${u}`)
|
|
211
|
+
}) as typeof fetch
|
|
212
|
+
|
|
213
|
+
const result = await extractPostEngagementHandler({ post_url: POST_URL, type: 'both' })
|
|
214
|
+
expect(result.isError).toBe(true)
|
|
215
|
+
expect(limaCalled).toBe(false)
|
|
216
|
+
})
|
|
217
|
+
|
|
218
|
+
it('does NOT fall back for a genuinely empty post — that would buy a second empty result', async () => {
|
|
219
|
+
let limaCalled = false
|
|
220
|
+
globalThis.fetch = vi.fn(async (url: string | URL) => {
|
|
221
|
+
const u = url.toString()
|
|
222
|
+
if (u.includes('/limadata')) { limaCalled = true; return res({}) }
|
|
223
|
+
if (u.includes('/contacts')) return res({ items: [], total: 0, pages: 0 })
|
|
224
|
+
if (u.includes('/jungler/tasks/task_1/status')) return res({ task_id: 'task_1', status: 'success', workbook_id: 'wb_1' })
|
|
225
|
+
if (u.includes('/jungler/workbooks')) return res({ task_id: 'task_1', status: 'PENDING', workbook_id: 'wb_1' })
|
|
226
|
+
throw new Error(`unexpected url ${u}`)
|
|
227
|
+
}) as typeof fetch
|
|
228
|
+
|
|
229
|
+
const parsed = JSON.parse((await extractPostEngagementHandler({ post_url: POST_URL, type: 'both' })).content[0].text)
|
|
230
|
+
expect(limaCalled).toBe(false)
|
|
231
|
+
expect(parsed.data.count).toBe(0)
|
|
232
|
+
})
|
|
233
|
+
|
|
70
234
|
it('surfaces the create error when the job cannot start', async () => {
|
|
71
235
|
globalThis.fetch = vi.fn(async () => res({ error: 'Invalid LinkedIn post URL' }, 400)) as typeof fetch
|
|
72
236
|
const result = await extractPostEngagementHandler({ post_url: POST_URL, type: 'both' })
|