fullstackgtm 0.40.0 → 0.42.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/connector.ts CHANGED
@@ -2,6 +2,7 @@ import { dedupeKey } from "./dedupe.ts";
2
2
  import { requiresHumanInput } from "./rules.ts";
3
3
  import type {
4
4
  CanonicalGtmSnapshot,
5
+ CreateRecordPayload,
5
6
  GtmConnector,
6
7
  PatchOperation,
7
8
  PatchOperationResult,
@@ -283,7 +284,43 @@ export async function applyPatchPlan(
283
284
  }
284
285
  }
285
286
 
287
+ // Bulk fast-path: route independent, approved create_record CONTACT ops
288
+ // through the connector's batch API (batched resolve-first read + batched
289
+ // writes) instead of one round-trip per record. Only ops that are safe to
290
+ // batch are eligible — every other op (grouped, needs an override, carries a
291
+ // company association, conflicted, or any non-create op) stays on the serial
292
+ // path below, so the safety contract is unchanged. Results are merged in by
293
+ // id; the serial loop short-circuits anything already resolved here.
294
+ const batched = new Map<string, PatchOperationResult>();
295
+ if (connector.applyCreateContactsBatch && !guardFailure) {
296
+ const eligible = plan.operations.filter(
297
+ (operation) =>
298
+ approved.has(operation.id) &&
299
+ operation.operation === "create_record" &&
300
+ operation.objectType === "contact" &&
301
+ options.valueOverrides?.[operation.id] === undefined &&
302
+ !requiresHumanInput(operation.afterValue) &&
303
+ !operation.groupId &&
304
+ !(operation.afterValue as CreateRecordPayload | undefined)?.associateCompanyName &&
305
+ !conflicts.has(operation.id) &&
306
+ !irreversibleStale.has(operation.id) &&
307
+ !(staleByFilter && staleByFilter.has(operation.objectId)),
308
+ );
309
+ // Only worth the batch machinery for more than a couple of creates.
310
+ if (eligible.length > 2) {
311
+ const batchResults = await connector.applyCreateContactsBatch(eligible);
312
+ for (const result of batchResults) batched.set(result.operationId, result);
313
+ }
314
+ }
315
+
286
316
  for (const operation of plan.operations) {
317
+ const batchedResult = batched.get(operation.id);
318
+ if (batchedResult) {
319
+ results.push(batchedResult);
320
+ attempted += 1;
321
+ if (batchedResult.status === "applied") applied += 1;
322
+ continue;
323
+ }
287
324
  if (!approved.has(operation.id)) {
288
325
  results.push({
289
326
  operationId: operation.id,
@@ -0,0 +1,242 @@
1
+ /**
2
+ * ATS job-board adapters for the `signals` layer — the free, no-auth example of
3
+ * "watch an account for hiring motion." Each public board JSON endpoint turns a
4
+ * company's board token into a list of open roles; the signals layer maps each
5
+ * role to a `job` signal whose `quote` is the role title plus a JD snippet that
6
+ * VERBATIM contains a configured keyword (so the downstream judge/drafter can
7
+ * gate a why-now on something a human can verify).
8
+ *
9
+ * Mirrors the `prospectSources.ts` raw-fetch idiom exactly: zero runtime deps
10
+ * (global fetch only, injectable for tests), a base-url override per provider,
11
+ * and a per-request `try { } catch { continue }` so one provider's outage never
12
+ * sinks a multi-source fetch. No SDK, no key.
13
+ *
14
+ * Governance: read-only. These adapters fetch public board JSON and return open
15
+ * roles. They write nothing — the signals layer (and only with `--save`) is the
16
+ * one that persists, and it never touches the CRM.
17
+ */
18
+
19
+ export type AtsBoardSource = "greenhouse" | "lever" | "ashby";
20
+
21
+ /** One open role, normalized across the three providers. */
22
+ export type AtsJob = {
23
+ /** Role title, verbatim from the board. */
24
+ title: string;
25
+ /**
26
+ * A short JD snippet. When `keywords` are supplied, this is a window around
27
+ * the first matching keyword (kept verbatim so the keyword survives into the
28
+ * evidence quote); otherwise it is the leading slice of the description.
29
+ */
30
+ jdSnippet: string;
31
+ /** Canonical posting URL (the signal's sourceUrl). */
32
+ url: string;
33
+ /**
34
+ * Stable per-role identity within a board (provider id when present, else the
35
+ * title). Used by the reposted-role heuristic to tell "same role, gone, back"
36
+ * from "two roles that happen to share a title".
37
+ */
38
+ firstSeenKey: string;
39
+ };
40
+
41
+ type FetchImpl = typeof fetch;
42
+
43
+ const DEFAULT_BASE_URLS: Record<AtsBoardSource, string> = {
44
+ greenhouse: "https://boards-api.greenhouse.io",
45
+ lever: "https://api.lever.co",
46
+ ashby: "https://api.ashbyhq.com",
47
+ };
48
+
49
+ /** JD snippet length — long enough to carry a verbatim keyword window. */
50
+ const SNIPPET_LEN = 240;
51
+
52
+ /**
53
+ * Fetch open roles for one board. Network failures (offline, non-2xx, malformed
54
+ * JSON) yield `[]` rather than throwing, matching the per-source resilience of
55
+ * the prospect sources — a single board's outage must not abort a watchlist run.
56
+ */
57
+ export async function fetchAtsJobs(opts: {
58
+ source: AtsBoardSource;
59
+ boardToken: string;
60
+ accountDomain: string;
61
+ keywords?: string[];
62
+ fetchImpl?: FetchImpl;
63
+ apiBaseUrl?: string;
64
+ }): Promise<AtsJob[]> {
65
+ const fetchImpl = opts.fetchImpl ?? fetch;
66
+ const base = (opts.apiBaseUrl ?? DEFAULT_BASE_URLS[opts.source]).replace(/\/$/, "");
67
+ const token = opts.boardToken.trim();
68
+ if (!token) return [];
69
+ const keywords = opts.keywords ?? [];
70
+ try {
71
+ if (opts.source === "greenhouse") {
72
+ return await fetchGreenhouse(fetchImpl, base, token, keywords);
73
+ }
74
+ if (opts.source === "lever") {
75
+ return await fetchLever(fetchImpl, base, token, keywords);
76
+ }
77
+ return await fetchAshby(fetchImpl, base, token, keywords);
78
+ } catch {
79
+ // Offline / 5xx / malformed payload: an empty board, never a crash.
80
+ return [];
81
+ }
82
+ }
83
+
84
+ // ---------------------------------------------------------------------------
85
+ // Greenhouse — GET /v1/boards/<token>/jobs?content=true
86
+
87
+ async function fetchGreenhouse(
88
+ fetchImpl: FetchImpl,
89
+ base: string,
90
+ token: string,
91
+ keywords: string[],
92
+ ): Promise<AtsJob[]> {
93
+ const response = await fetchImpl(`${base}/v1/boards/${encodeURIComponent(token)}/jobs?content=true`, {
94
+ headers: { Accept: "application/json" },
95
+ });
96
+ if (!response.ok) return [];
97
+ const data = (await response.json()) as {
98
+ jobs?: Array<{ id?: number | string; title?: string; absolute_url?: string; content?: string }>;
99
+ };
100
+ const out: AtsJob[] = [];
101
+ for (const job of data.jobs ?? []) {
102
+ const title = strOrEmpty(job.title);
103
+ if (!title) continue;
104
+ const description = decodeHtmlish(strOrEmpty(job.content));
105
+ out.push({
106
+ title,
107
+ jdSnippet: snippetFor(description, keywords),
108
+ url: strOrEmpty(job.absolute_url),
109
+ firstSeenKey: job.id != null ? String(job.id) : title,
110
+ });
111
+ }
112
+ return out;
113
+ }
114
+
115
+ // ---------------------------------------------------------------------------
116
+ // Lever — GET /v0/postings/<token>?mode=json (returns a bare array)
117
+
118
+ async function fetchLever(
119
+ fetchImpl: FetchImpl,
120
+ base: string,
121
+ token: string,
122
+ keywords: string[],
123
+ ): Promise<AtsJob[]> {
124
+ const response = await fetchImpl(`${base}/v0/postings/${encodeURIComponent(token)}?mode=json`, {
125
+ headers: { Accept: "application/json" },
126
+ });
127
+ if (!response.ok) return [];
128
+ const data = (await response.json()) as Array<{
129
+ id?: string;
130
+ text?: string;
131
+ hostedUrl?: string;
132
+ descriptionPlain?: string;
133
+ description?: string;
134
+ }>;
135
+ const out: AtsJob[] = [];
136
+ for (const posting of Array.isArray(data) ? data : []) {
137
+ const title = strOrEmpty(posting.text);
138
+ if (!title) continue;
139
+ const description = strOrEmpty(posting.descriptionPlain) || decodeHtmlish(strOrEmpty(posting.description));
140
+ out.push({
141
+ title,
142
+ jdSnippet: snippetFor(description, keywords),
143
+ url: strOrEmpty(posting.hostedUrl),
144
+ firstSeenKey: strOrEmpty(posting.id) || title,
145
+ });
146
+ }
147
+ return out;
148
+ }
149
+
150
+ // ---------------------------------------------------------------------------
151
+ // Ashby — POST /posting-api/job-board/<token> with { jobBoardName }
152
+
153
+ async function fetchAshby(
154
+ fetchImpl: FetchImpl,
155
+ base: string,
156
+ token: string,
157
+ keywords: string[],
158
+ ): Promise<AtsJob[]> {
159
+ const response = await fetchImpl(`${base}/posting-api/job-board/${encodeURIComponent(token)}`, {
160
+ method: "POST",
161
+ headers: { "Content-Type": "application/json", Accept: "application/json" },
162
+ body: JSON.stringify({ jobBoardName: token, includeCompensation: false }),
163
+ });
164
+ if (!response.ok) return [];
165
+ const data = (await response.json()) as {
166
+ jobs?: Array<{
167
+ id?: string;
168
+ title?: string;
169
+ jobUrl?: string;
170
+ applyUrl?: string;
171
+ descriptionPlain?: string;
172
+ descriptionHtml?: string;
173
+ }>;
174
+ };
175
+ const out: AtsJob[] = [];
176
+ for (const job of data.jobs ?? []) {
177
+ const title = strOrEmpty(job.title);
178
+ if (!title) continue;
179
+ const description = strOrEmpty(job.descriptionPlain) || decodeHtmlish(strOrEmpty(job.descriptionHtml));
180
+ out.push({
181
+ title,
182
+ jdSnippet: snippetFor(description, keywords),
183
+ url: strOrEmpty(job.jobUrl) || strOrEmpty(job.applyUrl),
184
+ firstSeenKey: strOrEmpty(job.id) || title,
185
+ });
186
+ }
187
+ return out;
188
+ }
189
+
190
+ // ---------------------------------------------------------------------------
191
+ // Snippet selection: a verbatim window around the first matching keyword.
192
+
193
+ /**
194
+ * Pick a verbatim JD snippet. If a keyword matches the description (case-
195
+ * insensitive), return a window centered on the first match — kept verbatim so
196
+ * the keyword text survives into the signal quote and the evidence gate holds.
197
+ * With no keywords (or no match), fall back to the leading slice. Returns "" for
198
+ * an empty description so the caller can decide whether the role still qualifies.
199
+ */
200
+ export function snippetFor(description: string, keywords: string[]): string {
201
+ const text = description.trim();
202
+ if (!text) return "";
203
+ for (const keyword of keywords) {
204
+ const needle = keyword.trim();
205
+ if (!needle) continue;
206
+ const idx = text.toLowerCase().indexOf(needle.toLowerCase());
207
+ if (idx === -1) continue;
208
+ const pad = Math.max(0, Math.floor((SNIPPET_LEN - needle.length) / 2));
209
+ let start = Math.max(0, idx - pad);
210
+ let end = Math.min(text.length, idx + needle.length + pad);
211
+ // Snap to word boundaries so the window doesn't start/end mid-word, but
212
+ // never past the keyword itself (the verbatim span must stay intact).
213
+ while (start > 0 && /\S/.test(text[start - 1]) && start < idx) start += 1;
214
+ while (end < text.length && /\S/.test(text[end]) && end > idx + needle.length) end -= 1;
215
+ return text.slice(start, end).trim();
216
+ }
217
+ return text.slice(0, SNIPPET_LEN).trim();
218
+ }
219
+
220
+ /**
221
+ * Minimal HTML-ish cleanup for board `content`/`description` HTML: drop tags and
222
+ * decode the handful of entities Greenhouse/Ashby emit. Dependency-free; good
223
+ * enough to surface a readable, keyword-matchable snippet (the quote does not
224
+ * need to be byte-perfect HTML, only verbatim text the keyword appears in).
225
+ */
226
+ function decodeHtmlish(raw: string): string {
227
+ return raw
228
+ .replace(/<\s*br\s*\/?\s*>/gi, " ")
229
+ .replace(/<\/?[^>]+>/g, " ")
230
+ .replace(/&amp;/g, "&")
231
+ .replace(/&lt;/g, "<")
232
+ .replace(/&gt;/g, ">")
233
+ .replace(/&quot;/g, '"')
234
+ .replace(/&#39;|&apos;/g, "'")
235
+ .replace(/&nbsp;/g, " ")
236
+ .replace(/\s+/g, " ")
237
+ .trim();
238
+ }
239
+
240
+ function strOrEmpty(value: unknown): string {
241
+ return typeof value === "string" ? value.trim() : "";
242
+ }
@@ -660,6 +660,130 @@ export function createHubspotConnector(options: HubspotConnectorOptions): Requir
660
660
  };
661
661
  }
662
662
 
663
+ /** Bulk resolve-first lookup: which match values already exist (value→id). */
664
+ async function searchContactsByValues(property: string, values: string[]): Promise<Map<string, string>> {
665
+ const found = new Map<string, string>();
666
+ for (let i = 0; i < values.length; i += 100) {
667
+ const chunk = values.slice(i, i + 100);
668
+ const data = await request(`/crm/v3/objects/contacts/search`, {
669
+ method: "POST",
670
+ body: JSON.stringify({
671
+ filterGroups: [{ filters: [{ propertyName: property, operator: "IN", values: chunk }] }],
672
+ properties: [property],
673
+ limit: 100,
674
+ }),
675
+ });
676
+ for (const rec of (data?.results ?? []) as Array<{ id?: string; properties?: Record<string, string> }>) {
677
+ const v = rec?.properties?.[property];
678
+ if (typeof v === "string" && rec.id) found.set(v.toLowerCase(), String(rec.id));
679
+ }
680
+ }
681
+ return found;
682
+ }
683
+
684
+ /**
685
+ * Batch create_record for contacts: bulk resolve-first (search IN), then
686
+ * bulk-create the genuine misses (batch/create, 100/call) — two API calls per
687
+ * ~100 leads instead of two per lead. On a batch-create rejection (e.g. an
688
+ * email collision HubSpot 409s the whole batch over), it falls back to
689
+ * per-record createRecord for that chunk, so one bad row never sinks the rest
690
+ * and resolve-first is preserved. Returns one result per input op.
691
+ */
692
+ async function applyCreateContactsBatch(operations: PatchOperation[]): Promise<PatchOperationResult[]> {
693
+ const byId = new Map<string, PatchOperationResult>();
694
+ // Group by the HubSpot search property (matchKey is usually uniform across a
695
+ // plan, but a mixed plan stays correct this way).
696
+ const groups = new Map<string, PatchOperation[]>();
697
+ for (const op of operations) {
698
+ const matchKey = (op.afterValue as CreateRecordPayload | undefined)?.matchKey || "email";
699
+ const searchProperty = HUBSPOT_DEFAULT_FIELD_MAPPINGS.contacts[matchKey] ?? matchKey;
700
+ let arr = groups.get(searchProperty);
701
+ if (!arr) groups.set(searchProperty, (arr = []));
702
+ arr.push(op);
703
+ }
704
+
705
+ for (const [searchProperty, ops] of groups) {
706
+ const wanted = ops
707
+ .map((op) => String((op.afterValue as CreateRecordPayload).matchValue ?? "").trim())
708
+ .filter(Boolean);
709
+ const existing = await searchContactsByValues(searchProperty, [...new Set(wanted.map((v) => v.toLowerCase()))]);
710
+
711
+ const toCreate: PatchOperation[] = [];
712
+ const seenInBatch = new Set<string>();
713
+ for (const op of ops) {
714
+ const payload = op.afterValue as CreateRecordPayload;
715
+ const matchKey = payload.matchKey || "email";
716
+ const matchValue = String(payload.matchValue ?? "").trim();
717
+ if (!matchValue) {
718
+ byId.set(op.id, { operationId: op.id, status: "skipped", detail: "create_record needs a non-empty matchValue to resolve-first." });
719
+ continue;
720
+ }
721
+ const lower = matchValue.toLowerCase();
722
+ const dedupeKey = `${matchKey}:${lower}`;
723
+ const already = createdContactsByMatch.get(dedupeKey);
724
+ if (already) {
725
+ byId.set(op.id, { operationId: op.id, status: "skipped", detail: `Contact ${matchKey}=${matchValue} already created earlier in this run; not duplicating.`, providerData: { id: already, existing: true } });
726
+ continue;
727
+ }
728
+ const existingId = existing.get(lower);
729
+ if (existingId) {
730
+ createdContactsByMatch.set(dedupeKey, existingId);
731
+ byId.set(op.id, { operationId: op.id, status: "skipped", detail: `Contact ${matchKey}=${matchValue} already exists (${existingId}); resolve-first declined to create.`, providerData: { id: existingId, existing: true } });
732
+ continue;
733
+ }
734
+ if (seenInBatch.has(lower)) {
735
+ byId.set(op.id, { operationId: op.id, status: "skipped", detail: `Contact ${matchKey}=${matchValue} duplicated within this batch; not duplicating.` });
736
+ continue;
737
+ }
738
+ seenInBatch.add(lower);
739
+ toCreate.push(op);
740
+ }
741
+
742
+ for (let i = 0; i < toCreate.length; i += 100) {
743
+ const chunk = toCreate.slice(i, i + 100);
744
+ const inputs = chunk.map((op) => {
745
+ const payload = op.afterValue as CreateRecordPayload;
746
+ const properties: Record<string, string> = { ...payload.properties };
747
+ if (payload.ownerId && !properties.hubspot_owner_id) properties.hubspot_owner_id = String(payload.ownerId);
748
+ properties.hs_object_source_detail_2 = `fullstackgtm acquire (${op.id})`;
749
+ return { properties };
750
+ });
751
+ try {
752
+ const data = await request(`/crm/v3/objects/contacts/batch/create`, {
753
+ method: "POST",
754
+ body: JSON.stringify({ inputs }),
755
+ });
756
+ const created = (data?.results ?? []) as Array<{ id?: string; properties?: Record<string, string> }>;
757
+ const idByValue = new Map<string, string>();
758
+ for (const rec of created) {
759
+ const v = rec?.properties?.[searchProperty];
760
+ if (typeof v === "string" && rec.id) idByValue.set(v.toLowerCase(), String(rec.id));
761
+ }
762
+ for (const op of chunk) {
763
+ const payload = op.afterValue as CreateRecordPayload;
764
+ const matchKey = payload.matchKey || "email";
765
+ const matchValue = String(payload.matchValue).trim();
766
+ const id = idByValue.get(matchValue.toLowerCase());
767
+ if (id) {
768
+ createdContactsByMatch.set(`${matchKey}:${matchValue.toLowerCase()}`, id);
769
+ byId.set(op.id, { operationId: op.id, status: "applied", detail: `Created contact ${matchKey}=${matchValue} (${id}).`, providerData: { id, created: true } });
770
+ } else {
771
+ // Result didn't echo this value — resolve it per-record to be safe.
772
+ byId.set(op.id, await createRecord(op));
773
+ }
774
+ }
775
+ } catch {
776
+ // Whole-chunk rejection (e.g. a collision). Isolate per record.
777
+ for (const op of chunk) byId.set(op.id, await createRecord(op));
778
+ }
779
+ }
780
+ }
781
+
782
+ return operations.map(
783
+ (op) => byId.get(op.id) ?? { operationId: op.id, status: "skipped", detail: "create_record was not processed." },
784
+ );
785
+ }
786
+
663
787
  async function createTask(operation: PatchOperation): Promise<PatchOperationResult> {
664
788
  const associationTypeId = TASK_ASSOCIATION_TYPE_IDS[operation.objectType];
665
789
  if (associationTypeId === undefined) {
@@ -926,6 +1050,7 @@ export function createHubspotConnector(options: HubspotConnectorOptions): Requir
926
1050
  fetchSnapshot,
927
1051
  fetchChanges,
928
1052
  applyOperation,
1053
+ applyCreateContactsBatch,
929
1054
  readField,
930
1055
  };
931
1056
  }
@@ -180,6 +180,52 @@ function normalizeLinkedin(value: string | undefined): string | undefined {
180
180
  * full name + a company domain (or name). Returns the prospects with `email`
181
181
  * filled where the waterfall found one; rows with no hit are returned unchanged.
182
182
  */
183
+ const sleep = (ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms));
184
+
185
+ /** Run `count` async tasks with at most `limit` in flight. Order-independent. */
186
+ async function runWithConcurrency<T>(count: number, limit: number, worker: (index: number) => Promise<T>): Promise<T[]> {
187
+ const out: T[] = new Array(count);
188
+ let next = 0;
189
+ const runners = Array.from({ length: Math.max(1, Math.min(limit, count || 1)) }, async () => {
190
+ for (let i = next++; i < count; i = next++) out[i] = await worker(i);
191
+ });
192
+ await Promise.all(runners);
193
+ return out;
194
+ }
195
+
196
+ /**
197
+ * POST a pipe0 sync run with exponential backoff on transient failure
198
+ * (throttle/5xx/network). pipe0's waterfall throttles under burst — a 429'd
199
+ * chunk returns all-failed — so concurrency must be paired with real backoff,
200
+ * not a single retry, or coverage collapses at scale (a parallel run without
201
+ * this dropped the work-email hit-rate from ~79% to ~17%). Retries 500ms →
202
+ * 1s → 2s → 4s (capped) so a throttled chunk lands on a later attempt instead
203
+ * of being lost.
204
+ */
205
+ async function pipe0Post(
206
+ fetchImpl: FetchImpl,
207
+ base: string,
208
+ apiKey: string,
209
+ payload: unknown,
210
+ maxAttempts = 5,
211
+ ): Promise<Pipe0RunResponse | undefined> {
212
+ for (let attempt = 0; attempt < maxAttempts; attempt++) {
213
+ try {
214
+ const response = await fetchImpl(`${base}/v1/pipes/run/sync`, {
215
+ method: "POST",
216
+ headers: { Authorization: `Bearer ${apiKey}`, "Content-Type": "application/json" },
217
+ body: JSON.stringify(payload),
218
+ });
219
+ if (response.ok) return (await response.json()) as Pipe0RunResponse;
220
+ // non-ok (429/5xx) — fall through to backoff + retry
221
+ } catch {
222
+ // network error — fall through to backoff + retry
223
+ }
224
+ if (attempt < maxAttempts - 1) await sleep(Math.min(8000, 500 * 2 ** attempt));
225
+ }
226
+ return undefined; // still failing after backoff — caller keeps the other chunks
227
+ }
228
+
183
229
  export async function pipe0ResolveWorkEmails(opts: {
184
230
  apiKey: string;
185
231
  prospects: Prospect[];
@@ -189,43 +235,48 @@ export async function pipe0ResolveWorkEmails(opts: {
189
235
  * batch rate-limit (a throttled chunk returns work_email status "failed"
190
236
  * for every row, so one big batch is all-or-nothing). Default 3. */
191
237
  chunkSize?: number;
238
+ /** Chunks resolved in parallel, bounded to respect pipe0's rate limit. Default
239
+ * 3 — paired with pipe0Post's backoff this cuts wall-clock from O(n) serial
240
+ * calls without throttling away coverage (higher defaults tanked the hit-rate
241
+ * at scale). */
242
+ concurrency?: number;
192
243
  }): Promise<Prospect[]> {
193
244
  const fetchImpl = opts.fetchImpl ?? fetch;
194
245
  const base = (opts.apiBaseUrl ?? "https://api.pipe0.com").replace(/\/$/, "");
195
246
  const chunkSize = Math.max(1, opts.chunkSize ?? 3);
247
+ const concurrency = Math.max(1, opts.concurrency ?? 3);
196
248
 
197
249
  // Only rows with the waterfall's required inputs (name + company_domain|name).
198
250
  const resolvable = opts.prospects.filter((p) => p.fullName && (p.companyDomain || p.companyName));
199
251
  if (resolvable.length === 0) return opts.prospects;
200
252
 
201
- const emailByKey = new Map<string, string>();
202
- for (let i = 0; i < resolvable.length; i += chunkSize) {
203
- const chunk = resolvable.slice(i, i + chunkSize);
204
- const input = chunk.map((p) => ({
253
+ const chunks: Prospect[][] = [];
254
+ for (let i = 0; i < resolvable.length; i += chunkSize) chunks.push(resolvable.slice(i, i + chunkSize));
255
+
256
+ const perChunk = await runWithConcurrency(chunks.length, concurrency, async (ci) => {
257
+ const input = chunks[ci].map((p) => ({
205
258
  name: p.fullName,
206
259
  ...(p.companyDomain ? { company_domain: p.companyDomain } : { company_name: p.companyName }),
207
260
  }));
208
- let body: Pipe0RunResponse;
209
- try {
210
- const response = await fetchImpl(`${base}/v1/pipes/run/sync`, {
211
- method: "POST",
212
- headers: { Authorization: `Bearer ${opts.apiKey}`, "Content-Type": "application/json" },
213
- body: JSON.stringify({ pipes: [{ pipe_id: "person:workemail:waterfall@1" }], input }),
214
- });
215
- if (!response.ok) continue; // throttled/5xx chunk — skip, keep the rest
216
- body = (await response.json()) as Pipe0RunResponse;
217
- } catch {
218
- continue; // network hiccup on one chunk must not sink the whole run
219
- }
220
- for (const recordId of body.order ?? []) {
221
- const fields = body.records?.[recordId]?.fields ?? {};
261
+ const body = await pipe0Post(fetchImpl, base, opts.apiKey, {
262
+ pipes: [{ pipe_id: "person:workemail:waterfall@1" }],
263
+ input,
264
+ });
265
+ const entries: Array<[string, string]> = [];
266
+ for (const recordId of body?.order ?? []) {
267
+ const fields = body!.records?.[recordId]?.fields ?? {};
222
268
  const email = fields.work_email?.status === "completed" ? fieldValue(fields.work_email) : undefined;
223
269
  if (email) {
224
270
  const domain = fieldValue(fields.company_domain) ?? fieldValue(fields.company_name);
225
- emailByKey.set(personKey(fieldValue(fields.name), domain), email);
271
+ entries.push([personKey(fieldValue(fields.name), domain), email]);
226
272
  }
227
273
  }
228
- }
274
+ return entries;
275
+ });
276
+
277
+ const emailByKey = new Map<string, string>();
278
+ for (const entries of perChunk) for (const [k, v] of entries) emailByKey.set(k, v);
279
+
229
280
  return opts.prospects.map((p) => {
230
281
  const email = emailByKey.get(personKey(p.fullName, p.companyDomain ?? p.companyName));
231
282
  return email ? { ...p, email } : p;
@@ -262,10 +313,13 @@ export async function pipe0ResolveCompanyDomains(opts: {
262
313
  apiBaseUrl?: string;
263
314
  fetchImpl?: FetchImpl;
264
315
  chunkSize?: number;
316
+ /** Chunks resolved in parallel (bounded). Default 3. */
317
+ concurrency?: number;
265
318
  }): Promise<Prospect[]> {
266
319
  const fetchImpl = opts.fetchImpl ?? fetch;
267
320
  const base = (opts.apiBaseUrl ?? "https://api.pipe0.com").replace(/\/$/, "");
268
321
  const chunkSize = Math.max(1, opts.chunkSize ?? 5);
322
+ const concurrency = Math.max(1, opts.concurrency ?? 3);
269
323
 
270
324
  const uniqueNames = [
271
325
  ...new Set(
@@ -277,30 +331,27 @@ export async function pipe0ResolveCompanyDomains(opts: {
277
331
  ];
278
332
  if (uniqueNames.length === 0) return opts.prospects;
279
333
 
280
- const domainByName = new Map<string, string>();
281
- for (let i = 0; i < uniqueNames.length; i += chunkSize) {
282
- const chunk = uniqueNames.slice(i, i + chunkSize);
283
- try {
284
- const response = await fetchImpl(`${base}/v1/pipes/run/sync`, {
285
- method: "POST",
286
- headers: { Authorization: `Bearer ${opts.apiKey}`, "Content-Type": "application/json" },
287
- body: JSON.stringify({
288
- pipes: [{ pipe_id: "company:identity@1" }],
289
- input: chunk.map((company_name) => ({ company_name })),
290
- }),
291
- });
292
- if (!response.ok) continue;
293
- const body = (await response.json()) as Pipe0RunResponse;
294
- for (const recordId of body.order ?? []) {
295
- const fields = body.records?.[recordId]?.fields ?? {};
296
- const name = fieldValue(fields.company_name) ?? fieldValue(fields.cleaned_company_name);
297
- const domain = hostFromUrl(fieldValue(fields.company_website_url));
298
- if (name && domain) domainByName.set(name.trim().toLowerCase(), domain);
299
- }
300
- } catch {
301
- continue;
334
+ const chunks: string[][] = [];
335
+ for (let i = 0; i < uniqueNames.length; i += chunkSize) chunks.push(uniqueNames.slice(i, i + chunkSize));
336
+
337
+ const perChunk = await runWithConcurrency(chunks.length, concurrency, async (ci) => {
338
+ const body = await pipe0Post(fetchImpl, base, opts.apiKey, {
339
+ pipes: [{ pipe_id: "company:identity@1" }],
340
+ input: chunks[ci].map((company_name) => ({ company_name })),
341
+ });
342
+ const entries: Array<[string, string]> = [];
343
+ for (const recordId of body?.order ?? []) {
344
+ const fields = body!.records?.[recordId]?.fields ?? {};
345
+ const name = fieldValue(fields.company_name) ?? fieldValue(fields.cleaned_company_name);
346
+ const domain = hostFromUrl(fieldValue(fields.company_website_url));
347
+ if (name && domain) entries.push([name.trim().toLowerCase(), domain]);
302
348
  }
303
- }
349
+ return entries;
350
+ });
351
+
352
+ const domainByName = new Map<string, string>();
353
+ for (const entries of perChunk) for (const [k, v] of entries) domainByName.set(k, v);
354
+
304
355
  return opts.prospects.map((p) => {
305
356
  if (p.companyDomain || !p.companyName) return p;
306
357
  const domain = domainByName.get(p.companyName.trim().toLowerCase());