fullstackgtm 0.49.0 → 0.50.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +77 -0
- package/DATA-FLOWS.md +1 -0
- package/README.md +48 -0
- package/dist/acquireCheckpoint.d.ts +33 -0
- package/dist/acquireCheckpoint.js +149 -0
- package/dist/acquireLinkedIn.d.ts +2 -0
- package/dist/acquireLinkedIn.js +1 -1
- package/dist/cli/audit.js +6 -1
- package/dist/cli/auth.js +25 -0
- package/dist/cli/backfill.js +4 -0
- package/dist/cli/call.js +28 -6
- package/dist/cli/draft.js +11 -1
- package/dist/cli/enrich.js +286 -81
- package/dist/cli/fix.js +6 -3
- package/dist/cli/help.js +30 -26
- package/dist/cli/icp.js +15 -1
- package/dist/cli/market.js +14 -1
- package/dist/cli/planOutput.d.ts +5 -0
- package/dist/cli/planOutput.js +27 -0
- package/dist/cli/plans.js +269 -24
- package/dist/cli/schedule.js +22 -11
- package/dist/cli/shared.d.ts +3 -1
- package/dist/cli/shared.js +2 -2
- package/dist/cli/signals.js +22 -3
- package/dist/cli/tam.js +10 -1
- package/dist/cli/ui.d.ts +10 -5
- package/dist/cli/ui.js +28 -9
- package/dist/connectors/hubspot.d.ts +4 -0
- package/dist/connectors/hubspot.js +36 -18
- package/dist/connectors/linkedin.d.ts +2 -0
- package/dist/connectors/linkedin.js +5 -0
- package/dist/connectors/prospectSources.d.ts +23 -0
- package/dist/connectors/prospectSources.js +21 -3
- package/dist/enrich.d.ts +44 -1
- package/dist/hostedAcquireCheckpoint.d.ts +43 -0
- package/dist/hostedAcquireCheckpoint.js +129 -0
- package/dist/hostedPatchPlan.d.ts +87 -0
- package/dist/hostedPatchPlan.js +275 -0
- package/dist/icp.js +13 -2
- package/dist/index.d.ts +4 -1
- package/dist/index.js +3 -0
- package/dist/planStore.d.ts +14 -0
- package/dist/planStore.js +73 -0
- package/dist/progress.d.ts +2 -2
- package/dist/progress.js +2 -2
- package/docs/api.md +24 -0
- package/docs/architecture.md +9 -0
- package/package.json +1 -1
- package/skills/fullstackgtm/SKILL.md +1 -1
- package/src/acquireCheckpoint.ts +186 -0
- package/src/acquireLinkedIn.ts +3 -1
- package/src/cli/audit.ts +5 -1
- package/src/cli/auth.ts +24 -0
- package/src/cli/backfill.ts +3 -0
- package/src/cli/call.ts +25 -6
- package/src/cli/draft.ts +9 -1
- package/src/cli/enrich.ts +325 -80
- package/src/cli/fix.ts +5 -3
- package/src/cli/help.ts +31 -26
- package/src/cli/icp.ts +13 -1
- package/src/cli/market.ts +12 -1
- package/src/cli/planOutput.ts +30 -0
- package/src/cli/plans.ts +259 -25
- package/src/cli/schedule.ts +21 -15
- package/src/cli/shared.ts +2 -1
- package/src/cli/signals.ts +22 -3
- package/src/cli/tam.ts +8 -1
- package/src/cli/ui.ts +31 -13
- package/src/connectors/hubspot.ts +41 -17
- package/src/connectors/linkedin.ts +6 -0
- package/src/connectors/prospectSources.ts +46 -4
- package/src/enrich.ts +47 -1
- package/src/hostedAcquireCheckpoint.ts +156 -0
- package/src/hostedPatchPlan.ts +291 -0
- package/src/icp.ts +14 -2
- package/src/index.ts +20 -0
- package/src/planStore.ts +87 -0
- package/src/progress.ts +2 -2
package/dist/cli/enrich.js
CHANGED
|
@@ -1,13 +1,17 @@
|
|
|
1
1
|
// Extracted verbatim from src/cli.ts (mechanical split, no behavior change).
|
|
2
2
|
import { existsSync, readFileSync } from "node:fs";
|
|
3
|
+
import { createHash } from "node:crypto";
|
|
3
4
|
import { resolve } from "node:path";
|
|
4
5
|
import { getCredential } from "../credentials.js";
|
|
5
6
|
import { patchPlanToMarkdown } from "../format.js";
|
|
6
7
|
import { createFilePlanStore } from "../planStore.js";
|
|
7
8
|
import { buildAcquirePlan, buildEnrichPlan, createFileEnrichRunStore, DEFAULT_STALE_DAYS, ENRICH_CONFIG_FILE_NAME, builtinAcquirePreset, builtinEnrichPreset, enrichRunId, inferIngestObjectType, latestStamps, loadEnrichConfig, parseCsv, resolveCrmField, selectStaleWork, stagedSourceRecords, staleDaysFor } from "../enrich.js";
|
|
8
9
|
import { loadMeter, remaining } from "../acquireMeter.js";
|
|
9
|
-
import { crmContactKeys, fetchExploriumProspects,
|
|
10
|
+
import { crmContactKeys, fetchExploriumProspects, fetchPipe0CrustdataProspectPage, partitionFreshProspects, pipe0ResolveCompanyDomains, pipe0ResolveWorkEmails, prospectIdentityKeys } from "../connectors/prospectSources.js";
|
|
10
11
|
import { loadSeen, recordSeen } from "../acquireSeen.js";
|
|
12
|
+
import { acquireCheckpointId, createFileAcquireCheckpointStore } from "../acquireCheckpoint.js";
|
|
13
|
+
import { readHostedAcquireCheckpoint, writeHostedAcquireCheckpoint } from "../hostedAcquireCheckpoint.js";
|
|
14
|
+
import { uploadHostedPatchPlan } from "../hostedPatchPlan.js";
|
|
11
15
|
import { progressReporter, reportCounts, reportEvent } from "../runReport.js";
|
|
12
16
|
import { ACQUIRE_STAGES, composeListeners, createProgressEmitter } from "../progress.js";
|
|
13
17
|
import { createLinkedInProvider, discoverLinkedInProspects } from "../acquireLinkedIn.js";
|
|
@@ -18,6 +22,7 @@ import { isOptionValue, loadIcp, numericOption, option, readSnapshot, saveReques
|
|
|
18
22
|
import { providerKey } from "./tam.js";
|
|
19
23
|
import { unknownSubcommandError } from "./suggest.js";
|
|
20
24
|
import { colorEnabled, createProgressRenderer, createStatusLine, formatBar, formatDuration, paint } from "./ui.js";
|
|
25
|
+
import { compactPlan, verbosePlanRequested } from "./planOutput.js";
|
|
21
26
|
/**
|
|
22
27
|
* The enrich layer: governed append/refresh of third-party data (Apollo pull,
|
|
23
28
|
* Clay ingest) into the CRM through the normal dry-run → approval → apply
|
|
@@ -37,7 +42,7 @@ enrich append [--source apollo] [--objects companies,contacts] [--save] [--conf
|
|
|
37
42
|
enrich refresh [--source apollo] [--stale-days <n>] [--save] [--config <path>]
|
|
38
43
|
[source options] [--run-label <label>] [--json]
|
|
39
44
|
enrich ingest <file.csv|payload.json|spool.jsonl|spool-dir> --source clay [--run-label <label>] [--objects companies|contacts] [--config <path>]
|
|
40
|
-
enrich acquire [--source <id>] [--max <
|
|
45
|
+
enrich acquire [--source <id>] [--max <new-leads>] [--scan-limit <candidates>] [--list <id>] [--assign-owner <id>] [--save] [--config <path>] [--json]
|
|
41
46
|
enrich status [--runs] [--source <id>] [--config <path>] [--json]
|
|
42
47
|
|
|
43
48
|
acquire creates NET-NEW leads from a staged prospect list (ingest first):
|
|
@@ -47,6 +52,9 @@ it dedupes each sourced row against the CRM, skips matches and ambiguities
|
|
|
47
52
|
remaining budget (records + spend, per day and per month; whichever is hit
|
|
48
53
|
first). Approval-gated like every write: \`--save\` → plans approve → apply.
|
|
49
54
|
The meter is charged only when a create actually lands at apply.
|
|
55
|
+
\`--max\` controls the desired net-new plan size; \`--scan-limit\` separately
|
|
56
|
+
bounds raw candidates scanned after ICP and dedupe filtering. Continuation is
|
|
57
|
+
persisted per query so scheduled runs advance instead of rereading page one.
|
|
50
58
|
Discovery sources (zero-config presets): explorium | pipe0 (ICP-driven search),
|
|
51
59
|
and linkedin (reads a HeyReach lead list — pass --list <id>, key on the LinkedIn
|
|
52
60
|
URL, $0/record; \`login heyreach\` or HEYREACH_API_KEY).
|
|
@@ -125,7 +133,7 @@ are phase 2. Recurring execution is the scheduler's job; enrich has no cron.`);
|
|
|
125
133
|
const today = new Date().toISOString().slice(0, 10);
|
|
126
134
|
// Prospects come from an API source (net-new discovery, e.g. Explorium +
|
|
127
135
|
// pipe0 work-email) or a staged ingest list (Clay/CSV/webhook).
|
|
128
|
-
const snapshot = await readSnapshot(rest);
|
|
136
|
+
const snapshot = await readSnapshot(rest, undefined, { persistProgress: true });
|
|
129
137
|
const icp = loadIcp(rest);
|
|
130
138
|
if (sourceConfig.kind === "api" && !icp) {
|
|
131
139
|
console.error("⚠ No ICP loaded — discovery will use the raw discovery.filters and skip fit scoring. " +
|
|
@@ -139,16 +147,38 @@ are phase 2. Recurring execution is the scheduler's job; enrich has no cron.`);
|
|
|
139
147
|
"acquire writes it on every new contact it creates, so coverage grows automatically.");
|
|
140
148
|
}
|
|
141
149
|
const seen = loadSeen();
|
|
150
|
+
const now = new Date();
|
|
151
|
+
const costPerRecord = config.acquire.costPerRecord?.[source] ?? 0;
|
|
152
|
+
const headroom = remaining(loadMeter(now), config.acquire.budget, costPerRecord, now);
|
|
153
|
+
const explicitMax = numericOption(rest, "--max");
|
|
154
|
+
const desiredCreates = explicitMax ?? config.acquire.discovery?.[source]?.size ?? 25;
|
|
155
|
+
const cap = headroom.maxRecords === null ? desiredCreates : Math.min(headroom.maxRecords, desiredCreates);
|
|
142
156
|
let records;
|
|
143
157
|
let apiSkippedCrm = 0;
|
|
144
158
|
let apiSkippedSeen = 0;
|
|
145
159
|
let apiProcessedKeys = [];
|
|
160
|
+
let acquireTelemetry;
|
|
161
|
+
let acquireCheckpointSync;
|
|
162
|
+
const renderer = createProgressRenderer(ACQUIRE_STAGES);
|
|
163
|
+
const acquireProgress = createProgressEmitter(composeListeners(renderer.listener, progressReporter()));
|
|
146
164
|
if (sourceConfig.kind === "api") {
|
|
147
|
-
const
|
|
165
|
+
const priorRun = await store.latest({ source, mode: "acquire" });
|
|
166
|
+
acquireProgress.stage(ACQUIRE_STAGES[0], 0, ACQUIRE_STAGES.length);
|
|
167
|
+
acquireProgress.note("loading saved audience position");
|
|
168
|
+
let api;
|
|
169
|
+
try {
|
|
170
|
+
api = await acquireFromApi(config, source, rest, icp, snapshot, seen, priorRun, cap, save, acquireProgress);
|
|
171
|
+
}
|
|
172
|
+
catch (error) {
|
|
173
|
+
renderer.done({ persist: true });
|
|
174
|
+
throw error;
|
|
175
|
+
}
|
|
148
176
|
records = api.records;
|
|
149
177
|
apiSkippedCrm = api.skippedCrm;
|
|
150
178
|
apiSkippedSeen = api.skippedSeen;
|
|
151
179
|
apiProcessedKeys = api.processedKeys;
|
|
180
|
+
acquireTelemetry = api.telemetry;
|
|
181
|
+
acquireCheckpointSync = { key: api.checkpointKey, hostedVersion: api.hostedVersion };
|
|
152
182
|
}
|
|
153
183
|
else {
|
|
154
184
|
const stagedLabel = option(rest, "--staged-run");
|
|
@@ -162,17 +192,13 @@ are phase 2. Recurring execution is the scheduler's job; enrich has no cron.`);
|
|
|
162
192
|
}
|
|
163
193
|
// Meter: how many MORE leads may we create right now? --max is an
|
|
164
194
|
// additional per-run ceiling, never a way to exceed the budget.
|
|
165
|
-
const now = new Date();
|
|
166
|
-
const costPerRecord = config.acquire.costPerRecord?.[source] ?? 0;
|
|
167
195
|
if (apiSkippedCrm || apiSkippedSeen) {
|
|
168
196
|
console.error(`Pre-email dedup: skipped ${apiSkippedCrm} already-in-CRM + ${apiSkippedSeen} seen before paying for emails ` +
|
|
169
197
|
`(≈$${((apiSkippedCrm + apiSkippedSeen) * costPerRecord).toFixed(2)} enrichment saved).`);
|
|
170
198
|
}
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
if (explicitMax !== undefined)
|
|
175
|
-
cap = cap === null ? explicitMax : Math.min(cap, explicitMax);
|
|
199
|
+
// `cap` is the desired create count bounded by the persistent meter. Raw
|
|
200
|
+
// discovery scanning has its own --scan-limit and may cross many duplicate-
|
|
201
|
+
// heavy pages to fill this target.
|
|
176
202
|
// Assignment: never create an ownerless lead. An explicit `acquire.assign`
|
|
177
203
|
// policy wins; a `--assign-owner <id>` flag is a quick fixed override;
|
|
178
204
|
// otherwise, when the portal has exactly one active owner, default every
|
|
@@ -202,11 +228,10 @@ are phase 2. Recurring execution is the scheduler's job; enrich has no cron.`);
|
|
|
202
228
|
// piped) and, via the composed reporter, heartbeat to a paired hosted app.
|
|
203
229
|
// The meter reading (creates vs headroom + budget burn) rides the same
|
|
204
230
|
// emitter, feeding the dashboard's gauge without printing anything new.
|
|
205
|
-
const renderer = createProgressRenderer(ACQUIRE_STAGES);
|
|
206
|
-
const acquireProgress = createProgressEmitter(composeListeners(renderer.listener, progressReporter()));
|
|
207
231
|
let result;
|
|
208
232
|
try {
|
|
209
|
-
acquireProgress.stage(ACQUIRE_STAGES[
|
|
233
|
+
acquireProgress.stage(ACQUIRE_STAGES[2], 2, ACQUIRE_STAGES.length);
|
|
234
|
+
acquireProgress.note("building governed create plan");
|
|
210
235
|
if (config.acquire.budget?.records?.perDay && headroom.records.day !== null) {
|
|
211
236
|
acquireProgress.meter(config.acquire.budget.records.perDay - headroom.records.day, config.acquire.budget.records.perDay, "records/day");
|
|
212
237
|
}
|
|
@@ -221,7 +246,7 @@ are phase 2. Recurring execution is the scheduler's job; enrich has no cron.`);
|
|
|
221
246
|
});
|
|
222
247
|
}
|
|
223
248
|
finally {
|
|
224
|
-
renderer.done();
|
|
249
|
+
renderer.done({ persist: true });
|
|
225
250
|
}
|
|
226
251
|
if (result.counts.unassigned > 0 && result.counts.created > 0) {
|
|
227
252
|
console.error(`⚠ ${result.counts.unassigned}/${result.counts.created} proposed lead(s) have no owner ` +
|
|
@@ -243,7 +268,7 @@ are phase 2. Recurring execution is the scheduler's job; enrich has no cron.`);
|
|
|
243
268
|
console.log(JSON.stringify({ plan: result.plan, counts: result.counts, estCostUsd: result.estCostUsd, meter: headroom }, null, 2));
|
|
244
269
|
}
|
|
245
270
|
else {
|
|
246
|
-
console.log(patchPlanToMarkdown(result.plan));
|
|
271
|
+
console.log(verbosePlanRequested(rest) ? patchPlanToMarkdown(result.plan) : compactPlan(result.plan));
|
|
247
272
|
console.log(meterLine);
|
|
248
273
|
if (gaugeLine)
|
|
249
274
|
console.log(gaugeLine);
|
|
@@ -251,32 +276,69 @@ are phase 2. Recurring execution is the scheduler's job; enrich has no cron.`);
|
|
|
251
276
|
}
|
|
252
277
|
return;
|
|
253
278
|
}
|
|
254
|
-
const run = await openEnrichRun(store, source, "
|
|
279
|
+
const run = await openEnrichRun(store, source, "acquire", option(rest, "--run-label"), today);
|
|
255
280
|
const planIds = [];
|
|
281
|
+
let hostedPlanUrl = null;
|
|
256
282
|
if (result.plan.operations.length > 0) {
|
|
257
|
-
await createFilePlanStore().save(result.plan);
|
|
283
|
+
const storedPlan = await createFilePlanStore().save(result.plan);
|
|
258
284
|
planIds.push(result.plan.id);
|
|
259
285
|
reportEvent("plan_saved", result.plan.id);
|
|
286
|
+
const mirror = await uploadHostedPatchPlan(storedPlan);
|
|
287
|
+
if (mirror.status === "saved") {
|
|
288
|
+
hostedPlanUrl = mirror.state.url;
|
|
289
|
+
reportEvent("plan_mirrored", mirror.state.patchPlanId);
|
|
290
|
+
}
|
|
291
|
+
else if (mirror.status === "unavailable" || mirror.status === "conflict") {
|
|
292
|
+
console.error(`Hosted plan sync pending: ${mirror.reason}. The local signed plan remains authoritative.`);
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
if (acquireTelemetry) {
|
|
296
|
+
acquireTelemetry.funnel.proposed = result.counts.created;
|
|
297
|
+
acquireTelemetry.funnel.withheldByMeter = result.counts.withheldByMeter;
|
|
260
298
|
}
|
|
261
299
|
await store.update({
|
|
262
300
|
...run,
|
|
263
301
|
completedAt: new Date().toISOString(),
|
|
264
|
-
cursor: null,
|
|
302
|
+
cursor: acquireTelemetry?.discovery.cursor ?? null,
|
|
303
|
+
counts: acquireTelemetry
|
|
304
|
+
? {
|
|
305
|
+
fetched: acquireTelemetry.funnel.discovered,
|
|
306
|
+
matched: acquireTelemetry.funnel.skippedCrm + acquireTelemetry.funnel.skippedSeen,
|
|
307
|
+
unmatched: acquireTelemetry.funnel.resolved,
|
|
308
|
+
ambiguous: result.counts.ambiguous,
|
|
309
|
+
opsEmitted: acquireTelemetry.funnel.proposed,
|
|
310
|
+
}
|
|
311
|
+
: run.counts,
|
|
312
|
+
acquireTelemetry,
|
|
265
313
|
planIds: [...(run.planIds ?? []), ...planIds],
|
|
266
314
|
});
|
|
267
315
|
// Remember everyone we email-resolved this run so the next run skips them
|
|
268
316
|
// pre-email (cross-run credit saver). Committed (--save) runs only.
|
|
269
317
|
if (apiProcessedKeys.length > 0)
|
|
270
318
|
recordSeen(apiProcessedKeys, now);
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
319
|
+
if (acquireTelemetry && acquireCheckpointSync) {
|
|
320
|
+
await persistAcquireCheckpoint(acquireCheckpointSync.key, acquireTelemetry.discovery, acquireCheckpointSync.hostedVersion);
|
|
321
|
+
}
|
|
274
322
|
if (planIds.length > 0) {
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
323
|
+
const hostedRuns = hostedRunsUrl();
|
|
324
|
+
console.log(`\nSaved plan ${result.plan.id}`);
|
|
325
|
+
console.log(` Leads ${result.counts.created} net-new`);
|
|
326
|
+
console.log(` Est. cost $${result.estCostUsd.toFixed(2)}`);
|
|
327
|
+
console.log(` Budget ${gaugeLine ?? meterLine}`);
|
|
328
|
+
if (hostedPlanUrl)
|
|
329
|
+
console.log(` Plan ${hostedPlanUrl}`);
|
|
330
|
+
if (hostedRuns)
|
|
331
|
+
console.log(` Observe ${hostedRuns}`);
|
|
332
|
+
console.log("\nNext steps");
|
|
333
|
+
console.log(` Review fullstackgtm plans show ${result.plan.id}`);
|
|
334
|
+
console.log(` Approve fullstackgtm plans approve ${result.plan.id} --operations all`);
|
|
335
|
+
console.log(` Apply fullstackgtm apply --plan-id ${result.plan.id} --provider hubspot`);
|
|
336
|
+
console.log("\nThe meter is charged only when a create lands at apply.");
|
|
278
337
|
}
|
|
279
338
|
else {
|
|
339
|
+
console.log(meterLine);
|
|
340
|
+
if (gaugeLine)
|
|
341
|
+
console.log(gaugeLine);
|
|
280
342
|
console.log("No net-new leads to create (everything sourced already matched, or was withheld by the meter).");
|
|
281
343
|
}
|
|
282
344
|
return;
|
|
@@ -399,7 +461,7 @@ are phase 2. Recurring execution is the scheduler's job; enrich has no cron.`);
|
|
|
399
461
|
console.log(JSON.stringify(result.plan, null, 2));
|
|
400
462
|
}
|
|
401
463
|
else {
|
|
402
|
-
console.log(patchPlanToMarkdown(result.plan));
|
|
464
|
+
console.log(verbosePlanRequested(rest) ? patchPlanToMarkdown(result.plan) : compactPlan(result.plan));
|
|
403
465
|
console.log(formatEnrichCounts(result.counts, result.ambiguities.length));
|
|
404
466
|
console.log("\nDry run — nothing written. Re-run with --save to persist the plan and the run record.");
|
|
405
467
|
}
|
|
@@ -442,69 +504,152 @@ function formatEnrichCounts(counts, ambiguities) {
|
|
|
442
504
|
* pipe0 (when resolveEmailsWith=pipe0) fills real work emails. Only rows that
|
|
443
505
|
* carry the dedupe key (email) survive — you cannot resolve-first without it.
|
|
444
506
|
*/
|
|
445
|
-
async function acquireFromApi(config, source, rest, icp, snapshot, seen) {
|
|
507
|
+
async function acquireFromApi(config, source, rest, icp, snapshot, seen, priorRun, targetFresh, persistCheckpoint, progress) {
|
|
446
508
|
const acquire = config.acquire;
|
|
447
509
|
const disc = acquire.discovery?.[source];
|
|
448
510
|
if (!disc) {
|
|
449
511
|
throw new Error(`enrich acquire: api source "${source}" needs an "acquire.discovery.${source}" config (provider + size).`);
|
|
450
512
|
}
|
|
451
513
|
const matchKey = acquire.create.contact?.matchKey ?? "email";
|
|
452
|
-
const
|
|
453
|
-
const
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
//
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
514
|
+
const explicitScanLimit = numericOption(rest, "--scan-limit");
|
|
515
|
+
const scanLimit = Math.max(targetFresh, Math.min(explicitScanLimit ?? disc.scanLimit ?? Math.max(100, targetFresh * 10), 5_000));
|
|
516
|
+
const listId = disc.listId ?? option(rest, "--list") ?? undefined;
|
|
517
|
+
if ((disc.provider === "linkedin" || disc.provider === "heyreach") && !listId) {
|
|
518
|
+
throw new Error("enrich acquire --source linkedin needs a HeyReach lead-list id: pass --list <id> or set acquire.discovery.linkedin.listId.");
|
|
519
|
+
}
|
|
520
|
+
const filters = disc.provider === "explorium"
|
|
521
|
+
? (icp ? icpToExploriumFilters(icp) : (disc.filters ?? {}))
|
|
522
|
+
: disc.provider === "pipe0"
|
|
523
|
+
? (icp ? icpToCrustdataFilters(icp) : (disc.filters ?? {}))
|
|
524
|
+
: {};
|
|
525
|
+
const queryFingerprint = createHash("sha256")
|
|
526
|
+
.update(JSON.stringify({ source, provider: disc.provider, listId, filters, icp: icp ?? null }))
|
|
527
|
+
.digest("hex");
|
|
528
|
+
const checkpointKey = {
|
|
529
|
+
provider: disc.provider,
|
|
530
|
+
source,
|
|
531
|
+
listId: listId ?? null,
|
|
532
|
+
queryFingerprint,
|
|
533
|
+
};
|
|
534
|
+
const checkpointStore = createFileAcquireCheckpointStore();
|
|
535
|
+
const localCheckpoint = await checkpointStore.get(checkpointKey);
|
|
536
|
+
const hostedRead = await readHostedAcquireCheckpoint(acquireCheckpointId(checkpointKey));
|
|
537
|
+
const hostedRecord = hostedRead.status === "found" ? hostedRead.record : null;
|
|
538
|
+
const hostedVersion = hostedRecord?.version ?? null;
|
|
539
|
+
// Once paired, hosted state lets another authenticated worker resume this
|
|
540
|
+
// audience. A newer offline local checkpoint is still preserved and pushed
|
|
541
|
+
// with CAS; otherwise hosted wins. Legacy run telemetry is a one-time
|
|
542
|
+
// migration source only when neither dedicated store has this exact key.
|
|
543
|
+
let persisted = localCheckpoint?.continuation;
|
|
544
|
+
if (hostedRecord &&
|
|
545
|
+
(!localCheckpoint || hostedRecord.updatedAt >= Date.parse(localCheckpoint.updatedAt))) {
|
|
546
|
+
persisted = hostedRecord.checkpoint;
|
|
547
|
+
if (persistCheckpoint) {
|
|
548
|
+
await checkpointStore.put(checkpointKey, {
|
|
549
|
+
cursor: hostedRecord.checkpoint.cursor,
|
|
550
|
+
offset: hostedRecord.checkpoint.offset,
|
|
551
|
+
exhausted: hostedRecord.checkpoint.exhausted,
|
|
552
|
+
}, new Date(hostedRecord.updatedAt));
|
|
553
|
+
}
|
|
554
|
+
}
|
|
555
|
+
const prior = priorRun?.acquireTelemetry?.discovery;
|
|
556
|
+
const resume = persisted ?? (prior?.queryFingerprint === queryFingerprint
|
|
557
|
+
? { cursor: prior.cursor, offset: prior.offset, exhausted: prior.exhausted }
|
|
558
|
+
: undefined);
|
|
559
|
+
let cursor = resume?.cursor ?? null;
|
|
560
|
+
let offset = resume?.offset ?? 0;
|
|
561
|
+
let exhausted = resume?.exhausted ?? false;
|
|
562
|
+
let discovered = 0;
|
|
563
|
+
let qualified = 0;
|
|
564
|
+
let skippedCrm = 0;
|
|
565
|
+
let skippedSeen = 0;
|
|
566
|
+
const prospects = [];
|
|
567
|
+
const crmKeys = crmContactKeys(snapshot);
|
|
568
|
+
const runSeen = new Set(seen);
|
|
569
|
+
// Pull successive pages until we have enough fresh candidates or hit the
|
|
570
|
+
// bounded scan ceiling. Page size follows the remaining target, so advancing
|
|
571
|
+
// a cursor never silently discards unused candidates from the final page.
|
|
572
|
+
while (prospects.length < targetFresh && discovered < scanLimit && !exhausted) {
|
|
573
|
+
// Cursor/offset providers can shrink the final request exactly. Explorium
|
|
574
|
+
// uses page numbers, so keep page_size stable or page boundaries would
|
|
575
|
+
// overlap/skip as the number of fresh candidates changes.
|
|
576
|
+
const requestSize = Math.max(1, Math.min(100, disc.provider === "explorium" ? targetFresh : targetFresh - prospects.length, scanLimit - discovered));
|
|
577
|
+
let page;
|
|
578
|
+
let exploriumPage = null;
|
|
579
|
+
if (disc.provider === "pipe0") {
|
|
580
|
+
const result = await fetchPipe0CrustdataProspectPage({
|
|
581
|
+
apiKey: providerKey("pipe0"), filters, limit: requestSize, cursor: cursor ?? undefined,
|
|
582
|
+
});
|
|
583
|
+
page = result.prospects;
|
|
584
|
+
cursor = result.nextCursor;
|
|
585
|
+
exhausted = result.nextCursor === null;
|
|
586
|
+
}
|
|
587
|
+
else if (disc.provider === "explorium") {
|
|
588
|
+
const pageNumber = offset + 1;
|
|
589
|
+
exploriumPage = pageNumber;
|
|
590
|
+
page = await fetchExploriumProspects({
|
|
591
|
+
apiKey: providerKey("explorium"),
|
|
592
|
+
filters: filters,
|
|
593
|
+
size: requestSize,
|
|
594
|
+
page: pageNumber,
|
|
595
|
+
});
|
|
596
|
+
exhausted = page.length < requestSize;
|
|
597
|
+
}
|
|
598
|
+
else if (disc.provider === "linkedin" || disc.provider === "heyreach") {
|
|
599
|
+
const provider = createLinkedInProvider(disc.provider, providerKey("heyreach"));
|
|
600
|
+
page = await discoverLinkedInProspects(provider, {
|
|
601
|
+
sourceId: listId, max: requestSize, cursor: String(offset),
|
|
602
|
+
});
|
|
603
|
+
offset += page.length;
|
|
604
|
+
exhausted = page.length < requestSize;
|
|
605
|
+
}
|
|
606
|
+
else {
|
|
607
|
+
throw new Error(`enrich acquire: unknown discovery provider "${disc.provider}" (explorium | pipe0 | linkedin).`);
|
|
608
|
+
}
|
|
609
|
+
discovered += page.length;
|
|
610
|
+
progress.items(discovered, scanLimit);
|
|
611
|
+
if (page.length === 0) {
|
|
612
|
+
exhausted = true;
|
|
613
|
+
break;
|
|
614
|
+
}
|
|
615
|
+
let fitted = page;
|
|
616
|
+
if (icp) {
|
|
617
|
+
const threshold = fitThreshold(icp);
|
|
618
|
+
fitted = page
|
|
619
|
+
.map((p) => ({ ...p, fitScore: scoreProspectAgainstIcp(p, icp).score }))
|
|
620
|
+
.filter((p) => (p.fitScore ?? 0) >= threshold);
|
|
621
|
+
}
|
|
622
|
+
qualified += fitted.length;
|
|
623
|
+
const partition = partitionFreshProspects(fitted, crmKeys, runSeen);
|
|
624
|
+
skippedCrm += partition.skippedCrm;
|
|
625
|
+
skippedSeen += partition.skippedSeen;
|
|
626
|
+
const remainingTarget = targetFresh - prospects.length;
|
|
627
|
+
for (const prospect of partition.fresh) {
|
|
628
|
+
if (prospects.length >= targetFresh)
|
|
629
|
+
break;
|
|
630
|
+
prospects.push(prospect);
|
|
631
|
+
for (const key of prospectIdentityKeys(prospect))
|
|
632
|
+
runSeen.add(key);
|
|
633
|
+
}
|
|
634
|
+
progress.note(`${discovered} scanned · ${qualified} qualified · ${prospects.length}/${targetFresh} fresh`);
|
|
635
|
+
// Explorium is page-number based. Only advance after consuming every fresh
|
|
636
|
+
// candidate on that page; otherwise the next run safely re-reads the page
|
|
637
|
+
// and CRM/seen dedupe exposes the remainder instead of losing it.
|
|
638
|
+
if (exploriumPage !== null && partition.fresh.length <= remainingTarget)
|
|
639
|
+
offset = exploriumPage;
|
|
640
|
+
}
|
|
641
|
+
if (discovered === 0 && !resume) {
|
|
485
642
|
console.error(`enrich acquire: ${disc.provider} discovered 0 prospects for this ICP — the plan will be empty. ` +
|
|
486
643
|
"This is usually an over-narrow filter, not an empty market: check the ICP's industry and seniority " +
|
|
487
644
|
"(provider vocab is finicky), widen one constraint, or run `fullstackgtm icp show` to inspect the " +
|
|
488
645
|
"generated discovery filters. (Distinct from dedup: nothing was returned to filter.)");
|
|
489
646
|
}
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
.filter((p) => (p.fitScore ?? 0) >= threshold);
|
|
497
|
-
if (discoveredCount > 0 && prospects.length === 0) {
|
|
498
|
-
console.error(`enrich acquire: all ${discoveredCount} discovered prospect(s) scored below the ICP fit threshold ` +
|
|
499
|
-
`(${fitThreshold(icp)}) — none qualified. Loosen the ICP persona or lower scoring.threshold.`);
|
|
500
|
-
}
|
|
501
|
-
}
|
|
502
|
-
// 2.5 Pre-email dedup — drop prospects already in the CRM (name+domain match
|
|
503
|
-
// vs the snapshot) or already processed in a prior run (the seen cache),
|
|
504
|
-
// BEFORE paying pipe0 for emails. The email-level dedup (buildAcquirePlan)
|
|
505
|
-
// and apply-time resolve-first remain the precise backstop.
|
|
506
|
-
const { fresh, skippedCrm, skippedSeen } = partitionFreshProspects(prospects, crmContactKeys(snapshot), seen);
|
|
507
|
-
prospects = fresh;
|
|
647
|
+
if (icp && discovered > 0 && qualified === 0) {
|
|
648
|
+
console.error(`enrich acquire: all ${discovered} discovered prospect(s) scored below the ICP fit threshold ` +
|
|
649
|
+
`(${fitThreshold(icp)}) — none qualified. Loosen the ICP persona or lower scoring.threshold.`);
|
|
650
|
+
}
|
|
651
|
+
progress.stage(ACQUIRE_STAGES[1], 1, ACQUIRE_STAGES.length);
|
|
652
|
+
progress.note("checking work-email requirements");
|
|
508
653
|
// 3. Resolve real work emails. Triggered either when email IS the dedupe key
|
|
509
654
|
// (Explorium's email is hashed, Crustdata returns none) or when a source
|
|
510
655
|
// that keys on something else opts in via `resolveEmailsWith: "pipe0"`
|
|
@@ -515,10 +660,16 @@ async function acquireFromApi(config, source, rest, icp, snapshot, seen) {
|
|
|
515
660
|
// names. Resolve domains first (pipe0 company:identity) so resolution can
|
|
516
661
|
// actually land — without it, name-only resolution fails for every lead.
|
|
517
662
|
if (prospects.some((p) => !p.companyDomain && p.companyName)) {
|
|
518
|
-
|
|
663
|
+
progress.note(`resolving company domains for ${prospects.length} candidate(s)`);
|
|
664
|
+
const resolved = await pipe0ResolveCompanyDomains({ apiKey: providerKey("pipe0"), prospects });
|
|
665
|
+
prospects.splice(0, prospects.length, ...resolved);
|
|
519
666
|
}
|
|
520
|
-
|
|
667
|
+
progress.note(`resolving work emails for ${prospects.length} candidate(s)`);
|
|
668
|
+
const resolved = await pipe0ResolveWorkEmails({ apiKey: providerKey("pipe0"), prospects });
|
|
669
|
+
prospects.splice(0, prospects.length, ...resolved);
|
|
521
670
|
}
|
|
671
|
+
progress.items(prospects.length, prospects.length);
|
|
672
|
+
progress.note(`${prospects.length} candidate(s) ready for planning`);
|
|
522
673
|
const processedKeys = prospects.flatMap(prospectIdentityKeys);
|
|
523
674
|
const records = prospects
|
|
524
675
|
.map((p) => {
|
|
@@ -531,7 +682,46 @@ async function acquireFromApi(config, source, rest, icp, snapshot, seen) {
|
|
|
531
682
|
};
|
|
532
683
|
})
|
|
533
684
|
.filter((record) => Boolean(record.keys[matchKey]));
|
|
534
|
-
|
|
685
|
+
const discovery = { queryFingerprint, cursor, offset, exhausted };
|
|
686
|
+
return {
|
|
687
|
+
records,
|
|
688
|
+
skippedCrm,
|
|
689
|
+
skippedSeen,
|
|
690
|
+
processedKeys,
|
|
691
|
+
telemetry: {
|
|
692
|
+
funnel: {
|
|
693
|
+
discovered,
|
|
694
|
+
qualified,
|
|
695
|
+
skippedCrm,
|
|
696
|
+
skippedSeen,
|
|
697
|
+
resolved: records.length,
|
|
698
|
+
proposed: 0,
|
|
699
|
+
withheldByMeter: 0,
|
|
700
|
+
},
|
|
701
|
+
discovery,
|
|
702
|
+
},
|
|
703
|
+
checkpointKey,
|
|
704
|
+
hostedVersion,
|
|
705
|
+
};
|
|
706
|
+
}
|
|
707
|
+
/** Commit continuation only after the saved plan/run is durable. */
|
|
708
|
+
async function persistAcquireCheckpoint(key, discovery, hostedVersion) {
|
|
709
|
+
const store = createFileAcquireCheckpointStore();
|
|
710
|
+
await store.put(key, {
|
|
711
|
+
cursor: discovery.cursor,
|
|
712
|
+
offset: discovery.offset,
|
|
713
|
+
exhausted: discovery.exhausted,
|
|
714
|
+
});
|
|
715
|
+
const hostedWrite = await writeHostedAcquireCheckpoint(acquireCheckpointId(key), discovery, hostedVersion);
|
|
716
|
+
if (hostedWrite.status === "conflict" && hostedWrite.current) {
|
|
717
|
+
const current = hostedWrite.current.checkpoint;
|
|
718
|
+
await store.put(key, { cursor: current.cursor, offset: current.offset, exhausted: current.exhausted }, new Date(hostedWrite.current.updatedAt));
|
|
719
|
+
console.error("Acquisition checkpoint advanced concurrently in the hosted workspace; " +
|
|
720
|
+
"the newer hosted continuation was kept locally for the next run.");
|
|
721
|
+
}
|
|
722
|
+
else if (hostedWrite.status === "unavailable") {
|
|
723
|
+
console.error(`Hosted checkpoint sync unavailable; local continuation retained (${hostedWrite.reason}).`);
|
|
724
|
+
}
|
|
535
725
|
}
|
|
536
726
|
/**
|
|
537
727
|
* Rich-only fuel gauge under the acquire meter line: one bar per configured
|
|
@@ -566,6 +756,10 @@ function formatAcquireMeter(headroom, costPerRecord) {
|
|
|
566
756
|
`Spend left: ${money(headroom.spendUsd.day)}/day, ${money(headroom.spendUsd.month)}/month ` +
|
|
567
757
|
`(≈$${costPerRecord.toFixed(2)}/lead).`);
|
|
568
758
|
}
|
|
759
|
+
function hostedRunsUrl() {
|
|
760
|
+
const baseUrl = getCredential("broker")?.baseUrl?.replace(/\/+$/, "");
|
|
761
|
+
return baseUrl ? `${baseUrl}/dashboard/runs` : null;
|
|
762
|
+
}
|
|
569
763
|
function resolveEnrichSource(config, rest) {
|
|
570
764
|
const requested = option(rest, "--source");
|
|
571
765
|
const declared = Object.keys(config.sources);
|
|
@@ -767,6 +961,7 @@ async function enrichStatus(store, rest, configFile) {
|
|
|
767
961
|
startedAt: last.startedAt,
|
|
768
962
|
completedAt: last.completedAt,
|
|
769
963
|
counts: last.counts,
|
|
964
|
+
acquireTelemetry: last.acquireTelemetry,
|
|
770
965
|
planIds: last.planIds,
|
|
771
966
|
},
|
|
772
967
|
interrupted: interrupted.map((run) => ({ runLabel: run.runLabel, cursor: run.cursor })),
|
|
@@ -790,6 +985,16 @@ async function enrichStatus(store, rest, configFile) {
|
|
|
790
985
|
` · ${last.counts.fetched} fetched, ${last.counts.matched} matched, ${last.counts.unmatched} unmatched,` +
|
|
791
986
|
` ${last.counts.ambiguous} ambiguous, ${last.counts.opsEmitted} ops` +
|
|
792
987
|
(last.planIds.length ? ` · plans: ${last.planIds.join(", ")}` : ""));
|
|
988
|
+
if (last.acquireTelemetry) {
|
|
989
|
+
const f = last.acquireTelemetry.funnel;
|
|
990
|
+
const d = last.acquireTelemetry.discovery;
|
|
991
|
+
console.log(` funnel: ${f.discovered} discovered → ${f.qualified} qualified → ` +
|
|
992
|
+
`${f.resolved} resolved → ${f.proposed} proposed` +
|
|
993
|
+
` · skipped ${f.skippedCrm} CRM + ${f.skippedSeen} seen` +
|
|
994
|
+
(f.withheldByMeter ? ` · ${f.withheldByMeter} meter-withheld` : ""));
|
|
995
|
+
console.log(` audience: ${d.exhausted ? "exhausted" : "continuing"}` +
|
|
996
|
+
`${d.cursor ? " · cursor saved" : d.offset ? ` · offset ${d.offset}` : ""}`);
|
|
997
|
+
}
|
|
793
998
|
for (const run of entry.interrupted) {
|
|
794
999
|
console.log(` interrupted: ${run.runLabel} at cursor ${run.cursor ?? "(start)"} — re-run with --save to resume`);
|
|
795
1000
|
}
|
package/dist/cli/fix.js
CHANGED
|
@@ -20,6 +20,7 @@ import { APPLY_STAGES, composeListeners, createProgressEmitter } from "../progre
|
|
|
20
20
|
import { progressReporter } from "../runReport.js";
|
|
21
21
|
import { confirmRequested, connectorFor, isOptionValue, numericOption, option, readSnapshot, repeatedOption, saveRequested } from "./shared.js";
|
|
22
22
|
import { box, colorEnabled, createProgressRenderer, paint } from "./ui.js";
|
|
23
|
+
import { compactPlan, verbosePlanRequested } from "./planOutput.js";
|
|
23
24
|
/**
|
|
24
25
|
* The resolve gate: exit 0 = safe to create, exit 2 = match found (exists or
|
|
25
26
|
* ambiguous — do NOT blind-create), exit 1 = error. Built for sync jobs and
|
|
@@ -101,9 +102,12 @@ async function emitPlan(plan, args) {
|
|
|
101
102
|
if (args.includes("--json")) {
|
|
102
103
|
console.log(JSON.stringify(plan, null, 2));
|
|
103
104
|
}
|
|
104
|
-
else {
|
|
105
|
+
else if (verbosePlanRequested(args)) {
|
|
105
106
|
console.log(patchPlanToMarkdown(plan));
|
|
106
107
|
}
|
|
108
|
+
else {
|
|
109
|
+
console.log(compactPlan(plan, { saved: saveRequested(args) }));
|
|
110
|
+
}
|
|
107
111
|
}
|
|
108
112
|
/**
|
|
109
113
|
* Governed duplicate cleanup: group by a normalized identity key, propose one
|
|
@@ -170,8 +174,7 @@ export async function reassignCommand(args) {
|
|
|
170
174
|
for (const plan of plans) {
|
|
171
175
|
if (store)
|
|
172
176
|
await store.save(plan);
|
|
173
|
-
console.log(
|
|
174
|
-
console.log(` ${plan.summary}`);
|
|
177
|
+
console.log(verbosePlanRequested(args) ? patchPlanToMarkdown(plan) : compactPlan(plan, { saved: Boolean(store) }));
|
|
175
178
|
}
|
|
176
179
|
if (store) {
|
|
177
180
|
console.log(`\nSaved ${plans.length} plan(s). For each: \`fullstackgtm plans show <id>\`, \`fullstackgtm plans approve <id> --operations <ids|all>\`, then \`fullstackgtm apply --plan-id <id> --provider <name>\`.`);
|