fullstackgtm 0.48.0 → 0.50.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. package/CHANGELOG.md +91 -0
  2. package/CONTRIBUTING.md +23 -4
  3. package/DATA-FLOWS.md +8 -2
  4. package/INSTALL_FOR_AGENTS.md +13 -4
  5. package/README.md +53 -1
  6. package/SECURITY.md +27 -3
  7. package/dist/acquireCheckpoint.d.ts +33 -0
  8. package/dist/acquireCheckpoint.js +149 -0
  9. package/dist/acquireLinkedIn.d.ts +2 -0
  10. package/dist/acquireLinkedIn.js +1 -1
  11. package/dist/cli/audit.js +10 -7
  12. package/dist/cli/auth.js +8 -4
  13. package/dist/cli/enrich.js +283 -79
  14. package/dist/cli/fix.js +20 -7
  15. package/dist/cli/help.js +23 -15
  16. package/dist/cli/market.js +180 -2
  17. package/dist/cli/plans.js +154 -9
  18. package/dist/cli/shared.d.ts +3 -1
  19. package/dist/cli/shared.js +2 -2
  20. package/dist/cli/ui.d.ts +10 -5
  21. package/dist/cli/ui.js +18 -5
  22. package/dist/cli.js +1 -1
  23. package/dist/config.d.ts +13 -1
  24. package/dist/config.js +23 -2
  25. package/dist/connectors/linkedin.d.ts +2 -0
  26. package/dist/connectors/linkedin.js +5 -0
  27. package/dist/connectors/prospectSources.d.ts +23 -0
  28. package/dist/connectors/prospectSources.js +25 -14
  29. package/dist/connectors/salesforce.js +12 -5
  30. package/dist/connectors/salesforceAuth.d.ts +9 -0
  31. package/dist/connectors/salesforceAuth.js +47 -5
  32. package/dist/connectors/theirstack.d.ts +0 -19
  33. package/dist/connectors/theirstack.js +3 -10
  34. package/dist/credentials.js +36 -26
  35. package/dist/enrich.d.ts +44 -1
  36. package/dist/hostedAcquireCheckpoint.d.ts +43 -0
  37. package/dist/hostedAcquireCheckpoint.js +129 -0
  38. package/dist/hostedPatchPlan.d.ts +87 -0
  39. package/dist/hostedPatchPlan.js +270 -0
  40. package/dist/icp.js +13 -2
  41. package/dist/index.d.ts +7 -3
  42. package/dist/index.js +6 -2
  43. package/dist/market.d.ts +1 -1
  44. package/dist/market.js +7 -127
  45. package/dist/marketClassify.d.ts +10 -0
  46. package/dist/marketClassify.js +52 -1
  47. package/dist/marketSourcing.js +7 -45
  48. package/dist/mcp.js +67 -115
  49. package/dist/planStore.d.ts +42 -1
  50. package/dist/planStore.js +268 -49
  51. package/dist/progress.d.ts +2 -2
  52. package/dist/progress.js +2 -2
  53. package/dist/providerError.d.ts +21 -0
  54. package/dist/providerError.js +30 -0
  55. package/dist/publicHttp.d.ts +28 -0
  56. package/dist/publicHttp.js +143 -0
  57. package/dist/secureFile.d.ts +15 -0
  58. package/dist/secureFile.js +164 -0
  59. package/dist/types.d.ts +1 -1
  60. package/docs/api.md +53 -2
  61. package/docs/architecture.md +22 -2
  62. package/llms.txt +7 -0
  63. package/package.json +5 -3
  64. package/skills/fullstackgtm/SKILL.md +8 -2
  65. package/src/acquireCheckpoint.ts +186 -0
  66. package/src/acquireLinkedIn.ts +3 -1
  67. package/src/cli/audit.ts +10 -7
  68. package/src/cli/auth.ts +8 -4
  69. package/src/cli/enrich.ts +322 -78
  70. package/src/cli/fix.ts +28 -7
  71. package/src/cli/help.ts +24 -15
  72. package/src/cli/market.ts +181 -2
  73. package/src/cli/plans.ts +159 -9
  74. package/src/cli/shared.ts +2 -1
  75. package/src/cli/ui.ts +19 -9
  76. package/src/cli.ts +1 -1
  77. package/src/config.ts +39 -1
  78. package/src/connectors/linkedin.ts +6 -0
  79. package/src/connectors/prospectSources.ts +50 -15
  80. package/src/connectors/salesforce.ts +12 -5
  81. package/src/connectors/salesforceAuth.ts +46 -6
  82. package/src/connectors/theirstack.ts +3 -10
  83. package/src/credentials.ts +47 -28
  84. package/src/enrich.ts +47 -1
  85. package/src/hostedAcquireCheckpoint.ts +156 -0
  86. package/src/hostedPatchPlan.ts +286 -0
  87. package/src/icp.ts +14 -2
  88. package/src/index.ts +24 -0
  89. package/src/market.ts +7 -111
  90. package/src/marketClassify.ts +61 -1
  91. package/src/marketSourcing.ts +7 -43
  92. package/src/mcp.ts +93 -136
  93. package/src/planStore.ts +322 -57
  94. package/src/progress.ts +2 -2
  95. package/src/providerError.ts +37 -0
  96. package/src/publicHttp.ts +147 -0
  97. package/src/secureFile.ts +169 -0
  98. package/src/types.ts +1 -0
@@ -1,13 +1,17 @@
1
1
  // Extracted verbatim from src/cli.ts (mechanical split, no behavior change).
2
2
  import { existsSync, readFileSync } from "node:fs";
3
+ import { createHash } from "node:crypto";
3
4
  import { resolve } from "node:path";
4
5
  import { getCredential } from "../credentials.js";
5
6
  import { patchPlanToMarkdown } from "../format.js";
6
7
  import { createFilePlanStore } from "../planStore.js";
7
8
  import { buildAcquirePlan, buildEnrichPlan, createFileEnrichRunStore, DEFAULT_STALE_DAYS, ENRICH_CONFIG_FILE_NAME, builtinAcquirePreset, builtinEnrichPreset, enrichRunId, inferIngestObjectType, latestStamps, loadEnrichConfig, parseCsv, resolveCrmField, selectStaleWork, stagedSourceRecords, staleDaysFor } from "../enrich.js";
8
9
  import { loadMeter, remaining } from "../acquireMeter.js";
9
- import { crmContactKeys, fetchExploriumProspects, fetchPipe0CrustdataProspects, partitionFreshProspects, pipe0ResolveCompanyDomains, pipe0ResolveWorkEmails, prospectIdentityKeys } from "../connectors/prospectSources.js";
10
+ import { crmContactKeys, fetchExploriumProspects, fetchPipe0CrustdataProspectPage, partitionFreshProspects, pipe0ResolveCompanyDomains, pipe0ResolveWorkEmails, prospectIdentityKeys } from "../connectors/prospectSources.js";
10
11
  import { loadSeen, recordSeen } from "../acquireSeen.js";
12
+ import { acquireCheckpointId, createFileAcquireCheckpointStore } from "../acquireCheckpoint.js";
13
+ import { readHostedAcquireCheckpoint, writeHostedAcquireCheckpoint } from "../hostedAcquireCheckpoint.js";
14
+ import { uploadHostedPatchPlan } from "../hostedPatchPlan.js";
11
15
  import { progressReporter, reportCounts, reportEvent } from "../runReport.js";
12
16
  import { ACQUIRE_STAGES, composeListeners, createProgressEmitter } from "../progress.js";
13
17
  import { createLinkedInProvider, discoverLinkedInProspects } from "../acquireLinkedIn.js";
@@ -37,7 +41,7 @@ enrich append [--source apollo] [--objects companies,contacts] [--save] [--conf
37
41
  enrich refresh [--source apollo] [--stale-days <n>] [--save] [--config <path>]
38
42
  [source options] [--run-label <label>] [--json]
39
43
  enrich ingest <file.csv|payload.json|spool.jsonl|spool-dir> --source clay [--run-label <label>] [--objects companies|contacts] [--config <path>]
40
- enrich acquire [--source <id>] [--max <n>] [--list <id>] [--assign-owner <id>] [--save] [--config <path>] [--json]
44
+ enrich acquire [--source <id>] [--max <new-leads>] [--scan-limit <candidates>] [--list <id>] [--assign-owner <id>] [--save] [--config <path>] [--json]
41
45
  enrich status [--runs] [--source <id>] [--config <path>] [--json]
42
46
 
43
47
  acquire creates NET-NEW leads from a staged prospect list (ingest first):
@@ -47,6 +51,9 @@ it dedupes each sourced row against the CRM, skips matches and ambiguities
47
51
  remaining budget (records + spend, per day and per month; whichever is hit
48
52
  first). Approval-gated like every write: \`--save\` → plans approve → apply.
49
53
  The meter is charged only when a create actually lands at apply.
54
+ \`--max\` controls the desired net-new plan size; \`--scan-limit\` separately
55
+ bounds raw candidates scanned after ICP and dedupe filtering. Continuation is
56
+ persisted per query so scheduled runs advance instead of rereading page one.
50
57
  Discovery sources (zero-config presets): explorium | pipe0 (ICP-driven search),
51
58
  and linkedin (reads a HeyReach lead list — pass --list <id>, key on the LinkedIn
52
59
  URL, $0/record; \`login heyreach\` or HEYREACH_API_KEY).
@@ -125,7 +132,7 @@ are phase 2. Recurring execution is the scheduler's job; enrich has no cron.`);
125
132
  const today = new Date().toISOString().slice(0, 10);
126
133
  // Prospects come from an API source (net-new discovery, e.g. Explorium +
127
134
  // pipe0 work-email) or a staged ingest list (Clay/CSV/webhook).
128
- const snapshot = await readSnapshot(rest);
135
+ const snapshot = await readSnapshot(rest, undefined, { persistProgress: true });
129
136
  const icp = loadIcp(rest);
130
137
  if (sourceConfig.kind === "api" && !icp) {
131
138
  console.error("⚠ No ICP loaded — discovery will use the raw discovery.filters and skip fit scoring. " +
@@ -139,16 +146,38 @@ are phase 2. Recurring execution is the scheduler's job; enrich has no cron.`);
139
146
  "acquire writes it on every new contact it creates, so coverage grows automatically.");
140
147
  }
141
148
  const seen = loadSeen();
149
+ const now = new Date();
150
+ const costPerRecord = config.acquire.costPerRecord?.[source] ?? 0;
151
+ const headroom = remaining(loadMeter(now), config.acquire.budget, costPerRecord, now);
152
+ const explicitMax = numericOption(rest, "--max");
153
+ const desiredCreates = explicitMax ?? config.acquire.discovery?.[source]?.size ?? 25;
154
+ const cap = headroom.maxRecords === null ? desiredCreates : Math.min(headroom.maxRecords, desiredCreates);
142
155
  let records;
143
156
  let apiSkippedCrm = 0;
144
157
  let apiSkippedSeen = 0;
145
158
  let apiProcessedKeys = [];
159
+ let acquireTelemetry;
160
+ let acquireCheckpointSync;
161
+ const renderer = createProgressRenderer(ACQUIRE_STAGES);
162
+ const acquireProgress = createProgressEmitter(composeListeners(renderer.listener, progressReporter()));
146
163
  if (sourceConfig.kind === "api") {
147
- const api = await acquireFromApi(config, source, rest, icp, snapshot, seen);
164
+ const priorRun = await store.latest({ source, mode: "acquire" });
165
+ acquireProgress.stage(ACQUIRE_STAGES[0], 0, ACQUIRE_STAGES.length);
166
+ acquireProgress.note("loading saved audience position");
167
+ let api;
168
+ try {
169
+ api = await acquireFromApi(config, source, rest, icp, snapshot, seen, priorRun, cap, save, acquireProgress);
170
+ }
171
+ catch (error) {
172
+ renderer.done({ persist: true });
173
+ throw error;
174
+ }
148
175
  records = api.records;
149
176
  apiSkippedCrm = api.skippedCrm;
150
177
  apiSkippedSeen = api.skippedSeen;
151
178
  apiProcessedKeys = api.processedKeys;
179
+ acquireTelemetry = api.telemetry;
180
+ acquireCheckpointSync = { key: api.checkpointKey, hostedVersion: api.hostedVersion };
152
181
  }
153
182
  else {
154
183
  const stagedLabel = option(rest, "--staged-run");
@@ -162,17 +191,13 @@ are phase 2. Recurring execution is the scheduler's job; enrich has no cron.`);
162
191
  }
163
192
  // Meter: how many MORE leads may we create right now? --max is an
164
193
  // additional per-run ceiling, never a way to exceed the budget.
165
- const now = new Date();
166
- const costPerRecord = config.acquire.costPerRecord?.[source] ?? 0;
167
194
  if (apiSkippedCrm || apiSkippedSeen) {
168
195
  console.error(`Pre-email dedup: skipped ${apiSkippedCrm} already-in-CRM + ${apiSkippedSeen} seen before paying for emails ` +
169
196
  `(≈$${((apiSkippedCrm + apiSkippedSeen) * costPerRecord).toFixed(2)} enrichment saved).`);
170
197
  }
171
- const headroom = remaining(loadMeter(now), config.acquire.budget, costPerRecord, now);
172
- const explicitMax = numericOption(rest, "--max");
173
- let cap = headroom.maxRecords;
174
- if (explicitMax !== undefined)
175
- cap = cap === null ? explicitMax : Math.min(cap, explicitMax);
198
+ // `cap` is the desired create count bounded by the persistent meter. Raw
199
+ // discovery scanning has its own --scan-limit and may cross many duplicate-
200
+ // heavy pages to fill this target.
176
201
  // Assignment: never create an ownerless lead. An explicit `acquire.assign`
177
202
  // policy wins; a `--assign-owner <id>` flag is a quick fixed override;
178
203
  // otherwise, when the portal has exactly one active owner, default every
@@ -202,11 +227,10 @@ are phase 2. Recurring execution is the scheduler's job; enrich has no cron.`);
202
227
  // piped) and, via the composed reporter, heartbeat to a paired hosted app.
203
228
  // The meter reading (creates vs headroom + budget burn) rides the same
204
229
  // emitter, feeding the dashboard's gauge without printing anything new.
205
- const renderer = createProgressRenderer(ACQUIRE_STAGES);
206
- const acquireProgress = createProgressEmitter(composeListeners(renderer.listener, progressReporter()));
207
230
  let result;
208
231
  try {
209
- acquireProgress.stage(ACQUIRE_STAGES[0], 0, ACQUIRE_STAGES.length);
232
+ acquireProgress.stage(ACQUIRE_STAGES[2], 2, ACQUIRE_STAGES.length);
233
+ acquireProgress.note("building governed create plan");
210
234
  if (config.acquire.budget?.records?.perDay && headroom.records.day !== null) {
211
235
  acquireProgress.meter(config.acquire.budget.records.perDay - headroom.records.day, config.acquire.budget.records.perDay, "records/day");
212
236
  }
@@ -221,7 +245,7 @@ are phase 2. Recurring execution is the scheduler's job; enrich has no cron.`);
221
245
  });
222
246
  }
223
247
  finally {
224
- renderer.done();
248
+ renderer.done({ persist: true });
225
249
  }
226
250
  if (result.counts.unassigned > 0 && result.counts.created > 0) {
227
251
  console.error(`⚠ ${result.counts.unassigned}/${result.counts.created} proposed lead(s) have no owner ` +
@@ -251,32 +275,69 @@ are phase 2. Recurring execution is the scheduler's job; enrich has no cron.`);
251
275
  }
252
276
  return;
253
277
  }
254
- const run = await openEnrichRun(store, source, "append", option(rest, "--run-label"), today);
278
+ const run = await openEnrichRun(store, source, "acquire", option(rest, "--run-label"), today);
255
279
  const planIds = [];
280
+ let hostedPlanUrl = null;
256
281
  if (result.plan.operations.length > 0) {
257
- await createFilePlanStore().save(result.plan);
282
+ const storedPlan = await createFilePlanStore().save(result.plan);
258
283
  planIds.push(result.plan.id);
259
284
  reportEvent("plan_saved", result.plan.id);
285
+ const mirror = await uploadHostedPatchPlan(storedPlan);
286
+ if (mirror.status === "saved") {
287
+ hostedPlanUrl = mirror.state.url;
288
+ reportEvent("plan_mirrored", mirror.state.patchPlanId);
289
+ }
290
+ else if (mirror.status === "unavailable" || mirror.status === "conflict") {
291
+ console.error(`Hosted plan sync pending: ${mirror.reason}. The local signed plan remains authoritative.`);
292
+ }
293
+ }
294
+ if (acquireTelemetry) {
295
+ acquireTelemetry.funnel.proposed = result.counts.created;
296
+ acquireTelemetry.funnel.withheldByMeter = result.counts.withheldByMeter;
260
297
  }
261
298
  await store.update({
262
299
  ...run,
263
300
  completedAt: new Date().toISOString(),
264
- cursor: null,
301
+ cursor: acquireTelemetry?.discovery.cursor ?? null,
302
+ counts: acquireTelemetry
303
+ ? {
304
+ fetched: acquireTelemetry.funnel.discovered,
305
+ matched: acquireTelemetry.funnel.skippedCrm + acquireTelemetry.funnel.skippedSeen,
306
+ unmatched: acquireTelemetry.funnel.resolved,
307
+ ambiguous: result.counts.ambiguous,
308
+ opsEmitted: acquireTelemetry.funnel.proposed,
309
+ }
310
+ : run.counts,
311
+ acquireTelemetry,
265
312
  planIds: [...(run.planIds ?? []), ...planIds],
266
313
  });
267
314
  // Remember everyone we email-resolved this run so the next run skips them
268
315
  // pre-email (cross-run credit saver). Committed (--save) runs only.
269
316
  if (apiProcessedKeys.length > 0)
270
317
  recordSeen(apiProcessedKeys, now);
271
- console.log(meterLine);
272
- if (gaugeLine)
273
- console.log(gaugeLine);
318
+ if (acquireTelemetry && acquireCheckpointSync) {
319
+ await persistAcquireCheckpoint(acquireCheckpointSync.key, acquireTelemetry.discovery, acquireCheckpointSync.hostedVersion);
320
+ }
274
321
  if (planIds.length > 0) {
275
- console.log(`Saved plan ${result.plan.id} — ${result.counts.created} net-new lead(s), est. $${result.estCostUsd.toFixed(2)}. ` +
276
- `Review \`fullstackgtm plans show ${result.plan.id}\`, approve \`fullstackgtm plans approve ${result.plan.id} --operations all\`, ` +
277
- `then \`fullstackgtm apply --plan-id ${result.plan.id} --provider hubspot\`. The meter is charged only when a create lands at apply.`);
322
+ const hostedRuns = hostedRunsUrl();
323
+ console.log(`\nSaved plan ${result.plan.id}`);
324
+ console.log(` Leads ${result.counts.created} net-new`);
325
+ console.log(` Est. cost $${result.estCostUsd.toFixed(2)}`);
326
+ console.log(` Budget ${gaugeLine ?? meterLine}`);
327
+ if (hostedPlanUrl)
328
+ console.log(` Plan ${hostedPlanUrl}`);
329
+ if (hostedRuns)
330
+ console.log(` Observe ${hostedRuns}`);
331
+ console.log("\nNext steps");
332
+ console.log(` Review fullstackgtm plans show ${result.plan.id}`);
333
+ console.log(` Approve fullstackgtm plans approve ${result.plan.id} --operations all`);
334
+ console.log(` Apply fullstackgtm apply --plan-id ${result.plan.id} --provider hubspot`);
335
+ console.log("\nThe meter is charged only when a create lands at apply.");
278
336
  }
279
337
  else {
338
+ console.log(meterLine);
339
+ if (gaugeLine)
340
+ console.log(gaugeLine);
280
341
  console.log("No net-new leads to create (everything sourced already matched, or was withheld by the meter).");
281
342
  }
282
343
  return;
@@ -442,69 +503,152 @@ function formatEnrichCounts(counts, ambiguities) {
442
503
  * pipe0 (when resolveEmailsWith=pipe0) fills real work emails. Only rows that
443
504
  * carry the dedupe key (email) survive — you cannot resolve-first without it.
444
505
  */
445
- async function acquireFromApi(config, source, rest, icp, snapshot, seen) {
506
+ async function acquireFromApi(config, source, rest, icp, snapshot, seen, priorRun, targetFresh, persistCheckpoint, progress) {
446
507
  const acquire = config.acquire;
447
508
  const disc = acquire.discovery?.[source];
448
509
  if (!disc) {
449
510
  throw new Error(`enrich acquire: api source "${source}" needs an "acquire.discovery.${source}" config (provider + size).`);
450
511
  }
451
512
  const matchKey = acquire.create.contact?.matchKey ?? "email";
452
- const maxOverride = numericOption(rest, "--max");
453
- const size = maxOverride !== undefined ? Math.min(maxOverride, disc.size ?? 25) : disc.size ?? 25;
454
- // 1. Discover. Filters come from the ICP when one is loaded (the whole point
455
- // targeted, not random); otherwise from the hand-written disc.filters.
456
- let prospects;
457
- if (disc.provider === "explorium") {
458
- const filters = icp
459
- ? icpToExploriumFilters(icp)
460
- : (disc.filters ?? {});
461
- prospects = await fetchExploriumProspects({ apiKey: providerKey("explorium"), filters, size });
462
- }
463
- else if (disc.provider === "pipe0") {
464
- const filters = icp ? icpToCrustdataFilters(icp) : (disc.filters ?? {});
465
- prospects = await fetchPipe0CrustdataProspects({ apiKey: providerKey("pipe0"), filters, limit: size });
466
- }
467
- else if (disc.provider === "linkedin" || disc.provider === "heyreach") {
468
- // LinkedIn reads a pre-populated lead list (not an ICP-driven query); the ICP
469
- // scores the pulled list below. List id: disc.listId or --list <id>.
470
- const listId = disc.listId ?? option(rest, "--list") ?? undefined;
471
- if (!listId) {
472
- throw new Error("enrich acquire --source linkedin needs a HeyReach lead-list id: pass --list <id> or set acquire.discovery.linkedin.listId.");
473
- }
474
- const provider = createLinkedInProvider(disc.provider, providerKey("heyreach"));
475
- prospects = await discoverLinkedInProspects(provider, { sourceId: listId, max: size });
476
- }
477
- else {
478
- throw new Error(`enrich acquire: unknown discovery provider "${disc.provider}" (explorium | pipe0 | linkedin).`);
479
- }
480
- // Surface a zero-discovery result LOUDLY rather than emit a silent empty plan.
481
- // A provider that returns 0 with no error usually means an over-narrow ICP
482
- // filter (most often the industry/seniority vocab) — not "no market exists".
483
- const discoveredCount = prospects.length;
484
- if (discoveredCount === 0) {
513
+ const explicitScanLimit = numericOption(rest, "--scan-limit");
514
+ const scanLimit = Math.max(targetFresh, Math.min(explicitScanLimit ?? disc.scanLimit ?? Math.max(100, targetFresh * 10), 5_000));
515
+ const listId = disc.listId ?? option(rest, "--list") ?? undefined;
516
+ if ((disc.provider === "linkedin" || disc.provider === "heyreach") && !listId) {
517
+ throw new Error("enrich acquire --source linkedin needs a HeyReach lead-list id: pass --list <id> or set acquire.discovery.linkedin.listId.");
518
+ }
519
+ const filters = disc.provider === "explorium"
520
+ ? (icp ? icpToExploriumFilters(icp) : (disc.filters ?? {}))
521
+ : disc.provider === "pipe0"
522
+ ? (icp ? icpToCrustdataFilters(icp) : (disc.filters ?? {}))
523
+ : {};
524
+ const queryFingerprint = createHash("sha256")
525
+ .update(JSON.stringify({ source, provider: disc.provider, listId, filters, icp: icp ?? null }))
526
+ .digest("hex");
527
+ const checkpointKey = {
528
+ provider: disc.provider,
529
+ source,
530
+ listId: listId ?? null,
531
+ queryFingerprint,
532
+ };
533
+ const checkpointStore = createFileAcquireCheckpointStore();
534
+ const localCheckpoint = await checkpointStore.get(checkpointKey);
535
+ const hostedRead = await readHostedAcquireCheckpoint(acquireCheckpointId(checkpointKey));
536
+ const hostedRecord = hostedRead.status === "found" ? hostedRead.record : null;
537
+ const hostedVersion = hostedRecord?.version ?? null;
538
+ // Once paired, hosted state lets another authenticated worker resume this
539
+ // audience. A newer offline local checkpoint is still preserved and pushed
540
+ // with CAS; otherwise hosted wins. Legacy run telemetry is a one-time
541
+ // migration source only when neither dedicated store has this exact key.
542
+ let persisted = localCheckpoint?.continuation;
543
+ if (hostedRecord &&
544
+ (!localCheckpoint || hostedRecord.updatedAt >= Date.parse(localCheckpoint.updatedAt))) {
545
+ persisted = hostedRecord.checkpoint;
546
+ if (persistCheckpoint) {
547
+ await checkpointStore.put(checkpointKey, {
548
+ cursor: hostedRecord.checkpoint.cursor,
549
+ offset: hostedRecord.checkpoint.offset,
550
+ exhausted: hostedRecord.checkpoint.exhausted,
551
+ }, new Date(hostedRecord.updatedAt));
552
+ }
553
+ }
554
+ const prior = priorRun?.acquireTelemetry?.discovery;
555
+ const resume = persisted ?? (prior?.queryFingerprint === queryFingerprint
556
+ ? { cursor: prior.cursor, offset: prior.offset, exhausted: prior.exhausted }
557
+ : undefined);
558
+ let cursor = resume?.cursor ?? null;
559
+ let offset = resume?.offset ?? 0;
560
+ let exhausted = resume?.exhausted ?? false;
561
+ let discovered = 0;
562
+ let qualified = 0;
563
+ let skippedCrm = 0;
564
+ let skippedSeen = 0;
565
+ const prospects = [];
566
+ const crmKeys = crmContactKeys(snapshot);
567
+ const runSeen = new Set(seen);
568
+ // Pull successive pages until we have enough fresh candidates or hit the
569
+ // bounded scan ceiling. Page size follows the remaining target, so advancing
570
+ // a cursor never silently discards unused candidates from the final page.
571
+ while (prospects.length < targetFresh && discovered < scanLimit && !exhausted) {
572
+ // Cursor/offset providers can shrink the final request exactly. Explorium
573
+ // uses page numbers, so keep page_size stable or page boundaries would
574
+ // overlap/skip as the number of fresh candidates changes.
575
+ const requestSize = Math.max(1, Math.min(100, disc.provider === "explorium" ? targetFresh : targetFresh - prospects.length, scanLimit - discovered));
576
+ let page;
577
+ let exploriumPage = null;
578
+ if (disc.provider === "pipe0") {
579
+ const result = await fetchPipe0CrustdataProspectPage({
580
+ apiKey: providerKey("pipe0"), filters, limit: requestSize, cursor: cursor ?? undefined,
581
+ });
582
+ page = result.prospects;
583
+ cursor = result.nextCursor;
584
+ exhausted = result.nextCursor === null;
585
+ }
586
+ else if (disc.provider === "explorium") {
587
+ const pageNumber = offset + 1;
588
+ exploriumPage = pageNumber;
589
+ page = await fetchExploriumProspects({
590
+ apiKey: providerKey("explorium"),
591
+ filters: filters,
592
+ size: requestSize,
593
+ page: pageNumber,
594
+ });
595
+ exhausted = page.length < requestSize;
596
+ }
597
+ else if (disc.provider === "linkedin" || disc.provider === "heyreach") {
598
+ const provider = createLinkedInProvider(disc.provider, providerKey("heyreach"));
599
+ page = await discoverLinkedInProspects(provider, {
600
+ sourceId: listId, max: requestSize, cursor: String(offset),
601
+ });
602
+ offset += page.length;
603
+ exhausted = page.length < requestSize;
604
+ }
605
+ else {
606
+ throw new Error(`enrich acquire: unknown discovery provider "${disc.provider}" (explorium | pipe0 | linkedin).`);
607
+ }
608
+ discovered += page.length;
609
+ progress.items(discovered, scanLimit);
610
+ if (page.length === 0) {
611
+ exhausted = true;
612
+ break;
613
+ }
614
+ let fitted = page;
615
+ if (icp) {
616
+ const threshold = fitThreshold(icp);
617
+ fitted = page
618
+ .map((p) => ({ ...p, fitScore: scoreProspectAgainstIcp(p, icp).score }))
619
+ .filter((p) => (p.fitScore ?? 0) >= threshold);
620
+ }
621
+ qualified += fitted.length;
622
+ const partition = partitionFreshProspects(fitted, crmKeys, runSeen);
623
+ skippedCrm += partition.skippedCrm;
624
+ skippedSeen += partition.skippedSeen;
625
+ const remainingTarget = targetFresh - prospects.length;
626
+ for (const prospect of partition.fresh) {
627
+ if (prospects.length >= targetFresh)
628
+ break;
629
+ prospects.push(prospect);
630
+ for (const key of prospectIdentityKeys(prospect))
631
+ runSeen.add(key);
632
+ }
633
+ progress.note(`${discovered} scanned · ${qualified} qualified · ${prospects.length}/${targetFresh} fresh`);
634
+ // Explorium is page-number based. Only advance after consuming every fresh
635
+ // candidate on that page; otherwise the next run safely re-reads the page
636
+ // and CRM/seen dedupe exposes the remainder instead of losing it.
637
+ if (exploriumPage !== null && partition.fresh.length <= remainingTarget)
638
+ offset = exploriumPage;
639
+ }
640
+ if (discovered === 0 && !resume) {
485
641
  console.error(`enrich acquire: ${disc.provider} discovered 0 prospects for this ICP — the plan will be empty. ` +
486
642
  "This is usually an over-narrow filter, not an empty market: check the ICP's industry and seniority " +
487
643
  "(provider vocab is finicky), widen one constraint, or run `fullstackgtm icp show` to inspect the " +
488
644
  "generated discovery filters. (Distinct from dedup: nothing was returned to filter.)");
489
645
  }
490
- // 2. ICP fit scoring BEFORE paying for emails only above-threshold prospects
491
- // proceed to (credit-spending) email resolution.
492
- if (icp) {
493
- const threshold = fitThreshold(icp);
494
- prospects = prospects
495
- .map((p) => ({ ...p, fitScore: scoreProspectAgainstIcp(p, icp).score }))
496
- .filter((p) => (p.fitScore ?? 0) >= threshold);
497
- if (discoveredCount > 0 && prospects.length === 0) {
498
- console.error(`enrich acquire: all ${discoveredCount} discovered prospect(s) scored below the ICP fit threshold ` +
499
- `(${fitThreshold(icp)}) — none qualified. Loosen the ICP persona or lower scoring.threshold.`);
500
- }
501
- }
502
- // 2.5 Pre-email dedup — drop prospects already in the CRM (name+domain match
503
- // vs the snapshot) or already processed in a prior run (the seen cache),
504
- // BEFORE paying pipe0 for emails. The email-level dedup (buildAcquirePlan)
505
- // and apply-time resolve-first remain the precise backstop.
506
- const { fresh, skippedCrm, skippedSeen } = partitionFreshProspects(prospects, crmContactKeys(snapshot), seen);
507
- prospects = fresh;
646
+ if (icp && discovered > 0 && qualified === 0) {
647
+ console.error(`enrich acquire: all ${discovered} discovered prospect(s) scored below the ICP fit threshold ` +
648
+ `(${fitThreshold(icp)}) — none qualified. Loosen the ICP persona or lower scoring.threshold.`);
649
+ }
650
+ progress.stage(ACQUIRE_STAGES[1], 1, ACQUIRE_STAGES.length);
651
+ progress.note("checking work-email requirements");
508
652
  // 3. Resolve real work emails. Triggered either when email IS the dedupe key
509
653
  // (Explorium's email is hashed, Crustdata returns none) or when a source
510
654
  // that keys on something else opts in via `resolveEmailsWith: "pipe0"`
@@ -515,10 +659,16 @@ async function acquireFromApi(config, source, rest, icp, snapshot, seen) {
515
659
  // names. Resolve domains first (pipe0 company:identity) so resolution can
516
660
  // actually land — without it, name-only resolution fails for every lead.
517
661
  if (prospects.some((p) => !p.companyDomain && p.companyName)) {
518
- prospects = await pipe0ResolveCompanyDomains({ apiKey: providerKey("pipe0"), prospects });
662
+ progress.note(`resolving company domains for ${prospects.length} candidate(s)`);
663
+ const resolved = await pipe0ResolveCompanyDomains({ apiKey: providerKey("pipe0"), prospects });
664
+ prospects.splice(0, prospects.length, ...resolved);
519
665
  }
520
- prospects = await pipe0ResolveWorkEmails({ apiKey: providerKey("pipe0"), prospects });
666
+ progress.note(`resolving work emails for ${prospects.length} candidate(s)`);
667
+ const resolved = await pipe0ResolveWorkEmails({ apiKey: providerKey("pipe0"), prospects });
668
+ prospects.splice(0, prospects.length, ...resolved);
521
669
  }
670
+ progress.items(prospects.length, prospects.length);
671
+ progress.note(`${prospects.length} candidate(s) ready for planning`);
522
672
  const processedKeys = prospects.flatMap(prospectIdentityKeys);
523
673
  const records = prospects
524
674
  .map((p) => {
@@ -531,7 +681,46 @@ async function acquireFromApi(config, source, rest, icp, snapshot, seen) {
531
681
  };
532
682
  })
533
683
  .filter((record) => Boolean(record.keys[matchKey]));
534
- return { records, skippedCrm, skippedSeen, processedKeys };
684
+ const discovery = { queryFingerprint, cursor, offset, exhausted };
685
+ return {
686
+ records,
687
+ skippedCrm,
688
+ skippedSeen,
689
+ processedKeys,
690
+ telemetry: {
691
+ funnel: {
692
+ discovered,
693
+ qualified,
694
+ skippedCrm,
695
+ skippedSeen,
696
+ resolved: records.length,
697
+ proposed: 0,
698
+ withheldByMeter: 0,
699
+ },
700
+ discovery,
701
+ },
702
+ checkpointKey,
703
+ hostedVersion,
704
+ };
705
+ }
706
+ /** Commit continuation only after the saved plan/run is durable. */
707
+ async function persistAcquireCheckpoint(key, discovery, hostedVersion) {
708
+ const store = createFileAcquireCheckpointStore();
709
+ await store.put(key, {
710
+ cursor: discovery.cursor,
711
+ offset: discovery.offset,
712
+ exhausted: discovery.exhausted,
713
+ });
714
+ const hostedWrite = await writeHostedAcquireCheckpoint(acquireCheckpointId(key), discovery, hostedVersion);
715
+ if (hostedWrite.status === "conflict" && hostedWrite.current) {
716
+ const current = hostedWrite.current.checkpoint;
717
+ await store.put(key, { cursor: current.cursor, offset: current.offset, exhausted: current.exhausted }, new Date(hostedWrite.current.updatedAt));
718
+ console.error("Acquisition checkpoint advanced concurrently in the hosted workspace; " +
719
+ "the newer hosted continuation was kept locally for the next run.");
720
+ }
721
+ else if (hostedWrite.status === "unavailable") {
722
+ console.error(`Hosted checkpoint sync unavailable; local continuation retained (${hostedWrite.reason}).`);
723
+ }
535
724
  }
536
725
  /**
537
726
  * Rich-only fuel gauge under the acquire meter line: one bar per configured
@@ -566,6 +755,10 @@ function formatAcquireMeter(headroom, costPerRecord) {
566
755
  `Spend left: ${money(headroom.spendUsd.day)}/day, ${money(headroom.spendUsd.month)}/month ` +
567
756
  `(≈$${costPerRecord.toFixed(2)}/lead).`);
568
757
  }
758
+ function hostedRunsUrl() {
759
+ const baseUrl = getCredential("broker")?.baseUrl?.replace(/\/+$/, "");
760
+ return baseUrl ? `${baseUrl}/dashboard/runs` : null;
761
+ }
569
762
  function resolveEnrichSource(config, rest) {
570
763
  const requested = option(rest, "--source");
571
764
  const declared = Object.keys(config.sources);
@@ -767,6 +960,7 @@ async function enrichStatus(store, rest, configFile) {
767
960
  startedAt: last.startedAt,
768
961
  completedAt: last.completedAt,
769
962
  counts: last.counts,
963
+ acquireTelemetry: last.acquireTelemetry,
770
964
  planIds: last.planIds,
771
965
  },
772
966
  interrupted: interrupted.map((run) => ({ runLabel: run.runLabel, cursor: run.cursor })),
@@ -790,6 +984,16 @@ async function enrichStatus(store, rest, configFile) {
790
984
  ` · ${last.counts.fetched} fetched, ${last.counts.matched} matched, ${last.counts.unmatched} unmatched,` +
791
985
  ` ${last.counts.ambiguous} ambiguous, ${last.counts.opsEmitted} ops` +
792
986
  (last.planIds.length ? ` · plans: ${last.planIds.join(", ")}` : ""));
987
+ if (last.acquireTelemetry) {
988
+ const f = last.acquireTelemetry.funnel;
989
+ const d = last.acquireTelemetry.discovery;
990
+ console.log(` funnel: ${f.discovered} discovered → ${f.qualified} qualified → ` +
991
+ `${f.resolved} resolved → ${f.proposed} proposed` +
992
+ ` · skipped ${f.skippedCrm} CRM + ${f.skippedSeen} seen` +
993
+ (f.withheldByMeter ? ` · ${f.withheldByMeter} meter-withheld` : ""));
994
+ console.log(` audience: ${d.exhausted ? "exhausted" : "continuing"}` +
995
+ `${d.cursor ? " · cursor saved" : d.offset ? ` · offset ${d.offset}` : ""}`);
996
+ }
793
997
  for (const run of entry.interrupted) {
794
998
  console.log(` interrupted: ${run.runLabel} at cursor ${run.cursor ?? "(start)"} — re-run with --save to resume`);
795
999
  }
package/dist/cli/fix.js CHANGED
@@ -2,10 +2,11 @@
2
2
  import { existsSync, readFileSync, writeFileSync } from "node:fs";
3
3
  import { resolve } from "node:path";
4
4
  import { auditSnapshot, defaultPolicy } from "../audit.js";
5
- import { loadConfig, mergePolicy, resolveConfiguredRules } from "../config.js";
5
+ import { loadConfig, mergePolicy, resolveConfiguredRules, rulePackageTrustFromCli } from "../config.js";
6
6
  import { applyPatchPlan } from "../connector.js";
7
7
  import { patchPlanToMarkdown } from "../format.js";
8
8
  import { createFilePlanStore } from "../planStore.js";
9
+ import { verifyApprovalDigests } from "../integrity.js";
9
10
  import { resolveRecord } from "../resolve.js";
10
11
  import { parseAssignmentPolicy } from "../assign.js";
11
12
  import { buildLeadRoutePlan } from "../route.js";
@@ -197,8 +198,9 @@ export async function fixCommand(args) {
197
198
  throw new Error("--min-confidence must be high or low");
198
199
  }
199
200
  const includeCreates = args.includes("--include-creates");
200
- const loaded = loadConfig(option(args, "--config") ?? undefined);
201
- const configured = await resolveConfiguredRules(loaded);
201
+ const explicitConfig = option(args, "--config") ?? undefined;
202
+ const loaded = loadConfig(explicitConfig);
203
+ const configured = await resolveConfiguredRules(loaded, undefined, rulePackageTrustFromCli(args, explicitConfig));
202
204
  const rule = configured.find((candidate) => candidate.id === ruleId);
203
205
  if (!rule) {
204
206
  throw new Error(`Unknown rule: ${ruleId}. Available rules: ${configured.map((r) => r.id).join(", ")}`);
@@ -266,22 +268,33 @@ export async function fixCommand(args) {
266
268
  return;
267
269
  }
268
270
  const connector = await connectorFor(provider, args);
271
+ const claimed = await store.claimApply(plan.id, { provider, source: "fix" });
272
+ const { claimId } = claimed;
273
+ const claimedVerification = verifyApprovalDigests(claimed.stored.plan.operations, claimed.stored.approvedOperationIds, claimed.stored.valueOverrides, claimed.stored.approvalDigests);
274
+ if (!claimedVerification.ok) {
275
+ await store.abortApplyPreflight(plan.id, claimId, "Approval integrity verification failed after the claim and before provider I/O.");
276
+ throw new Error(`Refusing to apply plan ${plan.id}: approval changed while acquiring the apply claim.`);
277
+ }
269
278
  // Live apply board on interactive terminals (stderr; inert otherwise); the
270
279
  // same emitter streams heartbeats to the paired hosted app on long runs.
271
280
  const renderer = createProgressRenderer(APPLY_STAGES);
272
281
  const progress = createProgressEmitter(composeListeners(renderer.listener, progressReporter()));
273
282
  let run;
274
283
  try {
275
- run = await applyPatchPlan(connector, plan, {
276
- approvedOperationIds: approvedIds,
277
- valueOverrides: overrides,
284
+ run = await applyPatchPlan(connector, claimed.stored.plan, {
285
+ approvedOperationIds: claimed.stored.approvedOperationIds,
286
+ valueOverrides: claimed.stored.valueOverrides,
278
287
  progress,
279
288
  });
280
289
  }
290
+ catch (error) {
291
+ await store.markApplyUncertain(plan.id, claimId);
292
+ throw error;
293
+ }
281
294
  finally {
282
295
  renderer.done();
283
296
  }
284
- await store.recordRun(plan.id, run);
297
+ await store.recordRun(plan.id, run, claimId);
285
298
  const counts = { applied: 0, conflict: 0, skipped: 0, failed: 0 };
286
299
  for (const result of run.results)
287
300
  counts[result.status] = (counts[result.status] ?? 0) + 1;