fullstackgtm 0.48.0 → 0.50.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +91 -0
- package/CONTRIBUTING.md +23 -4
- package/DATA-FLOWS.md +8 -2
- package/INSTALL_FOR_AGENTS.md +13 -4
- package/README.md +53 -1
- package/SECURITY.md +27 -3
- package/dist/acquireCheckpoint.d.ts +33 -0
- package/dist/acquireCheckpoint.js +149 -0
- package/dist/acquireLinkedIn.d.ts +2 -0
- package/dist/acquireLinkedIn.js +1 -1
- package/dist/cli/audit.js +10 -7
- package/dist/cli/auth.js +8 -4
- package/dist/cli/enrich.js +283 -79
- package/dist/cli/fix.js +20 -7
- package/dist/cli/help.js +23 -15
- package/dist/cli/market.js +180 -2
- package/dist/cli/plans.js +154 -9
- package/dist/cli/shared.d.ts +3 -1
- package/dist/cli/shared.js +2 -2
- package/dist/cli/ui.d.ts +10 -5
- package/dist/cli/ui.js +18 -5
- package/dist/cli.js +1 -1
- package/dist/config.d.ts +13 -1
- package/dist/config.js +23 -2
- package/dist/connectors/linkedin.d.ts +2 -0
- package/dist/connectors/linkedin.js +5 -0
- package/dist/connectors/prospectSources.d.ts +23 -0
- package/dist/connectors/prospectSources.js +25 -14
- package/dist/connectors/salesforce.js +12 -5
- package/dist/connectors/salesforceAuth.d.ts +9 -0
- package/dist/connectors/salesforceAuth.js +47 -5
- package/dist/connectors/theirstack.d.ts +0 -19
- package/dist/connectors/theirstack.js +3 -10
- package/dist/credentials.js +36 -26
- package/dist/enrich.d.ts +44 -1
- package/dist/hostedAcquireCheckpoint.d.ts +43 -0
- package/dist/hostedAcquireCheckpoint.js +129 -0
- package/dist/hostedPatchPlan.d.ts +87 -0
- package/dist/hostedPatchPlan.js +270 -0
- package/dist/icp.js +13 -2
- package/dist/index.d.ts +7 -3
- package/dist/index.js +6 -2
- package/dist/market.d.ts +1 -1
- package/dist/market.js +7 -127
- package/dist/marketClassify.d.ts +10 -0
- package/dist/marketClassify.js +52 -1
- package/dist/marketSourcing.js +7 -45
- package/dist/mcp.js +67 -115
- package/dist/planStore.d.ts +42 -1
- package/dist/planStore.js +268 -49
- package/dist/progress.d.ts +2 -2
- package/dist/progress.js +2 -2
- package/dist/providerError.d.ts +21 -0
- package/dist/providerError.js +30 -0
- package/dist/publicHttp.d.ts +28 -0
- package/dist/publicHttp.js +143 -0
- package/dist/secureFile.d.ts +15 -0
- package/dist/secureFile.js +164 -0
- package/dist/types.d.ts +1 -1
- package/docs/api.md +53 -2
- package/docs/architecture.md +22 -2
- package/llms.txt +7 -0
- package/package.json +5 -3
- package/skills/fullstackgtm/SKILL.md +8 -2
- package/src/acquireCheckpoint.ts +186 -0
- package/src/acquireLinkedIn.ts +3 -1
- package/src/cli/audit.ts +10 -7
- package/src/cli/auth.ts +8 -4
- package/src/cli/enrich.ts +322 -78
- package/src/cli/fix.ts +28 -7
- package/src/cli/help.ts +24 -15
- package/src/cli/market.ts +181 -2
- package/src/cli/plans.ts +159 -9
- package/src/cli/shared.ts +2 -1
- package/src/cli/ui.ts +19 -9
- package/src/cli.ts +1 -1
- package/src/config.ts +39 -1
- package/src/connectors/linkedin.ts +6 -0
- package/src/connectors/prospectSources.ts +50 -15
- package/src/connectors/salesforce.ts +12 -5
- package/src/connectors/salesforceAuth.ts +46 -6
- package/src/connectors/theirstack.ts +3 -10
- package/src/credentials.ts +47 -28
- package/src/enrich.ts +47 -1
- package/src/hostedAcquireCheckpoint.ts +156 -0
- package/src/hostedPatchPlan.ts +286 -0
- package/src/icp.ts +14 -2
- package/src/index.ts +24 -0
- package/src/market.ts +7 -111
- package/src/marketClassify.ts +61 -1
- package/src/marketSourcing.ts +7 -43
- package/src/mcp.ts +93 -136
- package/src/planStore.ts +322 -57
- package/src/progress.ts +2 -2
- package/src/providerError.ts +37 -0
- package/src/publicHttp.ts +147 -0
- package/src/secureFile.ts +169 -0
- package/src/types.ts +1 -0
package/dist/cli/enrich.js
CHANGED
|
@@ -1,13 +1,17 @@
|
|
|
1
1
|
// Extracted verbatim from src/cli.ts (mechanical split, no behavior change).
|
|
2
2
|
import { existsSync, readFileSync } from "node:fs";
|
|
3
|
+
import { createHash } from "node:crypto";
|
|
3
4
|
import { resolve } from "node:path";
|
|
4
5
|
import { getCredential } from "../credentials.js";
|
|
5
6
|
import { patchPlanToMarkdown } from "../format.js";
|
|
6
7
|
import { createFilePlanStore } from "../planStore.js";
|
|
7
8
|
import { buildAcquirePlan, buildEnrichPlan, createFileEnrichRunStore, DEFAULT_STALE_DAYS, ENRICH_CONFIG_FILE_NAME, builtinAcquirePreset, builtinEnrichPreset, enrichRunId, inferIngestObjectType, latestStamps, loadEnrichConfig, parseCsv, resolveCrmField, selectStaleWork, stagedSourceRecords, staleDaysFor } from "../enrich.js";
|
|
8
9
|
import { loadMeter, remaining } from "../acquireMeter.js";
|
|
9
|
-
import { crmContactKeys, fetchExploriumProspects,
|
|
10
|
+
import { crmContactKeys, fetchExploriumProspects, fetchPipe0CrustdataProspectPage, partitionFreshProspects, pipe0ResolveCompanyDomains, pipe0ResolveWorkEmails, prospectIdentityKeys } from "../connectors/prospectSources.js";
|
|
10
11
|
import { loadSeen, recordSeen } from "../acquireSeen.js";
|
|
12
|
+
import { acquireCheckpointId, createFileAcquireCheckpointStore } from "../acquireCheckpoint.js";
|
|
13
|
+
import { readHostedAcquireCheckpoint, writeHostedAcquireCheckpoint } from "../hostedAcquireCheckpoint.js";
|
|
14
|
+
import { uploadHostedPatchPlan } from "../hostedPatchPlan.js";
|
|
11
15
|
import { progressReporter, reportCounts, reportEvent } from "../runReport.js";
|
|
12
16
|
import { ACQUIRE_STAGES, composeListeners, createProgressEmitter } from "../progress.js";
|
|
13
17
|
import { createLinkedInProvider, discoverLinkedInProspects } from "../acquireLinkedIn.js";
|
|
@@ -37,7 +41,7 @@ enrich append [--source apollo] [--objects companies,contacts] [--save] [--conf
|
|
|
37
41
|
enrich refresh [--source apollo] [--stale-days <n>] [--save] [--config <path>]
|
|
38
42
|
[source options] [--run-label <label>] [--json]
|
|
39
43
|
enrich ingest <file.csv|payload.json|spool.jsonl|spool-dir> --source clay [--run-label <label>] [--objects companies|contacts] [--config <path>]
|
|
40
|
-
enrich acquire [--source <id>] [--max <
|
|
44
|
+
enrich acquire [--source <id>] [--max <new-leads>] [--scan-limit <candidates>] [--list <id>] [--assign-owner <id>] [--save] [--config <path>] [--json]
|
|
41
45
|
enrich status [--runs] [--source <id>] [--config <path>] [--json]
|
|
42
46
|
|
|
43
47
|
acquire creates NET-NEW leads from a staged prospect list (ingest first):
|
|
@@ -47,6 +51,9 @@ it dedupes each sourced row against the CRM, skips matches and ambiguities
|
|
|
47
51
|
remaining budget (records + spend, per day and per month; whichever is hit
|
|
48
52
|
first). Approval-gated like every write: \`--save\` → plans approve → apply.
|
|
49
53
|
The meter is charged only when a create actually lands at apply.
|
|
54
|
+
\`--max\` controls the desired net-new plan size; \`--scan-limit\` separately
|
|
55
|
+
bounds raw candidates scanned after ICP and dedupe filtering. Continuation is
|
|
56
|
+
persisted per query so scheduled runs advance instead of rereading page one.
|
|
50
57
|
Discovery sources (zero-config presets): explorium | pipe0 (ICP-driven search),
|
|
51
58
|
and linkedin (reads a HeyReach lead list — pass --list <id>, key on the LinkedIn
|
|
52
59
|
URL, $0/record; \`login heyreach\` or HEYREACH_API_KEY).
|
|
@@ -125,7 +132,7 @@ are phase 2. Recurring execution is the scheduler's job; enrich has no cron.`);
|
|
|
125
132
|
const today = new Date().toISOString().slice(0, 10);
|
|
126
133
|
// Prospects come from an API source (net-new discovery, e.g. Explorium +
|
|
127
134
|
// pipe0 work-email) or a staged ingest list (Clay/CSV/webhook).
|
|
128
|
-
const snapshot = await readSnapshot(rest);
|
|
135
|
+
const snapshot = await readSnapshot(rest, undefined, { persistProgress: true });
|
|
129
136
|
const icp = loadIcp(rest);
|
|
130
137
|
if (sourceConfig.kind === "api" && !icp) {
|
|
131
138
|
console.error("⚠ No ICP loaded — discovery will use the raw discovery.filters and skip fit scoring. " +
|
|
@@ -139,16 +146,38 @@ are phase 2. Recurring execution is the scheduler's job; enrich has no cron.`);
|
|
|
139
146
|
"acquire writes it on every new contact it creates, so coverage grows automatically.");
|
|
140
147
|
}
|
|
141
148
|
const seen = loadSeen();
|
|
149
|
+
const now = new Date();
|
|
150
|
+
const costPerRecord = config.acquire.costPerRecord?.[source] ?? 0;
|
|
151
|
+
const headroom = remaining(loadMeter(now), config.acquire.budget, costPerRecord, now);
|
|
152
|
+
const explicitMax = numericOption(rest, "--max");
|
|
153
|
+
const desiredCreates = explicitMax ?? config.acquire.discovery?.[source]?.size ?? 25;
|
|
154
|
+
const cap = headroom.maxRecords === null ? desiredCreates : Math.min(headroom.maxRecords, desiredCreates);
|
|
142
155
|
let records;
|
|
143
156
|
let apiSkippedCrm = 0;
|
|
144
157
|
let apiSkippedSeen = 0;
|
|
145
158
|
let apiProcessedKeys = [];
|
|
159
|
+
let acquireTelemetry;
|
|
160
|
+
let acquireCheckpointSync;
|
|
161
|
+
const renderer = createProgressRenderer(ACQUIRE_STAGES);
|
|
162
|
+
const acquireProgress = createProgressEmitter(composeListeners(renderer.listener, progressReporter()));
|
|
146
163
|
if (sourceConfig.kind === "api") {
|
|
147
|
-
const
|
|
164
|
+
const priorRun = await store.latest({ source, mode: "acquire" });
|
|
165
|
+
acquireProgress.stage(ACQUIRE_STAGES[0], 0, ACQUIRE_STAGES.length);
|
|
166
|
+
acquireProgress.note("loading saved audience position");
|
|
167
|
+
let api;
|
|
168
|
+
try {
|
|
169
|
+
api = await acquireFromApi(config, source, rest, icp, snapshot, seen, priorRun, cap, save, acquireProgress);
|
|
170
|
+
}
|
|
171
|
+
catch (error) {
|
|
172
|
+
renderer.done({ persist: true });
|
|
173
|
+
throw error;
|
|
174
|
+
}
|
|
148
175
|
records = api.records;
|
|
149
176
|
apiSkippedCrm = api.skippedCrm;
|
|
150
177
|
apiSkippedSeen = api.skippedSeen;
|
|
151
178
|
apiProcessedKeys = api.processedKeys;
|
|
179
|
+
acquireTelemetry = api.telemetry;
|
|
180
|
+
acquireCheckpointSync = { key: api.checkpointKey, hostedVersion: api.hostedVersion };
|
|
152
181
|
}
|
|
153
182
|
else {
|
|
154
183
|
const stagedLabel = option(rest, "--staged-run");
|
|
@@ -162,17 +191,13 @@ are phase 2. Recurring execution is the scheduler's job; enrich has no cron.`);
|
|
|
162
191
|
}
|
|
163
192
|
// Meter: how many MORE leads may we create right now? --max is an
|
|
164
193
|
// additional per-run ceiling, never a way to exceed the budget.
|
|
165
|
-
const now = new Date();
|
|
166
|
-
const costPerRecord = config.acquire.costPerRecord?.[source] ?? 0;
|
|
167
194
|
if (apiSkippedCrm || apiSkippedSeen) {
|
|
168
195
|
console.error(`Pre-email dedup: skipped ${apiSkippedCrm} already-in-CRM + ${apiSkippedSeen} seen before paying for emails ` +
|
|
169
196
|
`(≈$${((apiSkippedCrm + apiSkippedSeen) * costPerRecord).toFixed(2)} enrichment saved).`);
|
|
170
197
|
}
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
if (explicitMax !== undefined)
|
|
175
|
-
cap = cap === null ? explicitMax : Math.min(cap, explicitMax);
|
|
198
|
+
// `cap` is the desired create count bounded by the persistent meter. Raw
|
|
199
|
+
// discovery scanning has its own --scan-limit and may cross many duplicate-
|
|
200
|
+
// heavy pages to fill this target.
|
|
176
201
|
// Assignment: never create an ownerless lead. An explicit `acquire.assign`
|
|
177
202
|
// policy wins; a `--assign-owner <id>` flag is a quick fixed override;
|
|
178
203
|
// otherwise, when the portal has exactly one active owner, default every
|
|
@@ -202,11 +227,10 @@ are phase 2. Recurring execution is the scheduler's job; enrich has no cron.`);
|
|
|
202
227
|
// piped) and, via the composed reporter, heartbeat to a paired hosted app.
|
|
203
228
|
// The meter reading (creates vs headroom + budget burn) rides the same
|
|
204
229
|
// emitter, feeding the dashboard's gauge without printing anything new.
|
|
205
|
-
const renderer = createProgressRenderer(ACQUIRE_STAGES);
|
|
206
|
-
const acquireProgress = createProgressEmitter(composeListeners(renderer.listener, progressReporter()));
|
|
207
230
|
let result;
|
|
208
231
|
try {
|
|
209
|
-
acquireProgress.stage(ACQUIRE_STAGES[
|
|
232
|
+
acquireProgress.stage(ACQUIRE_STAGES[2], 2, ACQUIRE_STAGES.length);
|
|
233
|
+
acquireProgress.note("building governed create plan");
|
|
210
234
|
if (config.acquire.budget?.records?.perDay && headroom.records.day !== null) {
|
|
211
235
|
acquireProgress.meter(config.acquire.budget.records.perDay - headroom.records.day, config.acquire.budget.records.perDay, "records/day");
|
|
212
236
|
}
|
|
@@ -221,7 +245,7 @@ are phase 2. Recurring execution is the scheduler's job; enrich has no cron.`);
|
|
|
221
245
|
});
|
|
222
246
|
}
|
|
223
247
|
finally {
|
|
224
|
-
renderer.done();
|
|
248
|
+
renderer.done({ persist: true });
|
|
225
249
|
}
|
|
226
250
|
if (result.counts.unassigned > 0 && result.counts.created > 0) {
|
|
227
251
|
console.error(`⚠ ${result.counts.unassigned}/${result.counts.created} proposed lead(s) have no owner ` +
|
|
@@ -251,32 +275,69 @@ are phase 2. Recurring execution is the scheduler's job; enrich has no cron.`);
|
|
|
251
275
|
}
|
|
252
276
|
return;
|
|
253
277
|
}
|
|
254
|
-
const run = await openEnrichRun(store, source, "
|
|
278
|
+
const run = await openEnrichRun(store, source, "acquire", option(rest, "--run-label"), today);
|
|
255
279
|
const planIds = [];
|
|
280
|
+
let hostedPlanUrl = null;
|
|
256
281
|
if (result.plan.operations.length > 0) {
|
|
257
|
-
await createFilePlanStore().save(result.plan);
|
|
282
|
+
const storedPlan = await createFilePlanStore().save(result.plan);
|
|
258
283
|
planIds.push(result.plan.id);
|
|
259
284
|
reportEvent("plan_saved", result.plan.id);
|
|
285
|
+
const mirror = await uploadHostedPatchPlan(storedPlan);
|
|
286
|
+
if (mirror.status === "saved") {
|
|
287
|
+
hostedPlanUrl = mirror.state.url;
|
|
288
|
+
reportEvent("plan_mirrored", mirror.state.patchPlanId);
|
|
289
|
+
}
|
|
290
|
+
else if (mirror.status === "unavailable" || mirror.status === "conflict") {
|
|
291
|
+
console.error(`Hosted plan sync pending: ${mirror.reason}. The local signed plan remains authoritative.`);
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
if (acquireTelemetry) {
|
|
295
|
+
acquireTelemetry.funnel.proposed = result.counts.created;
|
|
296
|
+
acquireTelemetry.funnel.withheldByMeter = result.counts.withheldByMeter;
|
|
260
297
|
}
|
|
261
298
|
await store.update({
|
|
262
299
|
...run,
|
|
263
300
|
completedAt: new Date().toISOString(),
|
|
264
|
-
cursor: null,
|
|
301
|
+
cursor: acquireTelemetry?.discovery.cursor ?? null,
|
|
302
|
+
counts: acquireTelemetry
|
|
303
|
+
? {
|
|
304
|
+
fetched: acquireTelemetry.funnel.discovered,
|
|
305
|
+
matched: acquireTelemetry.funnel.skippedCrm + acquireTelemetry.funnel.skippedSeen,
|
|
306
|
+
unmatched: acquireTelemetry.funnel.resolved,
|
|
307
|
+
ambiguous: result.counts.ambiguous,
|
|
308
|
+
opsEmitted: acquireTelemetry.funnel.proposed,
|
|
309
|
+
}
|
|
310
|
+
: run.counts,
|
|
311
|
+
acquireTelemetry,
|
|
265
312
|
planIds: [...(run.planIds ?? []), ...planIds],
|
|
266
313
|
});
|
|
267
314
|
// Remember everyone we email-resolved this run so the next run skips them
|
|
268
315
|
// pre-email (cross-run credit saver). Committed (--save) runs only.
|
|
269
316
|
if (apiProcessedKeys.length > 0)
|
|
270
317
|
recordSeen(apiProcessedKeys, now);
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
318
|
+
if (acquireTelemetry && acquireCheckpointSync) {
|
|
319
|
+
await persistAcquireCheckpoint(acquireCheckpointSync.key, acquireTelemetry.discovery, acquireCheckpointSync.hostedVersion);
|
|
320
|
+
}
|
|
274
321
|
if (planIds.length > 0) {
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
322
|
+
const hostedRuns = hostedRunsUrl();
|
|
323
|
+
console.log(`\nSaved plan ${result.plan.id}`);
|
|
324
|
+
console.log(` Leads ${result.counts.created} net-new`);
|
|
325
|
+
console.log(` Est. cost $${result.estCostUsd.toFixed(2)}`);
|
|
326
|
+
console.log(` Budget ${gaugeLine ?? meterLine}`);
|
|
327
|
+
if (hostedPlanUrl)
|
|
328
|
+
console.log(` Plan ${hostedPlanUrl}`);
|
|
329
|
+
if (hostedRuns)
|
|
330
|
+
console.log(` Observe ${hostedRuns}`);
|
|
331
|
+
console.log("\nNext steps");
|
|
332
|
+
console.log(` Review fullstackgtm plans show ${result.plan.id}`);
|
|
333
|
+
console.log(` Approve fullstackgtm plans approve ${result.plan.id} --operations all`);
|
|
334
|
+
console.log(` Apply fullstackgtm apply --plan-id ${result.plan.id} --provider hubspot`);
|
|
335
|
+
console.log("\nThe meter is charged only when a create lands at apply.");
|
|
278
336
|
}
|
|
279
337
|
else {
|
|
338
|
+
console.log(meterLine);
|
|
339
|
+
if (gaugeLine)
|
|
340
|
+
console.log(gaugeLine);
|
|
280
341
|
console.log("No net-new leads to create (everything sourced already matched, or was withheld by the meter).");
|
|
281
342
|
}
|
|
282
343
|
return;
|
|
@@ -442,69 +503,152 @@ function formatEnrichCounts(counts, ambiguities) {
|
|
|
442
503
|
* pipe0 (when resolveEmailsWith=pipe0) fills real work emails. Only rows that
|
|
443
504
|
* carry the dedupe key (email) survive — you cannot resolve-first without it.
|
|
444
505
|
*/
|
|
445
|
-
async function acquireFromApi(config, source, rest, icp, snapshot, seen) {
|
|
506
|
+
async function acquireFromApi(config, source, rest, icp, snapshot, seen, priorRun, targetFresh, persistCheckpoint, progress) {
|
|
446
507
|
const acquire = config.acquire;
|
|
447
508
|
const disc = acquire.discovery?.[source];
|
|
448
509
|
if (!disc) {
|
|
449
510
|
throw new Error(`enrich acquire: api source "${source}" needs an "acquire.discovery.${source}" config (provider + size).`);
|
|
450
511
|
}
|
|
451
512
|
const matchKey = acquire.create.contact?.matchKey ?? "email";
|
|
452
|
-
const
|
|
453
|
-
const
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
//
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
513
|
+
const explicitScanLimit = numericOption(rest, "--scan-limit");
|
|
514
|
+
const scanLimit = Math.max(targetFresh, Math.min(explicitScanLimit ?? disc.scanLimit ?? Math.max(100, targetFresh * 10), 5_000));
|
|
515
|
+
const listId = disc.listId ?? option(rest, "--list") ?? undefined;
|
|
516
|
+
if ((disc.provider === "linkedin" || disc.provider === "heyreach") && !listId) {
|
|
517
|
+
throw new Error("enrich acquire --source linkedin needs a HeyReach lead-list id: pass --list <id> or set acquire.discovery.linkedin.listId.");
|
|
518
|
+
}
|
|
519
|
+
const filters = disc.provider === "explorium"
|
|
520
|
+
? (icp ? icpToExploriumFilters(icp) : (disc.filters ?? {}))
|
|
521
|
+
: disc.provider === "pipe0"
|
|
522
|
+
? (icp ? icpToCrustdataFilters(icp) : (disc.filters ?? {}))
|
|
523
|
+
: {};
|
|
524
|
+
const queryFingerprint = createHash("sha256")
|
|
525
|
+
.update(JSON.stringify({ source, provider: disc.provider, listId, filters, icp: icp ?? null }))
|
|
526
|
+
.digest("hex");
|
|
527
|
+
const checkpointKey = {
|
|
528
|
+
provider: disc.provider,
|
|
529
|
+
source,
|
|
530
|
+
listId: listId ?? null,
|
|
531
|
+
queryFingerprint,
|
|
532
|
+
};
|
|
533
|
+
const checkpointStore = createFileAcquireCheckpointStore();
|
|
534
|
+
const localCheckpoint = await checkpointStore.get(checkpointKey);
|
|
535
|
+
const hostedRead = await readHostedAcquireCheckpoint(acquireCheckpointId(checkpointKey));
|
|
536
|
+
const hostedRecord = hostedRead.status === "found" ? hostedRead.record : null;
|
|
537
|
+
const hostedVersion = hostedRecord?.version ?? null;
|
|
538
|
+
// Once paired, hosted state lets another authenticated worker resume this
|
|
539
|
+
// audience. A newer offline local checkpoint is still preserved and pushed
|
|
540
|
+
// with CAS; otherwise hosted wins. Legacy run telemetry is a one-time
|
|
541
|
+
// migration source only when neither dedicated store has this exact key.
|
|
542
|
+
let persisted = localCheckpoint?.continuation;
|
|
543
|
+
if (hostedRecord &&
|
|
544
|
+
(!localCheckpoint || hostedRecord.updatedAt >= Date.parse(localCheckpoint.updatedAt))) {
|
|
545
|
+
persisted = hostedRecord.checkpoint;
|
|
546
|
+
if (persistCheckpoint) {
|
|
547
|
+
await checkpointStore.put(checkpointKey, {
|
|
548
|
+
cursor: hostedRecord.checkpoint.cursor,
|
|
549
|
+
offset: hostedRecord.checkpoint.offset,
|
|
550
|
+
exhausted: hostedRecord.checkpoint.exhausted,
|
|
551
|
+
}, new Date(hostedRecord.updatedAt));
|
|
552
|
+
}
|
|
553
|
+
}
|
|
554
|
+
const prior = priorRun?.acquireTelemetry?.discovery;
|
|
555
|
+
const resume = persisted ?? (prior?.queryFingerprint === queryFingerprint
|
|
556
|
+
? { cursor: prior.cursor, offset: prior.offset, exhausted: prior.exhausted }
|
|
557
|
+
: undefined);
|
|
558
|
+
let cursor = resume?.cursor ?? null;
|
|
559
|
+
let offset = resume?.offset ?? 0;
|
|
560
|
+
let exhausted = resume?.exhausted ?? false;
|
|
561
|
+
let discovered = 0;
|
|
562
|
+
let qualified = 0;
|
|
563
|
+
let skippedCrm = 0;
|
|
564
|
+
let skippedSeen = 0;
|
|
565
|
+
const prospects = [];
|
|
566
|
+
const crmKeys = crmContactKeys(snapshot);
|
|
567
|
+
const runSeen = new Set(seen);
|
|
568
|
+
// Pull successive pages until we have enough fresh candidates or hit the
|
|
569
|
+
// bounded scan ceiling. Page size follows the remaining target, so advancing
|
|
570
|
+
// a cursor never silently discards unused candidates from the final page.
|
|
571
|
+
while (prospects.length < targetFresh && discovered < scanLimit && !exhausted) {
|
|
572
|
+
// Cursor/offset providers can shrink the final request exactly. Explorium
|
|
573
|
+
// uses page numbers, so keep page_size stable or page boundaries would
|
|
574
|
+
// overlap/skip as the number of fresh candidates changes.
|
|
575
|
+
const requestSize = Math.max(1, Math.min(100, disc.provider === "explorium" ? targetFresh : targetFresh - prospects.length, scanLimit - discovered));
|
|
576
|
+
let page;
|
|
577
|
+
let exploriumPage = null;
|
|
578
|
+
if (disc.provider === "pipe0") {
|
|
579
|
+
const result = await fetchPipe0CrustdataProspectPage({
|
|
580
|
+
apiKey: providerKey("pipe0"), filters, limit: requestSize, cursor: cursor ?? undefined,
|
|
581
|
+
});
|
|
582
|
+
page = result.prospects;
|
|
583
|
+
cursor = result.nextCursor;
|
|
584
|
+
exhausted = result.nextCursor === null;
|
|
585
|
+
}
|
|
586
|
+
else if (disc.provider === "explorium") {
|
|
587
|
+
const pageNumber = offset + 1;
|
|
588
|
+
exploriumPage = pageNumber;
|
|
589
|
+
page = await fetchExploriumProspects({
|
|
590
|
+
apiKey: providerKey("explorium"),
|
|
591
|
+
filters: filters,
|
|
592
|
+
size: requestSize,
|
|
593
|
+
page: pageNumber,
|
|
594
|
+
});
|
|
595
|
+
exhausted = page.length < requestSize;
|
|
596
|
+
}
|
|
597
|
+
else if (disc.provider === "linkedin" || disc.provider === "heyreach") {
|
|
598
|
+
const provider = createLinkedInProvider(disc.provider, providerKey("heyreach"));
|
|
599
|
+
page = await discoverLinkedInProspects(provider, {
|
|
600
|
+
sourceId: listId, max: requestSize, cursor: String(offset),
|
|
601
|
+
});
|
|
602
|
+
offset += page.length;
|
|
603
|
+
exhausted = page.length < requestSize;
|
|
604
|
+
}
|
|
605
|
+
else {
|
|
606
|
+
throw new Error(`enrich acquire: unknown discovery provider "${disc.provider}" (explorium | pipe0 | linkedin).`);
|
|
607
|
+
}
|
|
608
|
+
discovered += page.length;
|
|
609
|
+
progress.items(discovered, scanLimit);
|
|
610
|
+
if (page.length === 0) {
|
|
611
|
+
exhausted = true;
|
|
612
|
+
break;
|
|
613
|
+
}
|
|
614
|
+
let fitted = page;
|
|
615
|
+
if (icp) {
|
|
616
|
+
const threshold = fitThreshold(icp);
|
|
617
|
+
fitted = page
|
|
618
|
+
.map((p) => ({ ...p, fitScore: scoreProspectAgainstIcp(p, icp).score }))
|
|
619
|
+
.filter((p) => (p.fitScore ?? 0) >= threshold);
|
|
620
|
+
}
|
|
621
|
+
qualified += fitted.length;
|
|
622
|
+
const partition = partitionFreshProspects(fitted, crmKeys, runSeen);
|
|
623
|
+
skippedCrm += partition.skippedCrm;
|
|
624
|
+
skippedSeen += partition.skippedSeen;
|
|
625
|
+
const remainingTarget = targetFresh - prospects.length;
|
|
626
|
+
for (const prospect of partition.fresh) {
|
|
627
|
+
if (prospects.length >= targetFresh)
|
|
628
|
+
break;
|
|
629
|
+
prospects.push(prospect);
|
|
630
|
+
for (const key of prospectIdentityKeys(prospect))
|
|
631
|
+
runSeen.add(key);
|
|
632
|
+
}
|
|
633
|
+
progress.note(`${discovered} scanned · ${qualified} qualified · ${prospects.length}/${targetFresh} fresh`);
|
|
634
|
+
// Explorium is page-number based. Only advance after consuming every fresh
|
|
635
|
+
// candidate on that page; otherwise the next run safely re-reads the page
|
|
636
|
+
// and CRM/seen dedupe exposes the remainder instead of losing it.
|
|
637
|
+
if (exploriumPage !== null && partition.fresh.length <= remainingTarget)
|
|
638
|
+
offset = exploriumPage;
|
|
639
|
+
}
|
|
640
|
+
if (discovered === 0 && !resume) {
|
|
485
641
|
console.error(`enrich acquire: ${disc.provider} discovered 0 prospects for this ICP — the plan will be empty. ` +
|
|
486
642
|
"This is usually an over-narrow filter, not an empty market: check the ICP's industry and seniority " +
|
|
487
643
|
"(provider vocab is finicky), widen one constraint, or run `fullstackgtm icp show` to inspect the " +
|
|
488
644
|
"generated discovery filters. (Distinct from dedup: nothing was returned to filter.)");
|
|
489
645
|
}
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
.filter((p) => (p.fitScore ?? 0) >= threshold);
|
|
497
|
-
if (discoveredCount > 0 && prospects.length === 0) {
|
|
498
|
-
console.error(`enrich acquire: all ${discoveredCount} discovered prospect(s) scored below the ICP fit threshold ` +
|
|
499
|
-
`(${fitThreshold(icp)}) — none qualified. Loosen the ICP persona or lower scoring.threshold.`);
|
|
500
|
-
}
|
|
501
|
-
}
|
|
502
|
-
// 2.5 Pre-email dedup — drop prospects already in the CRM (name+domain match
|
|
503
|
-
// vs the snapshot) or already processed in a prior run (the seen cache),
|
|
504
|
-
// BEFORE paying pipe0 for emails. The email-level dedup (buildAcquirePlan)
|
|
505
|
-
// and apply-time resolve-first remain the precise backstop.
|
|
506
|
-
const { fresh, skippedCrm, skippedSeen } = partitionFreshProspects(prospects, crmContactKeys(snapshot), seen);
|
|
507
|
-
prospects = fresh;
|
|
646
|
+
if (icp && discovered > 0 && qualified === 0) {
|
|
647
|
+
console.error(`enrich acquire: all ${discovered} discovered prospect(s) scored below the ICP fit threshold ` +
|
|
648
|
+
`(${fitThreshold(icp)}) — none qualified. Loosen the ICP persona or lower scoring.threshold.`);
|
|
649
|
+
}
|
|
650
|
+
progress.stage(ACQUIRE_STAGES[1], 1, ACQUIRE_STAGES.length);
|
|
651
|
+
progress.note("checking work-email requirements");
|
|
508
652
|
// 3. Resolve real work emails. Triggered either when email IS the dedupe key
|
|
509
653
|
// (Explorium's email is hashed, Crustdata returns none) or when a source
|
|
510
654
|
// that keys on something else opts in via `resolveEmailsWith: "pipe0"`
|
|
@@ -515,10 +659,16 @@ async function acquireFromApi(config, source, rest, icp, snapshot, seen) {
|
|
|
515
659
|
// names. Resolve domains first (pipe0 company:identity) so resolution can
|
|
516
660
|
// actually land — without it, name-only resolution fails for every lead.
|
|
517
661
|
if (prospects.some((p) => !p.companyDomain && p.companyName)) {
|
|
518
|
-
|
|
662
|
+
progress.note(`resolving company domains for ${prospects.length} candidate(s)`);
|
|
663
|
+
const resolved = await pipe0ResolveCompanyDomains({ apiKey: providerKey("pipe0"), prospects });
|
|
664
|
+
prospects.splice(0, prospects.length, ...resolved);
|
|
519
665
|
}
|
|
520
|
-
|
|
666
|
+
progress.note(`resolving work emails for ${prospects.length} candidate(s)`);
|
|
667
|
+
const resolved = await pipe0ResolveWorkEmails({ apiKey: providerKey("pipe0"), prospects });
|
|
668
|
+
prospects.splice(0, prospects.length, ...resolved);
|
|
521
669
|
}
|
|
670
|
+
progress.items(prospects.length, prospects.length);
|
|
671
|
+
progress.note(`${prospects.length} candidate(s) ready for planning`);
|
|
522
672
|
const processedKeys = prospects.flatMap(prospectIdentityKeys);
|
|
523
673
|
const records = prospects
|
|
524
674
|
.map((p) => {
|
|
@@ -531,7 +681,46 @@ async function acquireFromApi(config, source, rest, icp, snapshot, seen) {
|
|
|
531
681
|
};
|
|
532
682
|
})
|
|
533
683
|
.filter((record) => Boolean(record.keys[matchKey]));
|
|
534
|
-
|
|
684
|
+
const discovery = { queryFingerprint, cursor, offset, exhausted };
|
|
685
|
+
return {
|
|
686
|
+
records,
|
|
687
|
+
skippedCrm,
|
|
688
|
+
skippedSeen,
|
|
689
|
+
processedKeys,
|
|
690
|
+
telemetry: {
|
|
691
|
+
funnel: {
|
|
692
|
+
discovered,
|
|
693
|
+
qualified,
|
|
694
|
+
skippedCrm,
|
|
695
|
+
skippedSeen,
|
|
696
|
+
resolved: records.length,
|
|
697
|
+
proposed: 0,
|
|
698
|
+
withheldByMeter: 0,
|
|
699
|
+
},
|
|
700
|
+
discovery,
|
|
701
|
+
},
|
|
702
|
+
checkpointKey,
|
|
703
|
+
hostedVersion,
|
|
704
|
+
};
|
|
705
|
+
}
|
|
706
|
+
/** Commit continuation only after the saved plan/run is durable. */
|
|
707
|
+
async function persistAcquireCheckpoint(key, discovery, hostedVersion) {
|
|
708
|
+
const store = createFileAcquireCheckpointStore();
|
|
709
|
+
await store.put(key, {
|
|
710
|
+
cursor: discovery.cursor,
|
|
711
|
+
offset: discovery.offset,
|
|
712
|
+
exhausted: discovery.exhausted,
|
|
713
|
+
});
|
|
714
|
+
const hostedWrite = await writeHostedAcquireCheckpoint(acquireCheckpointId(key), discovery, hostedVersion);
|
|
715
|
+
if (hostedWrite.status === "conflict" && hostedWrite.current) {
|
|
716
|
+
const current = hostedWrite.current.checkpoint;
|
|
717
|
+
await store.put(key, { cursor: current.cursor, offset: current.offset, exhausted: current.exhausted }, new Date(hostedWrite.current.updatedAt));
|
|
718
|
+
console.error("Acquisition checkpoint advanced concurrently in the hosted workspace; " +
|
|
719
|
+
"the newer hosted continuation was kept locally for the next run.");
|
|
720
|
+
}
|
|
721
|
+
else if (hostedWrite.status === "unavailable") {
|
|
722
|
+
console.error(`Hosted checkpoint sync unavailable; local continuation retained (${hostedWrite.reason}).`);
|
|
723
|
+
}
|
|
535
724
|
}
|
|
536
725
|
/**
|
|
537
726
|
* Rich-only fuel gauge under the acquire meter line: one bar per configured
|
|
@@ -566,6 +755,10 @@ function formatAcquireMeter(headroom, costPerRecord) {
|
|
|
566
755
|
`Spend left: ${money(headroom.spendUsd.day)}/day, ${money(headroom.spendUsd.month)}/month ` +
|
|
567
756
|
`(≈$${costPerRecord.toFixed(2)}/lead).`);
|
|
568
757
|
}
|
|
758
|
+
function hostedRunsUrl() {
|
|
759
|
+
const baseUrl = getCredential("broker")?.baseUrl?.replace(/\/+$/, "");
|
|
760
|
+
return baseUrl ? `${baseUrl}/dashboard/runs` : null;
|
|
761
|
+
}
|
|
569
762
|
function resolveEnrichSource(config, rest) {
|
|
570
763
|
const requested = option(rest, "--source");
|
|
571
764
|
const declared = Object.keys(config.sources);
|
|
@@ -767,6 +960,7 @@ async function enrichStatus(store, rest, configFile) {
|
|
|
767
960
|
startedAt: last.startedAt,
|
|
768
961
|
completedAt: last.completedAt,
|
|
769
962
|
counts: last.counts,
|
|
963
|
+
acquireTelemetry: last.acquireTelemetry,
|
|
770
964
|
planIds: last.planIds,
|
|
771
965
|
},
|
|
772
966
|
interrupted: interrupted.map((run) => ({ runLabel: run.runLabel, cursor: run.cursor })),
|
|
@@ -790,6 +984,16 @@ async function enrichStatus(store, rest, configFile) {
|
|
|
790
984
|
` · ${last.counts.fetched} fetched, ${last.counts.matched} matched, ${last.counts.unmatched} unmatched,` +
|
|
791
985
|
` ${last.counts.ambiguous} ambiguous, ${last.counts.opsEmitted} ops` +
|
|
792
986
|
(last.planIds.length ? ` · plans: ${last.planIds.join(", ")}` : ""));
|
|
987
|
+
if (last.acquireTelemetry) {
|
|
988
|
+
const f = last.acquireTelemetry.funnel;
|
|
989
|
+
const d = last.acquireTelemetry.discovery;
|
|
990
|
+
console.log(` funnel: ${f.discovered} discovered → ${f.qualified} qualified → ` +
|
|
991
|
+
`${f.resolved} resolved → ${f.proposed} proposed` +
|
|
992
|
+
` · skipped ${f.skippedCrm} CRM + ${f.skippedSeen} seen` +
|
|
993
|
+
(f.withheldByMeter ? ` · ${f.withheldByMeter} meter-withheld` : ""));
|
|
994
|
+
console.log(` audience: ${d.exhausted ? "exhausted" : "continuing"}` +
|
|
995
|
+
`${d.cursor ? " · cursor saved" : d.offset ? ` · offset ${d.offset}` : ""}`);
|
|
996
|
+
}
|
|
793
997
|
for (const run of entry.interrupted) {
|
|
794
998
|
console.log(` interrupted: ${run.runLabel} at cursor ${run.cursor ?? "(start)"} — re-run with --save to resume`);
|
|
795
999
|
}
|
package/dist/cli/fix.js
CHANGED
|
@@ -2,10 +2,11 @@
|
|
|
2
2
|
import { existsSync, readFileSync, writeFileSync } from "node:fs";
|
|
3
3
|
import { resolve } from "node:path";
|
|
4
4
|
import { auditSnapshot, defaultPolicy } from "../audit.js";
|
|
5
|
-
import { loadConfig, mergePolicy, resolveConfiguredRules } from "../config.js";
|
|
5
|
+
import { loadConfig, mergePolicy, resolveConfiguredRules, rulePackageTrustFromCli } from "../config.js";
|
|
6
6
|
import { applyPatchPlan } from "../connector.js";
|
|
7
7
|
import { patchPlanToMarkdown } from "../format.js";
|
|
8
8
|
import { createFilePlanStore } from "../planStore.js";
|
|
9
|
+
import { verifyApprovalDigests } from "../integrity.js";
|
|
9
10
|
import { resolveRecord } from "../resolve.js";
|
|
10
11
|
import { parseAssignmentPolicy } from "../assign.js";
|
|
11
12
|
import { buildLeadRoutePlan } from "../route.js";
|
|
@@ -197,8 +198,9 @@ export async function fixCommand(args) {
|
|
|
197
198
|
throw new Error("--min-confidence must be high or low");
|
|
198
199
|
}
|
|
199
200
|
const includeCreates = args.includes("--include-creates");
|
|
200
|
-
const
|
|
201
|
-
const
|
|
201
|
+
const explicitConfig = option(args, "--config") ?? undefined;
|
|
202
|
+
const loaded = loadConfig(explicitConfig);
|
|
203
|
+
const configured = await resolveConfiguredRules(loaded, undefined, rulePackageTrustFromCli(args, explicitConfig));
|
|
202
204
|
const rule = configured.find((candidate) => candidate.id === ruleId);
|
|
203
205
|
if (!rule) {
|
|
204
206
|
throw new Error(`Unknown rule: ${ruleId}. Available rules: ${configured.map((r) => r.id).join(", ")}`);
|
|
@@ -266,22 +268,33 @@ export async function fixCommand(args) {
|
|
|
266
268
|
return;
|
|
267
269
|
}
|
|
268
270
|
const connector = await connectorFor(provider, args);
|
|
271
|
+
const claimed = await store.claimApply(plan.id, { provider, source: "fix" });
|
|
272
|
+
const { claimId } = claimed;
|
|
273
|
+
const claimedVerification = verifyApprovalDigests(claimed.stored.plan.operations, claimed.stored.approvedOperationIds, claimed.stored.valueOverrides, claimed.stored.approvalDigests);
|
|
274
|
+
if (!claimedVerification.ok) {
|
|
275
|
+
await store.abortApplyPreflight(plan.id, claimId, "Approval integrity verification failed after the claim and before provider I/O.");
|
|
276
|
+
throw new Error(`Refusing to apply plan ${plan.id}: approval changed while acquiring the apply claim.`);
|
|
277
|
+
}
|
|
269
278
|
// Live apply board on interactive terminals (stderr; inert otherwise); the
|
|
270
279
|
// same emitter streams heartbeats to the paired hosted app on long runs.
|
|
271
280
|
const renderer = createProgressRenderer(APPLY_STAGES);
|
|
272
281
|
const progress = createProgressEmitter(composeListeners(renderer.listener, progressReporter()));
|
|
273
282
|
let run;
|
|
274
283
|
try {
|
|
275
|
-
run = await applyPatchPlan(connector, plan, {
|
|
276
|
-
approvedOperationIds:
|
|
277
|
-
valueOverrides:
|
|
284
|
+
run = await applyPatchPlan(connector, claimed.stored.plan, {
|
|
285
|
+
approvedOperationIds: claimed.stored.approvedOperationIds,
|
|
286
|
+
valueOverrides: claimed.stored.valueOverrides,
|
|
278
287
|
progress,
|
|
279
288
|
});
|
|
280
289
|
}
|
|
290
|
+
catch (error) {
|
|
291
|
+
await store.markApplyUncertain(plan.id, claimId);
|
|
292
|
+
throw error;
|
|
293
|
+
}
|
|
281
294
|
finally {
|
|
282
295
|
renderer.done();
|
|
283
296
|
}
|
|
284
|
-
await store.recordRun(plan.id, run);
|
|
297
|
+
await store.recordRun(plan.id, run, claimId);
|
|
285
298
|
const counts = { applied: 0, conflict: 0, skipped: 0, failed: 0 };
|
|
286
299
|
for (const result of run.results)
|
|
287
300
|
counts[result.status] = (counts[result.status] ?? 0) + 1;
|