fullstackgtm 0.49.0 → 0.50.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +53 -0
- package/DATA-FLOWS.md +1 -0
- package/README.md +41 -0
- package/dist/acquireCheckpoint.d.ts +33 -0
- package/dist/acquireCheckpoint.js +149 -0
- package/dist/acquireLinkedIn.d.ts +2 -0
- package/dist/acquireLinkedIn.js +1 -1
- package/dist/cli/enrich.js +283 -79
- package/dist/cli/help.js +13 -9
- package/dist/cli/plans.js +100 -6
- package/dist/cli/shared.d.ts +3 -1
- package/dist/cli/shared.js +2 -2
- package/dist/cli/ui.d.ts +10 -5
- package/dist/cli/ui.js +16 -5
- package/dist/connectors/linkedin.d.ts +2 -0
- package/dist/connectors/linkedin.js +5 -0
- package/dist/connectors/prospectSources.d.ts +23 -0
- package/dist/connectors/prospectSources.js +21 -3
- package/dist/enrich.d.ts +44 -1
- package/dist/hostedAcquireCheckpoint.d.ts +43 -0
- package/dist/hostedAcquireCheckpoint.js +129 -0
- package/dist/hostedPatchPlan.d.ts +87 -0
- package/dist/hostedPatchPlan.js +270 -0
- package/dist/icp.js +13 -2
- package/dist/index.d.ts +4 -1
- package/dist/index.js +3 -0
- package/dist/planStore.d.ts +14 -0
- package/dist/planStore.js +73 -0
- package/dist/progress.d.ts +2 -2
- package/dist/progress.js +2 -2
- package/docs/api.md +24 -0
- package/docs/architecture.md +9 -0
- package/package.json +1 -1
- package/skills/fullstackgtm/SKILL.md +1 -1
- package/src/acquireCheckpoint.ts +186 -0
- package/src/acquireLinkedIn.ts +3 -1
- package/src/cli/enrich.ts +322 -78
- package/src/cli/help.ts +14 -9
- package/src/cli/plans.ts +95 -6
- package/src/cli/shared.ts +2 -1
- package/src/cli/ui.ts +18 -9
- package/src/connectors/linkedin.ts +6 -0
- package/src/connectors/prospectSources.ts +46 -4
- package/src/enrich.ts +47 -1
- package/src/hostedAcquireCheckpoint.ts +156 -0
- package/src/hostedPatchPlan.ts +286 -0
- package/src/icp.ts +14 -2
- package/src/index.ts +20 -0
- package/src/planStore.ts +87 -0
- package/src/progress.ts +2 -2
package/dist/cli/plans.js
CHANGED
|
@@ -14,6 +14,7 @@ import { createFilePlanStore } from "../planStore.js";
|
|
|
14
14
|
import { ENRICH_CONFIG_FILE_NAME, loadEnrichConfig } from "../enrich.js";
|
|
15
15
|
import { loadMeter, recordConsumption, remaining } from "../acquireMeter.js";
|
|
16
16
|
import { progressReporter, reportCounts } from "../runReport.js";
|
|
17
|
+
import { claimHostedPlanApply, reconcileHostedPatchPlan, releaseHostedPlanApply, reportHostedPlanLifecycle, uploadHostedPatchPlan } from "../hostedPatchPlan.js";
|
|
17
18
|
import { APPLY_STAGES, composeListeners, createProgressEmitter } from "../progress.js";
|
|
18
19
|
import { connectorFor, isOptionValue, numericOption, option, repeatedOption, selectedRules } from "./shared.js";
|
|
19
20
|
import { colorEnabled, createProgressRenderer, paint, planStatusWord, stylizePlanMarkdown, table, truncateToWidth } from "./ui.js";
|
|
@@ -132,9 +133,20 @@ export async function apply(args) {
|
|
|
132
133
|
let valueOverrides;
|
|
133
134
|
const store = planId ? createFilePlanStore() : null;
|
|
134
135
|
if (planId && store) {
|
|
135
|
-
|
|
136
|
+
let stored = await store.get(planId);
|
|
136
137
|
if (!stored)
|
|
137
138
|
throw new Error(`No stored plan with id ${planId}.`);
|
|
139
|
+
const reconciliation = await reconcileHostedPatchPlan(store, stored);
|
|
140
|
+
if (reconciliation.status === "conflict")
|
|
141
|
+
throw new Error(`Refusing to apply plan ${planId}: ${reconciliation.reason}.`);
|
|
142
|
+
if (reconciliation.status === "updated") {
|
|
143
|
+
stored = reconciliation.stored;
|
|
144
|
+
console.error(`Synced hosted ${reconciliation.remoteStatus} state for ${planId}.`);
|
|
145
|
+
}
|
|
146
|
+
if (stored.status === "applied") {
|
|
147
|
+
console.log(`Plan ${planId} was already applied by another replica; imported its execution receipt. No provider writes were attempted.`);
|
|
148
|
+
return;
|
|
149
|
+
}
|
|
138
150
|
if (stored.status !== "approved") {
|
|
139
151
|
throw new Error(`Plan ${planId} is ${stored.status}; approve operations first with \`fullstackgtm plans approve ${planId} --operations <ids|all>\`.`);
|
|
140
152
|
}
|
|
@@ -211,11 +223,38 @@ export async function apply(args) {
|
|
|
211
223
|
// transmits nothing; a CRM provider writes records. Same governed apply path.
|
|
212
224
|
const connector = channel ? createChannelConnector(channel) : await connectorFor(provider, args);
|
|
213
225
|
let applyClaimId;
|
|
226
|
+
let hostedApplyClaimId;
|
|
214
227
|
if (planId && store) {
|
|
215
228
|
const claimed = await store.claimApply(planId, { provider: provider ?? `channel:${channel}`, source: "cli" });
|
|
216
229
|
applyClaimId = claimed.claimId;
|
|
230
|
+
const hostedClaim = await claimHostedPlanApply(claimed.stored);
|
|
231
|
+
if (hostedClaim.status === "applied") {
|
|
232
|
+
await store.abortApplyPreflight(planId, claimed.claimId, "Another replica completed apply before provider I/O.");
|
|
233
|
+
const refreshed = await store.get(planId);
|
|
234
|
+
if (refreshed)
|
|
235
|
+
await reconcileHostedPatchPlan(store, refreshed);
|
|
236
|
+
console.log(`Plan ${planId} was already applied by another replica. No provider writes were attempted.`);
|
|
237
|
+
return;
|
|
238
|
+
}
|
|
239
|
+
if (hostedClaim.status === "conflict") {
|
|
240
|
+
await store.abortApplyPreflight(planId, claimed.claimId, `Hosted execution claim failed: ${hostedClaim.reason}`);
|
|
241
|
+
throw new Error(`Refusing to apply plan ${planId}: could not coordinate with its hosted replica (${hostedClaim.reason}).`);
|
|
242
|
+
}
|
|
243
|
+
if (hostedClaim.status === "unavailable") {
|
|
244
|
+
console.error(`Hosted replica is unavailable (${hostedClaim.reason}); continuing local-first. Resolve-before-create and provider conflict guards remain active, and the execution receipt will sync on a later check-in.`);
|
|
245
|
+
}
|
|
246
|
+
if (hostedClaim.status === "claimed") {
|
|
247
|
+
hostedApplyClaimId = hostedClaim.claimId;
|
|
248
|
+
await store.recordHostedClaim(planId, claimed.claimId, hostedClaim.claimId);
|
|
249
|
+
}
|
|
217
250
|
const claimedVerification = verifyApprovalDigests(claimed.stored.plan.operations, claimed.stored.approvedOperationIds, claimed.stored.valueOverrides, claimed.stored.approvalDigests);
|
|
218
251
|
if (!claimedVerification.ok) {
|
|
252
|
+
if (hostedApplyClaimId) {
|
|
253
|
+
const released = await releaseHostedPlanApply(claimed.stored, hostedApplyClaimId, "Approval integrity verification failed after claim and before provider I/O.");
|
|
254
|
+
if (released.status === "conflict" || released.status === "unavailable") {
|
|
255
|
+
console.error(`Hosted apply claim may require recovery: ${released.reason}.`);
|
|
256
|
+
}
|
|
257
|
+
}
|
|
219
258
|
await store.abortApplyPreflight(planId, claimed.claimId, "Approval integrity verification failed after the claim and before provider I/O.");
|
|
220
259
|
throw new Error(`Refusing to apply plan ${planId}: approval changed while acquiring the apply claim.`);
|
|
221
260
|
}
|
|
@@ -248,7 +287,12 @@ export async function apply(args) {
|
|
|
248
287
|
renderer.done();
|
|
249
288
|
}
|
|
250
289
|
if (planId && store) {
|
|
251
|
-
await store.recordRun(planId, run, applyClaimId);
|
|
290
|
+
const recorded = await store.recordRun(planId, run, applyClaimId);
|
|
291
|
+
const durableHostedClaimId = recorded.applyAttempts?.find((attempt) => attempt.id === applyClaimId)?.hostedClaimId;
|
|
292
|
+
const mirrored = await reportHostedPlanLifecycle(recorded, { claimId: durableHostedClaimId ?? hostedApplyClaimId });
|
|
293
|
+
if (mirrored.status === "unavailable" || mirrored.status === "conflict") {
|
|
294
|
+
console.error(`Hosted plan status is stale: ${mirrored.reason}. Local apply history remains authoritative.`);
|
|
295
|
+
}
|
|
252
296
|
}
|
|
253
297
|
// Charge the acquire meter for the creates that actually landed.
|
|
254
298
|
if (createOps.length > 0) {
|
|
@@ -353,7 +397,14 @@ export async function plansCommand(args) {
|
|
|
353
397
|
const [subcommand, ...rest] = args;
|
|
354
398
|
if (subcommand === "list" || subcommand === undefined) {
|
|
355
399
|
const status = option(rest, "--status");
|
|
356
|
-
|
|
400
|
+
let plans = await store.list(status ?? undefined);
|
|
401
|
+
const synced = await Promise.all(plans.map(async (stored) => {
|
|
402
|
+
const result = await reconcileHostedPatchPlan(store, stored);
|
|
403
|
+
if (result.status === "updated")
|
|
404
|
+
return result.stored;
|
|
405
|
+
return stored;
|
|
406
|
+
}));
|
|
407
|
+
plans = status ? synced.filter((stored) => stored.status === status) : synced;
|
|
357
408
|
if (rest.includes("--json") || args.includes("--json")) {
|
|
358
409
|
console.log(JSON.stringify(plans.map((stored) => ({
|
|
359
410
|
id: stored.plan.id,
|
|
@@ -407,9 +458,17 @@ export async function plansCommand(args) {
|
|
|
407
458
|
const planId = rest.find((arg) => !arg.startsWith("--"));
|
|
408
459
|
if (!planId)
|
|
409
460
|
throw new Error("Usage: fullstackgtm plans show <planId>");
|
|
410
|
-
|
|
461
|
+
let stored = await store.get(planId);
|
|
411
462
|
if (!stored)
|
|
412
463
|
throw new Error(`No stored plan with id ${planId}.`);
|
|
464
|
+
const reconciliation = await reconcileHostedPatchPlan(store, stored);
|
|
465
|
+
if (reconciliation.status === "updated") {
|
|
466
|
+
stored = reconciliation.stored;
|
|
467
|
+
console.error(`Synced hosted ${reconciliation.remoteStatus} state for ${planId}.`);
|
|
468
|
+
}
|
|
469
|
+
else if (reconciliation.status === "conflict") {
|
|
470
|
+
console.error(`Hosted sync conflict: ${reconciliation.reason}.`);
|
|
471
|
+
}
|
|
413
472
|
if (rest.includes("--json")) {
|
|
414
473
|
console.log(JSON.stringify(stored, null, 2));
|
|
415
474
|
return;
|
|
@@ -418,6 +477,11 @@ export async function plansCommand(args) {
|
|
|
418
477
|
console.log(`Status: ${planStatusWord(stored.status, showPaint)}`);
|
|
419
478
|
console.log(`Approved operations: ${stored.approvedOperationIds.join(", ") || "none"}`);
|
|
420
479
|
console.log(`Runs: ${stored.runs.length}`);
|
|
480
|
+
const mirror = await uploadHostedPatchPlan(stored);
|
|
481
|
+
if (mirror.status === "saved")
|
|
482
|
+
console.log(`Hosted review: ${mirror.state.url}`);
|
|
483
|
+
else if (mirror.status === "unavailable" || mirror.status === "conflict")
|
|
484
|
+
console.log(`Hosted review: sync pending (${mirror.reason})`);
|
|
421
485
|
if (stored.applyAttempts?.length) {
|
|
422
486
|
console.log("Apply attempts:");
|
|
423
487
|
for (const attempt of stored.applyAttempts) {
|
|
@@ -470,6 +534,10 @@ export async function plansCommand(args) {
|
|
|
470
534
|
...fileOverrides,
|
|
471
535
|
...explicitOverrides,
|
|
472
536
|
});
|
|
537
|
+
const mirror = await reportHostedPlanLifecycle(updated);
|
|
538
|
+
if (mirror.status === "unavailable" || mirror.status === "conflict") {
|
|
539
|
+
console.error(`Hosted plan status is stale: ${mirror.reason}. Local approval remains authoritative.`);
|
|
540
|
+
}
|
|
473
541
|
console.log(`Approved ${updated.approvedOperationIds.length} operation(s) on ${planId}. Apply with \`fullstackgtm apply --plan-id ${planId} --provider <name>\`.`);
|
|
474
542
|
return;
|
|
475
543
|
}
|
|
@@ -477,7 +545,11 @@ export async function plansCommand(args) {
|
|
|
477
545
|
const planId = rest.find((arg) => !arg.startsWith("--"));
|
|
478
546
|
if (!planId)
|
|
479
547
|
throw new Error("Usage: fullstackgtm plans reject <planId>");
|
|
480
|
-
await store.reject(planId);
|
|
548
|
+
const rejected = await store.reject(planId);
|
|
549
|
+
const mirror = await reportHostedPlanLifecycle(rejected);
|
|
550
|
+
if (mirror.status === "unavailable" || mirror.status === "conflict") {
|
|
551
|
+
console.error(`Hosted plan status is stale: ${mirror.reason}. Local rejection remains authoritative.`);
|
|
552
|
+
}
|
|
481
553
|
console.log(`Rejected ${planId}.`);
|
|
482
554
|
return;
|
|
483
555
|
}
|
|
@@ -503,5 +575,27 @@ export async function plansCommand(args) {
|
|
|
503
575
|
`No writes were replayed. Re-audit provider state, review the plan, then approve operations again before any apply.`);
|
|
504
576
|
return;
|
|
505
577
|
}
|
|
506
|
-
|
|
578
|
+
if (subcommand === "sync") {
|
|
579
|
+
const plans = await store.list();
|
|
580
|
+
let updated = 0;
|
|
581
|
+
const warnings = [];
|
|
582
|
+
for (const stored of plans) {
|
|
583
|
+
const result = await reconcileHostedPatchPlan(store, stored);
|
|
584
|
+
if (result.status === "updated")
|
|
585
|
+
updated += 1;
|
|
586
|
+
else if (result.status === "conflict" || result.status === "unavailable")
|
|
587
|
+
warnings.push(`${stored.plan.id}: ${result.reason}`);
|
|
588
|
+
if (stored.status === "applied") {
|
|
589
|
+
const hostedClaimId = [...(stored.applyAttempts ?? [])].reverse().find((attempt) => attempt.hostedClaimId)?.hostedClaimId;
|
|
590
|
+
const pushed = await reportHostedPlanLifecycle(stored, { claimId: hostedClaimId });
|
|
591
|
+
if (pushed.status === "conflict" || pushed.status === "unavailable")
|
|
592
|
+
warnings.push(`${stored.plan.id} receipt: ${pushed.reason}`);
|
|
593
|
+
}
|
|
594
|
+
}
|
|
595
|
+
console.log(`Plan replicas synchronized: ${updated} updated, ${plans.length - updated} unchanged.`);
|
|
596
|
+
for (const warning of warnings)
|
|
597
|
+
console.error(` ${warning}`);
|
|
598
|
+
return;
|
|
599
|
+
}
|
|
600
|
+
throw unknownSubcommandError("plans", subcommand, ["list", "show", "sync", "approve", "reject", "recover"]);
|
|
507
601
|
}
|
package/dist/cli/shared.d.ts
CHANGED
|
@@ -23,7 +23,9 @@ export declare function saveRequested(args: string[]): boolean;
|
|
|
23
23
|
export declare function confirmRequested(args: string[], ...legacyAliases: string[]): boolean;
|
|
24
24
|
export declare function numericOption(args: string[], name: string): number | undefined;
|
|
25
25
|
export declare function connectorFor(provider: string, args: string[], progress?: ProgressEmitter): Promise<GtmConnector>;
|
|
26
|
-
export declare function readSnapshot(args: string[], progress?: ProgressEmitter
|
|
26
|
+
export declare function readSnapshot(args: string[], progress?: ProgressEmitter, options?: {
|
|
27
|
+
persistProgress?: boolean;
|
|
28
|
+
}): Promise<CanonicalGtmSnapshot>;
|
|
27
29
|
/**
|
|
28
30
|
* Validate that an --input file actually has the canonical snapshot shape
|
|
29
31
|
* (the JSON `snapshot --out` writes) instead of blindly casting — a plan or
|
package/dist/cli/shared.js
CHANGED
|
@@ -131,7 +131,7 @@ export async function connectorFor(provider, args, progress) {
|
|
|
131
131
|
throw new Error(`Unknown provider: ${provider}.${providerSuggestion ? ` Did you mean ${providerSuggestion}?` : ""} ` +
|
|
132
132
|
"Supported providers: hubspot, salesforce, stripe");
|
|
133
133
|
}
|
|
134
|
-
export async function readSnapshot(args, progress) {
|
|
134
|
+
export async function readSnapshot(args, progress, options = {}) {
|
|
135
135
|
const provider = option(args, "--provider");
|
|
136
136
|
if (provider) {
|
|
137
137
|
// A verb driving its own progress board (e.g. `backfill stripe`) passes
|
|
@@ -153,7 +153,7 @@ export async function readSnapshot(args, progress) {
|
|
|
153
153
|
return await connector.fetchSnapshot();
|
|
154
154
|
}
|
|
155
155
|
finally {
|
|
156
|
-
renderer.done();
|
|
156
|
+
renderer.done({ persist: options.persistProgress });
|
|
157
157
|
}
|
|
158
158
|
}
|
|
159
159
|
if (args.includes("--demo")) {
|
package/dist/cli/ui.d.ts
CHANGED
|
@@ -88,14 +88,17 @@ export declare function startElapsedStatus(label: (elapsed: string) => string, s
|
|
|
88
88
|
};
|
|
89
89
|
export type Checklist = {
|
|
90
90
|
update(id: string, state: "pending" | "running" | "ok" | "warn" | "fail", note?: string): void;
|
|
91
|
-
/** Stop animating
|
|
92
|
-
done(
|
|
91
|
+
/** Stop animating; optionally leave the final board in the terminal history. */
|
|
92
|
+
done(options?: {
|
|
93
|
+
persist?: boolean;
|
|
94
|
+
}): void;
|
|
93
95
|
readonly active: boolean;
|
|
94
96
|
};
|
|
95
97
|
/**
|
|
96
98
|
* A multi-line live board — one line per item, each flipping ○ → ⠹ → ✓ as work
|
|
97
99
|
* progresses (the audit rule registry renders through this). Repaints with
|
|
98
|
-
* cursor-up; erases
|
|
100
|
+
* cursor-up; done() normally erases it, while done({ persist: true }) leaves a
|
|
101
|
+
* final static history for multi-phase human workflows.
|
|
99
102
|
*/
|
|
100
103
|
export declare function createChecklist(items: Array<{
|
|
101
104
|
id: string;
|
|
@@ -107,8 +110,10 @@ export type ProgressRenderer = {
|
|
|
107
110
|
* listener via `composeListeners` when the run should also heartbeat).
|
|
108
111
|
*/
|
|
109
112
|
listener: ProgressListener;
|
|
110
|
-
/** Stop animating
|
|
111
|
-
done(
|
|
113
|
+
/** Stop animating; optionally leave the completed board in terminal history. */
|
|
114
|
+
done(options?: {
|
|
115
|
+
persist?: boolean;
|
|
116
|
+
}): void;
|
|
112
117
|
readonly active: boolean;
|
|
113
118
|
};
|
|
114
119
|
/**
|
package/dist/cli/ui.js
CHANGED
|
@@ -241,7 +241,8 @@ const NOOP_CHECKLIST = { update() { }, done() { }, active: false };
|
|
|
241
241
|
/**
|
|
242
242
|
* A multi-line live board — one line per item, each flipping ○ → ⠹ → ✓ as work
|
|
243
243
|
* progresses (the audit rule registry renders through this). Repaints with
|
|
244
|
-
* cursor-up; erases
|
|
244
|
+
* cursor-up; done() normally erases it, while done({ persist: true }) leaves a
|
|
245
|
+
* final static history for multi-phase human workflows.
|
|
245
246
|
*/
|
|
246
247
|
export function createChecklist(items, stream = process.stderr, env = process.env) {
|
|
247
248
|
if (!animationEnabled(stream, env) || items.length === 0)
|
|
@@ -280,14 +281,22 @@ export function createChecklist(items, stream = process.stderr, env = process.en
|
|
|
280
281
|
if (!entry)
|
|
281
282
|
return;
|
|
282
283
|
entry.state = state;
|
|
283
|
-
|
|
284
|
+
if (note !== undefined)
|
|
285
|
+
entry.note = note;
|
|
284
286
|
start();
|
|
285
287
|
render();
|
|
286
288
|
},
|
|
287
|
-
done() {
|
|
289
|
+
done(options = {}) {
|
|
288
290
|
if (timer)
|
|
289
291
|
clearInterval(timer);
|
|
290
292
|
timer = null;
|
|
293
|
+
if (options.persist) {
|
|
294
|
+
// The caller's final state update already painted the completed board.
|
|
295
|
+
// Freeze that frame in scrollback; repainting here can leave both the
|
|
296
|
+
// last live frame and the final frame visible in some terminals.
|
|
297
|
+
painted = 0;
|
|
298
|
+
return;
|
|
299
|
+
}
|
|
291
300
|
if (painted > 0) {
|
|
292
301
|
// Erase the board: cursor up over every painted line, clear each.
|
|
293
302
|
stream.write(`\u001b[${painted}A${"\u001b[2K\n".repeat(painted)}\u001b[${painted}A`);
|
|
@@ -344,8 +353,10 @@ export function createProgressRenderer(stages, stream = process.stderr, env = pr
|
|
|
344
353
|
if (current)
|
|
345
354
|
board.update(current, "running", noteFor(event, snapshot));
|
|
346
355
|
},
|
|
347
|
-
done() {
|
|
348
|
-
|
|
356
|
+
done(options) {
|
|
357
|
+
if (current)
|
|
358
|
+
board.update(current, "ok");
|
|
359
|
+
board.done(options);
|
|
349
360
|
},
|
|
350
361
|
active: true,
|
|
351
362
|
};
|
|
@@ -38,6 +38,8 @@ export type FetchProspectsOptions = {
|
|
|
38
38
|
sourceId?: string;
|
|
39
39
|
/** Hard cap on prospects returned; also bounds pagination. */
|
|
40
40
|
max?: number;
|
|
41
|
+
/** Opaque continuation cursor. HeyReach uses the decimal list offset. */
|
|
42
|
+
cursor?: string;
|
|
41
43
|
};
|
|
42
44
|
export type ConnectionStatus = {
|
|
43
45
|
ok: boolean;
|
|
@@ -85,6 +85,11 @@ export function createHeyReachProvider(options) {
|
|
|
85
85
|
const max = opts.max ?? Number.POSITIVE_INFINITY;
|
|
86
86
|
const out = [];
|
|
87
87
|
let offset = 0;
|
|
88
|
+
if (opts.cursor !== undefined) {
|
|
89
|
+
if (!/^\d+$/.test(opts.cursor))
|
|
90
|
+
throw new Error(`Invalid HeyReach cursor: ${opts.cursor}`);
|
|
91
|
+
offset = Number(opts.cursor);
|
|
92
|
+
}
|
|
88
93
|
while (out.length < max) {
|
|
89
94
|
const data = await call("/list/GetLeadsFromList", "POST", { listId, offset, limit: pageSize });
|
|
90
95
|
const items = toArray(data);
|
|
@@ -46,6 +46,8 @@ export declare function fetchExploriumProspects(opts: {
|
|
|
46
46
|
apiKey: string;
|
|
47
47
|
filters: ExploriumFilters;
|
|
48
48
|
size?: number;
|
|
49
|
+
/** One-based result page. */
|
|
50
|
+
page?: number;
|
|
49
51
|
apiBaseUrl?: string;
|
|
50
52
|
fetchImpl?: FetchImpl;
|
|
51
53
|
}): Promise<Prospect[]>;
|
|
@@ -53,9 +55,30 @@ export declare function fetchPipe0CrustdataProspects(opts: {
|
|
|
53
55
|
apiKey: string;
|
|
54
56
|
filters: Record<string, unknown>;
|
|
55
57
|
limit?: number;
|
|
58
|
+
/** Opaque cursor returned by a previous page. */
|
|
59
|
+
cursor?: string;
|
|
56
60
|
apiBaseUrl?: string;
|
|
57
61
|
fetchImpl?: FetchImpl;
|
|
58
62
|
}): Promise<Prospect[]>;
|
|
63
|
+
/** A page from pipe0/Crustdata's cursor-based people search. */
|
|
64
|
+
export type Pipe0CrustdataProspectPage = {
|
|
65
|
+
prospects: Prospect[];
|
|
66
|
+
/** Opaque cursor for the next page, or null when the search is exhausted. */
|
|
67
|
+
nextCursor: string | null;
|
|
68
|
+
};
|
|
69
|
+
/**
|
|
70
|
+
* Fetch one page and expose its continuation cursor. The array-returning
|
|
71
|
+
* `fetchPipe0CrustdataProspects` remains available for existing consumers.
|
|
72
|
+
*/
|
|
73
|
+
export declare function fetchPipe0CrustdataProspectPage(opts: {
|
|
74
|
+
apiKey: string;
|
|
75
|
+
filters: Record<string, unknown>;
|
|
76
|
+
limit?: number;
|
|
77
|
+
/** Opaque cursor returned by a previous page. */
|
|
78
|
+
cursor?: string;
|
|
79
|
+
apiBaseUrl?: string;
|
|
80
|
+
fetchImpl?: FetchImpl;
|
|
81
|
+
}): Promise<Pipe0CrustdataProspectPage>;
|
|
59
82
|
export declare const EXPLORIUM_BUSINESS_COUNT_CAP = 60000;
|
|
60
83
|
export type BusinessCountProbe = {
|
|
61
84
|
/** matching companies (the account universe); == cap when saturated. */
|
|
@@ -24,7 +24,7 @@ export async function fetchExploriumProspects(opts) {
|
|
|
24
24
|
const response = await fetchImpl(`${base}/v1/prospects`, {
|
|
25
25
|
method: "POST",
|
|
26
26
|
headers: { "api_key": opts.apiKey, "Content-Type": "application/json" },
|
|
27
|
-
body: JSON.stringify({ mode: "full", size, page_size: size, page: 1, filters: opts.filters }),
|
|
27
|
+
body: JSON.stringify({ mode: "full", size, page_size: size, page: opts.page ?? 1, filters: opts.filters }),
|
|
28
28
|
});
|
|
29
29
|
if (!response.ok) {
|
|
30
30
|
throw new ProviderHttpError("Explorium", "prospect search", response.status);
|
|
@@ -48,14 +48,24 @@ export async function fetchExploriumProspects(opts) {
|
|
|
48
48
|
// (no separate connection needed). Returns profiles; pair with
|
|
49
49
|
// pipe0ResolveWorkEmails to get real emails.
|
|
50
50
|
export async function fetchPipe0CrustdataProspects(opts) {
|
|
51
|
+
return (await fetchPipe0CrustdataProspectPage(opts)).prospects;
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Fetch one page and expose its continuation cursor. The array-returning
|
|
55
|
+
* `fetchPipe0CrustdataProspects` remains available for existing consumers.
|
|
56
|
+
*/
|
|
57
|
+
export async function fetchPipe0CrustdataProspectPage(opts) {
|
|
51
58
|
const fetchImpl = opts.fetchImpl ?? fetch;
|
|
52
59
|
const base = (opts.apiBaseUrl ?? "https://api.pipe0.com").replace(/\/$/, "");
|
|
53
60
|
const limit = Math.min(opts.limit ?? 25, 100);
|
|
61
|
+
const config = { limit, filters: opts.filters };
|
|
62
|
+
if (opts.cursor !== undefined)
|
|
63
|
+
config.cursor = opts.cursor;
|
|
54
64
|
const response = await fetchImpl(`${base}/v1/searches/run/sync`, {
|
|
55
65
|
method: "POST",
|
|
56
66
|
headers: { Authorization: `Bearer ${opts.apiKey}`, "Content-Type": "application/json" },
|
|
57
67
|
body: JSON.stringify({
|
|
58
|
-
searches: [{ search_id: "people:profiles:crustdata@1", config
|
|
68
|
+
searches: [{ search_id: "people:profiles:crustdata@1", config }],
|
|
59
69
|
}),
|
|
60
70
|
});
|
|
61
71
|
if (!response.ok) {
|
|
@@ -68,7 +78,7 @@ export async function fetchPipe0CrustdataProspects(opts) {
|
|
|
68
78
|
if (upstreamErrors.length > 0 && !(body.results ?? []).length) {
|
|
69
79
|
throw new Error(`pipe0/Crustdata search error: ${upstreamErrors.map((e) => `${e.code ?? "error"}: ${e.message ?? ""}`).join("; ")}`);
|
|
70
80
|
}
|
|
71
|
-
|
|
81
|
+
const prospects = (body.results ?? []).map((row) => {
|
|
72
82
|
const fullName = strField(row.name);
|
|
73
83
|
const { firstName, lastName } = splitName(fullName);
|
|
74
84
|
return {
|
|
@@ -81,6 +91,14 @@ export async function fetchPipe0CrustdataProspects(opts) {
|
|
|
81
91
|
sourceId: strField(row.profile_url) ?? fullName,
|
|
82
92
|
};
|
|
83
93
|
});
|
|
94
|
+
// Current pipe0 responses expose `next_cursor` at the top level. Accept the
|
|
95
|
+
// status-envelope variants too so older deployments remain usable.
|
|
96
|
+
const status = body.search_statuses?.[0];
|
|
97
|
+
const rawCursor = body.next_cursor ?? status?.next_cursor ?? status?.cursor;
|
|
98
|
+
return {
|
|
99
|
+
prospects,
|
|
100
|
+
nextCursor: typeof rawCursor === "string" && rawCursor.length > 0 ? rawCursor : null,
|
|
101
|
+
};
|
|
84
102
|
}
|
|
85
103
|
function strField(field) {
|
|
86
104
|
const v = field?.value;
|
package/dist/enrich.d.ts
CHANGED
|
@@ -86,6 +86,8 @@ export type AcquireDiscoveryConfig = {
|
|
|
86
86
|
provider: "explorium" | "pipe0" | "linkedin" | "heyreach";
|
|
87
87
|
filters?: Record<string, unknown>;
|
|
88
88
|
size?: number;
|
|
89
|
+
/** Maximum raw provider candidates scanned per run while seeking fresh leads. */
|
|
90
|
+
scanLimit?: number;
|
|
89
91
|
resolveEmailsWith?: "pipe0";
|
|
90
92
|
/** LinkedIn sources only: the provider-native list id to read (HeyReach lead-list id). */
|
|
91
93
|
listId?: string;
|
|
@@ -286,7 +288,46 @@ export declare function selectStaleWork(config: EnrichConfig, runs: EnrichRun[],
|
|
|
286
288
|
now?: () => Date;
|
|
287
289
|
staleDaysOverride?: number;
|
|
288
290
|
}): EnrichWorkItem[];
|
|
289
|
-
export type EnrichRunMode = EnrichMode | "ingest";
|
|
291
|
+
export type EnrichRunMode = EnrichMode | "ingest" | "acquire";
|
|
292
|
+
/**
|
|
293
|
+
* Provider continuation state for an acquisition query. The fingerprint binds
|
|
294
|
+
* a cursor/offset to the ICP + provider filters that produced it, preventing a
|
|
295
|
+
* changed query from accidentally resuming in the middle of the old audience.
|
|
296
|
+
*/
|
|
297
|
+
export type AcquireDiscoveryCheckpoint = {
|
|
298
|
+
queryFingerprint: string;
|
|
299
|
+
/** Opaque provider cursor (Pipe0 and other cursor-based sources). */
|
|
300
|
+
cursor?: string | null;
|
|
301
|
+
/** Zero-based provider offset (LinkedIn/HeyReach and other offset sources). */
|
|
302
|
+
offset?: number | null;
|
|
303
|
+
/** True only when the provider has positively reported no next page. */
|
|
304
|
+
exhausted: boolean;
|
|
305
|
+
};
|
|
306
|
+
/**
|
|
307
|
+
* Truthful acquisition funnel for one run. Each field counts candidates, not
|
|
308
|
+
* provider requests, making duplicate saturation and resolution loss visible
|
|
309
|
+
* instead of reporting every run as zero.
|
|
310
|
+
*/
|
|
311
|
+
export type AcquireRunFunnel = {
|
|
312
|
+
/** Raw candidates returned across all discovery pages scanned this run. */
|
|
313
|
+
discovered: number;
|
|
314
|
+
/** Candidates that met the configured ICP threshold. */
|
|
315
|
+
qualified: number;
|
|
316
|
+
/** Qualified candidates removed by the CRM pre-email dedupe. */
|
|
317
|
+
skippedCrm: number;
|
|
318
|
+
/** Qualified candidates removed by the cross-run seen ledger. */
|
|
319
|
+
skippedSeen: number;
|
|
320
|
+
/** Fresh candidates carrying the configured resolve-first key. */
|
|
321
|
+
resolved: number;
|
|
322
|
+
/** Governed create operations emitted into the saved/dry-run plan. */
|
|
323
|
+
proposed: number;
|
|
324
|
+
/** Otherwise-proposable candidates held back by the acquire meter. */
|
|
325
|
+
withheldByMeter: number;
|
|
326
|
+
};
|
|
327
|
+
export type AcquireRunTelemetry = {
|
|
328
|
+
funnel: AcquireRunFunnel;
|
|
329
|
+
discovery: AcquireDiscoveryCheckpoint;
|
|
330
|
+
};
|
|
290
331
|
export type EnrichRun = {
|
|
291
332
|
id: string;
|
|
292
333
|
runLabel: string;
|
|
@@ -298,6 +339,8 @@ export type EnrichRun = {
|
|
|
298
339
|
/** Resume point for an interrupted pull (last processed pull key). */
|
|
299
340
|
cursor: string | null;
|
|
300
341
|
counts: EnrichCounts;
|
|
342
|
+
/** Acquisition-only funnel + provider continuation state (absent on legacy runs). */
|
|
343
|
+
acquireTelemetry?: AcquireRunTelemetry;
|
|
301
344
|
planIds: string[];
|
|
302
345
|
stamps: EnrichStamp[];
|
|
303
346
|
/** Staged source rows (ingest mode only), consumed by append/refresh. */
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import type { AcquireDiscoveryCheckpoint } from "./enrich.ts";
|
|
2
|
+
export type HostedAcquireCheckpoint = {
|
|
3
|
+
checkpoint: AcquireDiscoveryCheckpoint;
|
|
4
|
+
/** Opaque server revision used only for compare-and-swap. */
|
|
5
|
+
version: number;
|
|
6
|
+
updatedAt: number;
|
|
7
|
+
};
|
|
8
|
+
export type HostedCheckpointReadResult = {
|
|
9
|
+
status: "unpaired";
|
|
10
|
+
} | {
|
|
11
|
+
status: "missing";
|
|
12
|
+
} | {
|
|
13
|
+
status: "found";
|
|
14
|
+
record: HostedAcquireCheckpoint;
|
|
15
|
+
} | {
|
|
16
|
+
status: "unavailable";
|
|
17
|
+
reason: string;
|
|
18
|
+
};
|
|
19
|
+
export type HostedCheckpointWriteResult = {
|
|
20
|
+
status: "unpaired";
|
|
21
|
+
} | {
|
|
22
|
+
status: "saved";
|
|
23
|
+
record: HostedAcquireCheckpoint;
|
|
24
|
+
} | {
|
|
25
|
+
status: "conflict";
|
|
26
|
+
current: HostedAcquireCheckpoint | null;
|
|
27
|
+
} | {
|
|
28
|
+
status: "unavailable";
|
|
29
|
+
reason: string;
|
|
30
|
+
};
|
|
31
|
+
type Options = {
|
|
32
|
+
fetchImpl?: typeof fetch;
|
|
33
|
+
timeoutMs?: number;
|
|
34
|
+
};
|
|
35
|
+
/** Read a query/list checkpoint. Inert unless this profile is paired. */
|
|
36
|
+
export declare function readHostedAcquireCheckpoint(key: string, options?: Options): Promise<HostedCheckpointReadResult>;
|
|
37
|
+
/**
|
|
38
|
+
* Mirror a local checkpoint using compare-and-swap. Pass null only when the
|
|
39
|
+
* caller observed no hosted record (sent as version zero); pass the exact
|
|
40
|
+
* positive version returned by read for an update.
|
|
41
|
+
*/
|
|
42
|
+
export declare function writeHostedAcquireCheckpoint(key: string, checkpoint: AcquireDiscoveryCheckpoint, expectedVersion: number | null, options?: Options): Promise<HostedCheckpointWriteResult>;
|
|
43
|
+
export {};
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Authenticated acquisition-checkpoint transport for paired CLI installs.
|
|
3
|
+
*
|
|
4
|
+
* The local checkpoint remains authoritative for running a job. This module
|
|
5
|
+
* only mirrors/query-scoped continuation state to the paired hosted app so a
|
|
6
|
+
* different authenticated worker can resume it. Writes use compare-and-swap:
|
|
7
|
+
* a stale client receives a conflict and must explicitly reconcile instead of
|
|
8
|
+
* silently moving a provider cursor backwards or overwriting another worker.
|
|
9
|
+
*/
|
|
10
|
+
import { getCredential } from "./credentials.js";
|
|
11
|
+
const ENDPOINT = "/api/cli/acquisition-checkpoint";
|
|
12
|
+
const DEFAULT_TIMEOUT_MS = 4000;
|
|
13
|
+
const CHECKPOINT_KEY = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$/;
|
|
14
|
+
function validKey(key) {
|
|
15
|
+
if (!CHECKPOINT_KEY.test(key)) {
|
|
16
|
+
throw new Error("Acquisition checkpoint key must be 1-256 URL-safe characters.");
|
|
17
|
+
}
|
|
18
|
+
return key;
|
|
19
|
+
}
|
|
20
|
+
function broker() {
|
|
21
|
+
const credential = getCredential("broker");
|
|
22
|
+
if (!credential?.baseUrl || !credential.accessToken)
|
|
23
|
+
return null;
|
|
24
|
+
let url;
|
|
25
|
+
try {
|
|
26
|
+
url = new URL(credential.baseUrl);
|
|
27
|
+
}
|
|
28
|
+
catch {
|
|
29
|
+
return null;
|
|
30
|
+
}
|
|
31
|
+
const local = url.hostname === "localhost" || url.hostname === "127.0.0.1" || url.hostname === "[::1]";
|
|
32
|
+
if (url.protocol !== "https:" && !(url.protocol === "http:" && local))
|
|
33
|
+
return null;
|
|
34
|
+
return { baseUrl: url.href.replace(/\/+$/, ""), accessToken: credential.accessToken };
|
|
35
|
+
}
|
|
36
|
+
function isCheckpoint(value) {
|
|
37
|
+
if (!value || typeof value !== "object")
|
|
38
|
+
return false;
|
|
39
|
+
const item = value;
|
|
40
|
+
return (typeof item.queryFingerprint === "string" &&
|
|
41
|
+
item.queryFingerprint.length > 0 &&
|
|
42
|
+
typeof item.exhausted === "boolean" &&
|
|
43
|
+
(item.cursor === undefined || item.cursor === null || typeof item.cursor === "string") &&
|
|
44
|
+
(item.offset === undefined || item.offset === null ||
|
|
45
|
+
(typeof item.offset === "number" && Number.isSafeInteger(item.offset) && item.offset >= 0)));
|
|
46
|
+
}
|
|
47
|
+
function parseRecord(value) {
|
|
48
|
+
if (!value || typeof value !== "object")
|
|
49
|
+
return null;
|
|
50
|
+
const body = value;
|
|
51
|
+
if (!isCheckpoint(body.checkpoint) || !Number.isSafeInteger(body.version) || body.version < 1)
|
|
52
|
+
return null;
|
|
53
|
+
if (!Number.isSafeInteger(body.updatedAt) || body.updatedAt < 0)
|
|
54
|
+
return null;
|
|
55
|
+
return {
|
|
56
|
+
checkpoint: body.checkpoint,
|
|
57
|
+
version: body.version,
|
|
58
|
+
updatedAt: body.updatedAt,
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
function reasonFor(status) {
|
|
62
|
+
if (status === 401 || status === 403)
|
|
63
|
+
return "hosted authentication was rejected; pair the CLI again";
|
|
64
|
+
return `hosted checkpoint request failed (HTTP ${status})`;
|
|
65
|
+
}
|
|
66
|
+
/** Read a query/list checkpoint. Inert unless this profile is paired. */
|
|
67
|
+
export async function readHostedAcquireCheckpoint(key, options = {}) {
|
|
68
|
+
validKey(key);
|
|
69
|
+
const paired = broker();
|
|
70
|
+
if (!paired)
|
|
71
|
+
return { status: "unpaired" };
|
|
72
|
+
try {
|
|
73
|
+
const response = await (options.fetchImpl ?? fetch)(`${paired.baseUrl}${ENDPOINT}?key=${encodeURIComponent(key)}`, {
|
|
74
|
+
headers: { Authorization: `Bearer ${paired.accessToken}`, Accept: "application/json" },
|
|
75
|
+
signal: AbortSignal.timeout(options.timeoutMs ?? DEFAULT_TIMEOUT_MS),
|
|
76
|
+
});
|
|
77
|
+
if (response.status === 404)
|
|
78
|
+
return { status: "missing" };
|
|
79
|
+
if (!response.ok)
|
|
80
|
+
return { status: "unavailable", reason: reasonFor(response.status) };
|
|
81
|
+
const record = parseRecord(await response.json());
|
|
82
|
+
return record
|
|
83
|
+
? { status: "found", record }
|
|
84
|
+
: { status: "unavailable", reason: "hosted checkpoint response was invalid" };
|
|
85
|
+
}
|
|
86
|
+
catch {
|
|
87
|
+
return { status: "unavailable", reason: "hosted checkpoint request was unavailable" };
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* Mirror a local checkpoint using compare-and-swap. Pass null only when the
|
|
92
|
+
* caller observed no hosted record (sent as version zero); pass the exact
|
|
93
|
+
* positive version returned by read for an update.
|
|
94
|
+
*/
|
|
95
|
+
export async function writeHostedAcquireCheckpoint(key, checkpoint, expectedVersion, options = {}) {
|
|
96
|
+
validKey(key);
|
|
97
|
+
if (!isCheckpoint(checkpoint))
|
|
98
|
+
throw new Error("Refusing to sync an invalid acquisition checkpoint.");
|
|
99
|
+
if (expectedVersion !== null && (!Number.isSafeInteger(expectedVersion) || expectedVersion < 1)) {
|
|
100
|
+
throw new Error("expectedVersion must be null or a positive integer.");
|
|
101
|
+
}
|
|
102
|
+
const paired = broker();
|
|
103
|
+
if (!paired)
|
|
104
|
+
return { status: "unpaired" };
|
|
105
|
+
try {
|
|
106
|
+
const response = await (options.fetchImpl ?? fetch)(`${paired.baseUrl}${ENDPOINT}`, {
|
|
107
|
+
method: "POST",
|
|
108
|
+
headers: { Authorization: `Bearer ${paired.accessToken}`, "Content-Type": "application/json" },
|
|
109
|
+
body: JSON.stringify({ key, checkpoint, expectedVersion: expectedVersion ?? 0 }),
|
|
110
|
+
signal: AbortSignal.timeout(options.timeoutMs ?? DEFAULT_TIMEOUT_MS),
|
|
111
|
+
});
|
|
112
|
+
if (response.status === 409) {
|
|
113
|
+
const body = await response.json().catch(() => null);
|
|
114
|
+
const current = body && typeof body === "object"
|
|
115
|
+
? parseRecord(body.current)
|
|
116
|
+
: null;
|
|
117
|
+
return { status: "conflict", current };
|
|
118
|
+
}
|
|
119
|
+
if (!response.ok)
|
|
120
|
+
return { status: "unavailable", reason: reasonFor(response.status) };
|
|
121
|
+
const record = parseRecord(await response.json());
|
|
122
|
+
return record
|
|
123
|
+
? { status: "saved", record }
|
|
124
|
+
: { status: "unavailable", reason: "hosted checkpoint response was invalid" };
|
|
125
|
+
}
|
|
126
|
+
catch {
|
|
127
|
+
return { status: "unavailable", reason: "hosted checkpoint request was unavailable" };
|
|
128
|
+
}
|
|
129
|
+
}
|