fullstackgtm 0.45.0 → 0.47.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +115 -1
- package/INSTALL_FOR_AGENTS.md +13 -11
- package/README.md +27 -18
- package/dist/backfill.d.ts +86 -0
- package/dist/backfill.js +251 -0
- package/dist/bin.js +4 -1
- package/dist/cli/auth.d.ts +2 -0
- package/dist/cli/auth.js +119 -12
- package/dist/cli/backfill.d.ts +1 -0
- package/dist/cli/backfill.js +125 -0
- package/dist/cli/backfillRuns.d.ts +1 -0
- package/dist/cli/backfillRuns.js +187 -0
- package/dist/cli/call.js +23 -9
- package/dist/cli/enrich.js +28 -10
- package/dist/cli/fix.js +9 -7
- package/dist/cli/help.js +60 -23
- package/dist/cli/plans.js +13 -11
- package/dist/cli/shared.d.ts +4 -3
- package/dist/cli/shared.js +31 -20
- package/dist/cli/ui.d.ts +21 -0
- package/dist/cli/ui.js +53 -1
- package/dist/cli.js +37 -41
- package/dist/connector.d.ts +8 -0
- package/dist/connector.js +104 -21
- package/dist/connectors/hubspot.d.ts +8 -1
- package/dist/connectors/hubspot.js +406 -13
- package/dist/connectors/hubspotAuth.js +5 -1
- package/dist/connectors/linkedin.js +14 -14
- package/dist/connectors/salesforce.d.ts +8 -1
- package/dist/connectors/salesforce.js +163 -2
- package/dist/connectors/stripe.d.ts +37 -0
- package/dist/connectors/stripe.js +103 -31
- package/dist/enrich.d.ts +7 -0
- package/dist/enrich.js +7 -0
- package/dist/health.d.ts +11 -69
- package/dist/health.js +4 -134
- package/dist/healthScore.d.ts +71 -0
- package/dist/healthScore.js +143 -0
- package/dist/index.d.ts +3 -1
- package/dist/index.js +3 -1
- package/dist/llm.d.ts +29 -0
- package/dist/llm.js +206 -0
- package/dist/market.d.ts +39 -1
- package/dist/market.js +116 -44
- package/dist/marketClassify.d.ts +9 -1
- package/dist/marketClassify.js +10 -1
- package/dist/marketTaxonomy.d.ts +6 -1
- package/dist/marketTaxonomy.js +4 -2
- package/dist/mcp-bin.js +29 -0
- package/dist/mcp.js +117 -4
- package/dist/progress.d.ts +96 -0
- package/dist/progress.js +142 -0
- package/dist/runReport.d.ts +24 -0
- package/dist/runReport.js +139 -4
- package/dist/types.d.ts +33 -1
- package/docs/api.md +4 -2
- package/docs/architecture.md +2 -0
- package/docs/linkedin-connector-spec.md +1 -1
- package/llms.txt +3 -3
- package/package.json +10 -3
- package/skills/fullstackgtm/SKILL.md +1 -0
- package/src/backfill.ts +340 -0
- package/src/bin.ts +5 -1
- package/src/cli/auth.ts +135 -15
- package/src/cli/backfill.ts +156 -0
- package/src/cli/backfillRuns.ts +198 -0
- package/src/cli/call.ts +26 -10
- package/src/cli/enrich.ts +33 -10
- package/src/cli/fix.ts +11 -10
- package/src/cli/help.ts +61 -23
- package/src/cli/plans.ts +15 -14
- package/src/cli/shared.ts +44 -27
- package/src/cli/ui.ts +72 -1
- package/src/cli.ts +38 -41
- package/src/connector.ts +110 -16
- package/src/connectors/hubspot.ts +423 -14
- package/src/connectors/hubspotAuth.ts +5 -1
- package/src/connectors/linkedin.ts +14 -14
- package/src/connectors/salesforce.ts +168 -3
- package/src/connectors/stripe.ts +154 -34
- package/src/enrich.ts +13 -0
- package/src/health.ts +14 -213
- package/src/healthScore.ts +223 -0
- package/src/index.ts +28 -1
- package/src/llm.ts +239 -0
- package/src/market.ts +157 -44
- package/src/marketClassify.ts +18 -1
- package/src/marketTaxonomy.ts +10 -2
- package/src/mcp-bin.ts +32 -0
- package/src/mcp.ts +140 -6
- package/src/progress.ts +197 -0
- package/src/runReport.ts +159 -4
- package/src/types.ts +35 -2
- package/docs/dx-punch-list.md +0 -87
- package/docs/roadmap-to-1.0.md +0 -211
package/dist/market.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { type ProgressEmitter } from "./progress.ts";
|
|
1
2
|
import type { GtmEvidence } from "./types.ts";
|
|
2
3
|
/**
|
|
3
4
|
* The Market Map: a live model of the competitive category a company sells
|
|
@@ -163,12 +164,24 @@ export type FetchPage = (url: string) => Promise<{
|
|
|
163
164
|
}>;
|
|
164
165
|
export declare function assertPublicUrl(rawUrl: string): Promise<URL>;
|
|
165
166
|
export type CaptureOptions = {
|
|
166
|
-
/** Directory for captures; defaults to <marketHome>/captures. */
|
|
167
|
+
/** Directory for captures; defaults to <marketHome>/captures. Ignored when `store` is given. */
|
|
167
168
|
dir?: string;
|
|
168
169
|
runLabel?: string;
|
|
169
170
|
/** Injectable for tests; defaults to global fetch. */
|
|
170
171
|
fetchPage?: FetchPage;
|
|
171
172
|
now?: () => Date;
|
|
173
|
+
/**
|
|
174
|
+
* Storage seam (see MarketStore below). Defaults to the profile-home file
|
|
175
|
+
* store; the hosted app passes a Convex-backed store. Same engine, same
|
|
176
|
+
* evidence chain, different persistence.
|
|
177
|
+
*/
|
|
178
|
+
store?: MarketStore;
|
|
179
|
+
/**
|
|
180
|
+
* Progress emission over MARKET_CAPTURE_STAGES ("sources" → "capture" →
|
|
181
|
+
* "persist" here; "classify" belongs to classifyMarket). Presentation-only:
|
|
182
|
+
* a throwing listener never fails the capture.
|
|
183
|
+
*/
|
|
184
|
+
progress?: ProgressEmitter;
|
|
172
185
|
};
|
|
173
186
|
export type CaptureResult = {
|
|
174
187
|
entries: CaptureEntry[];
|
|
@@ -197,6 +210,31 @@ export declare function loadCaptureTexts(category: string, directory?: string):
|
|
|
197
210
|
entries: CaptureEntry[];
|
|
198
211
|
textByHash: Map<string, string>;
|
|
199
212
|
};
|
|
213
|
+
export interface MarketStore {
|
|
214
|
+
/** Operator-facing location of the capture manifest (file path or logical label). */
|
|
215
|
+
captureLocation(): string;
|
|
216
|
+
/** Persist one page's extracted text under its content hash (idempotent). */
|
|
217
|
+
saveCaptureText(captureHash: string, text: string): Promise<void>;
|
|
218
|
+
/** Append one capture pass's manifest entries (append-only; never rewrites history). */
|
|
219
|
+
appendCaptureEntries(entries: CaptureEntry[]): Promise<void>;
|
|
220
|
+
/** Full manifest + resolvable texts — the evidence-verification input. */
|
|
221
|
+
loadCaptureTexts(): Promise<{
|
|
222
|
+
entries: CaptureEntry[];
|
|
223
|
+
textByHash: Map<string, string>;
|
|
224
|
+
}>;
|
|
225
|
+
/** Append-only observation runs (the same ObservationStore contract as before). */
|
|
226
|
+
observations: ObservationStore;
|
|
227
|
+
}
|
|
228
|
+
/** The CLI's store: captures + observations under the profile market home. */
|
|
229
|
+
export declare function createFileMarketStore(category: string, options?: {
|
|
230
|
+
capturesDir?: string;
|
|
231
|
+
observationsDir?: string;
|
|
232
|
+
}): MarketStore;
|
|
233
|
+
/**
|
|
234
|
+
* In-memory store: seam tests, and throwaway grounding captures (e.g. the
|
|
235
|
+
* taxonomy proposer's bootstrap pass) that should never persist anywhere.
|
|
236
|
+
*/
|
|
237
|
+
export declare function createMemoryMarketStore(category: string): MarketStore;
|
|
200
238
|
/**
|
|
201
239
|
* Whitespace-only normalization for span matching, plus one extraction
|
|
202
240
|
* artifact: the HTML-to-text step can emit a line break before punctuation
|
package/dist/market.js
CHANGED
|
@@ -4,6 +4,7 @@ import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from
|
|
|
4
4
|
import { isIP } from "node:net";
|
|
5
5
|
import { join } from "node:path";
|
|
6
6
|
import { credentialsDir } from "./credentials.js";
|
|
7
|
+
import { MARKET_CAPTURE_STAGES, nullProgressEmitter } from "./progress.js";
|
|
7
8
|
const INTENSITY_RANK = {
|
|
8
9
|
loud: 3,
|
|
9
10
|
quiet: 2,
|
|
@@ -283,58 +284,57 @@ const defaultFetchPage = async (url) => {
|
|
|
283
284
|
throw new Error(`market capture: too many redirects (>${MAX_REDIRECTS}) for ${url}`);
|
|
284
285
|
};
|
|
285
286
|
export async function captureMarket(config, options = {}) {
|
|
286
|
-
const
|
|
287
|
+
const store = options.store ?? createFileMarketStore(config.category, { capturesDir: options.dir });
|
|
288
|
+
const progress = options.progress ?? nullProgressEmitter();
|
|
287
289
|
const runLabel = options.runLabel ?? "run-1";
|
|
288
290
|
const fetchPage = options.fetchPage ?? defaultFetchPage;
|
|
289
291
|
const fetchedAt = (options.now ?? (() => new Date()))().toISOString();
|
|
290
|
-
|
|
291
|
-
const
|
|
292
|
-
const manifest = existsSync(manifestPath)
|
|
293
|
-
? JSON.parse(readFileSync(manifestPath, "utf8"))
|
|
294
|
-
: [];
|
|
295
|
-
const entries = [];
|
|
292
|
+
progress.stage(MARKET_CAPTURE_STAGES[0], 0, MARKET_CAPTURE_STAGES.length);
|
|
293
|
+
const targets = [];
|
|
296
294
|
for (const vendor of config.vendors) {
|
|
297
|
-
|
|
298
|
-
{ kind: "home", url: vendor.urls.home },
|
|
299
|
-
];
|
|
295
|
+
targets.push({ vendorId: vendor.id, kind: "home", url: vendor.urls.home });
|
|
300
296
|
if (vendor.urls.pricing)
|
|
301
|
-
targets.push({ kind: "pricing", url: vendor.urls.pricing });
|
|
297
|
+
targets.push({ vendorId: vendor.id, kind: "pricing", url: vendor.urls.pricing });
|
|
302
298
|
for (const url of vendor.urls.product)
|
|
303
|
-
targets.push({ kind: "product", url });
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
// Content-addressed: an unchanged page dedupes to the same file.
|
|
320
|
-
writeFileSync(join(dir, `${captureHash}.txt`), text);
|
|
321
|
-
}
|
|
322
|
-
const entry = {
|
|
323
|
-
runLabel,
|
|
324
|
-
vendorId: vendor.id,
|
|
325
|
-
kind: target.kind,
|
|
326
|
-
url: target.url,
|
|
327
|
-
fetchedAt,
|
|
328
|
-
httpStatus: status,
|
|
329
|
-
captureHash,
|
|
330
|
-
textChars: text.length,
|
|
331
|
-
};
|
|
332
|
-
manifest.push(entry);
|
|
333
|
-
entries.push(entry);
|
|
299
|
+
targets.push({ vendorId: vendor.id, kind: "product", url });
|
|
300
|
+
}
|
|
301
|
+
progress.note(`${config.vendors.length} vendor(s), ${targets.length} page(s)`);
|
|
302
|
+
progress.stage(MARKET_CAPTURE_STAGES[1], 1, MARKET_CAPTURE_STAGES.length);
|
|
303
|
+
const entries = [];
|
|
304
|
+
for (const target of targets) {
|
|
305
|
+
let status = null;
|
|
306
|
+
let text = "";
|
|
307
|
+
try {
|
|
308
|
+
const page = await fetchPage(target.url);
|
|
309
|
+
status = page.status;
|
|
310
|
+
if (page.status === 200)
|
|
311
|
+
text = extractReadableText(page.body);
|
|
312
|
+
}
|
|
313
|
+
catch {
|
|
314
|
+
status = null;
|
|
334
315
|
}
|
|
316
|
+
let captureHash = null;
|
|
317
|
+
if (text) {
|
|
318
|
+
captureHash = createHash("sha256").update(text).digest("hex");
|
|
319
|
+
// Content-addressed: an unchanged page dedupes to the same key.
|
|
320
|
+
await store.saveCaptureText(captureHash, text);
|
|
321
|
+
}
|
|
322
|
+
entries.push({
|
|
323
|
+
runLabel,
|
|
324
|
+
vendorId: target.vendorId,
|
|
325
|
+
kind: target.kind,
|
|
326
|
+
url: target.url,
|
|
327
|
+
fetchedAt,
|
|
328
|
+
httpStatus: status,
|
|
329
|
+
captureHash,
|
|
330
|
+
textChars: text.length,
|
|
331
|
+
});
|
|
332
|
+
progress.items(entries.length, targets.length);
|
|
335
333
|
}
|
|
336
|
-
|
|
337
|
-
|
|
334
|
+
progress.stage(MARKET_CAPTURE_STAGES[3], 3, MARKET_CAPTURE_STAGES.length);
|
|
335
|
+
await store.appendCaptureEntries(entries);
|
|
336
|
+
progress.flush();
|
|
337
|
+
return { entries, manifestPath: store.captureLocation() };
|
|
338
338
|
}
|
|
339
339
|
export function createFileObservationStore(category, directory) {
|
|
340
340
|
const dir = directory ?? join(marketHome(category), "observations");
|
|
@@ -459,6 +459,78 @@ export function loadCaptureTexts(category, directory) {
|
|
|
459
459
|
}
|
|
460
460
|
return { entries, textByHash };
|
|
461
461
|
}
|
|
462
|
+
/** The CLI's store: captures + observations under the profile market home. */
|
|
463
|
+
export function createFileMarketStore(category, options = {}) {
|
|
464
|
+
const dir = options.capturesDir ?? join(marketHome(category), "captures");
|
|
465
|
+
const manifestPath = join(dir, "manifest.json");
|
|
466
|
+
return {
|
|
467
|
+
captureLocation: () => manifestPath,
|
|
468
|
+
async saveCaptureText(captureHash, text) {
|
|
469
|
+
mkdirSync(dir, { recursive: true });
|
|
470
|
+
writeFileSync(join(dir, `${captureHash}.txt`), text);
|
|
471
|
+
},
|
|
472
|
+
async appendCaptureEntries(entries) {
|
|
473
|
+
mkdirSync(dir, { recursive: true });
|
|
474
|
+
const manifest = existsSync(manifestPath)
|
|
475
|
+
? JSON.parse(readFileSync(manifestPath, "utf8"))
|
|
476
|
+
: [];
|
|
477
|
+
manifest.push(...entries);
|
|
478
|
+
writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`);
|
|
479
|
+
},
|
|
480
|
+
async loadCaptureTexts() {
|
|
481
|
+
return loadCaptureTexts(category, dir);
|
|
482
|
+
},
|
|
483
|
+
observations: createFileObservationStore(category, options.observationsDir),
|
|
484
|
+
};
|
|
485
|
+
}
|
|
486
|
+
/**
|
|
487
|
+
* In-memory store: seam tests, and throwaway grounding captures (e.g. the
|
|
488
|
+
* taxonomy proposer's bootstrap pass) that should never persist anywhere.
|
|
489
|
+
*/
|
|
490
|
+
export function createMemoryMarketStore(category) {
|
|
491
|
+
const entries = [];
|
|
492
|
+
const textByHash = new Map();
|
|
493
|
+
const sets = new Map();
|
|
494
|
+
const sorted = () => [...sets.values()].sort((a, b) => a.runAt.localeCompare(b.runAt));
|
|
495
|
+
return {
|
|
496
|
+
captureLocation: () => `memory://${category}/captures`,
|
|
497
|
+
async saveCaptureText(captureHash, text) {
|
|
498
|
+
textByHash.set(captureHash, text);
|
|
499
|
+
},
|
|
500
|
+
async appendCaptureEntries(newEntries) {
|
|
501
|
+
entries.push(...newEntries);
|
|
502
|
+
},
|
|
503
|
+
async loadCaptureTexts() {
|
|
504
|
+
return { entries: [...entries], textByHash: new Map(textByHash) };
|
|
505
|
+
},
|
|
506
|
+
observations: {
|
|
507
|
+
async append(set) {
|
|
508
|
+
if (set.category !== category) {
|
|
509
|
+
throw new Error(`Observation set category "${set.category}" does not match store "${category}"`);
|
|
510
|
+
}
|
|
511
|
+
if (sets.has(set.runLabel)) {
|
|
512
|
+
throw new Error(`Run "${set.runLabel}" already exists — observations are append-only; use a new run label`);
|
|
513
|
+
}
|
|
514
|
+
sets.set(set.runLabel, set);
|
|
515
|
+
return set;
|
|
516
|
+
},
|
|
517
|
+
async get(runLabel) {
|
|
518
|
+
return sets.get(runLabel) ?? null;
|
|
519
|
+
},
|
|
520
|
+
async list() {
|
|
521
|
+
return sorted().map((set) => ({
|
|
522
|
+
runLabel: set.runLabel,
|
|
523
|
+
runAt: set.runAt,
|
|
524
|
+
observations: set.observations.length,
|
|
525
|
+
}));
|
|
526
|
+
},
|
|
527
|
+
async latest() {
|
|
528
|
+
const all = sorted();
|
|
529
|
+
return all.length ? all[all.length - 1] : null;
|
|
530
|
+
},
|
|
531
|
+
},
|
|
532
|
+
};
|
|
533
|
+
}
|
|
462
534
|
/**
|
|
463
535
|
* Whitespace-only normalization for span matching, plus one extraction
|
|
464
536
|
* artifact: the HTML-to-text step can emit a line break before punctuation
|
package/dist/marketClassify.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { type LlmCallOptions } from "./llm.ts";
|
|
2
|
-
import { type CaptureEntry, type MarketClaim, type MarketConfig, type ObservationSet } from "./market.ts";
|
|
2
|
+
import { type CaptureEntry, type MarketClaim, type MarketConfig, type MarketStore, type ObservationSet } from "./market.ts";
|
|
3
|
+
import { type ProgressEmitter } from "./progress.ts";
|
|
3
4
|
export type ClassifyMarketOptions = {
|
|
4
5
|
llm: LlmCallOptions;
|
|
5
6
|
/** Observation run label to produce; must be new (the store is append-only). */
|
|
@@ -10,9 +11,16 @@ export type ClassifyMarketOptions = {
|
|
|
10
11
|
vendors?: string[];
|
|
11
12
|
/** Captures directory override (tests); defaults to the profile market home. */
|
|
12
13
|
capturesDir?: string;
|
|
14
|
+
/** Storage seam: when given, captures are read through it instead of the file layout. */
|
|
15
|
+
store?: MarketStore;
|
|
13
16
|
now?: () => Date;
|
|
14
17
|
/** Per-vendor progress (presentation only — a throwing callback never fails the run). */
|
|
15
18
|
onVendor?: (done: number, total: number, vendorId: string) => void;
|
|
19
|
+
/**
|
|
20
|
+
* Progress emission over MARKET_CAPTURE_STAGES (the "classify" stage, items
|
|
21
|
+
* per vendor). Presentation-only: a throwing listener never fails the run.
|
|
22
|
+
*/
|
|
23
|
+
progress?: ProgressEmitter;
|
|
16
24
|
};
|
|
17
25
|
export type ClassifyMarketResult = {
|
|
18
26
|
set: ObservationSet;
|
package/dist/marketClassify.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { DEFAULT_MODELS, forcedToolCall } from "./llm.js";
|
|
2
2
|
import { loadCaptureTexts, observationId, verifyEvidenceSpans, } from "./market.js";
|
|
3
|
+
import { MARKET_CAPTURE_STAGES, nullProgressEmitter } from "./progress.js";
|
|
3
4
|
/**
|
|
4
5
|
* LLM intensity classification for the market map — the same
|
|
5
6
|
* semi-deterministic posture as call extraction, with one upgrade calls
|
|
@@ -78,7 +79,10 @@ function claimsBlock(claims) {
|
|
|
78
79
|
}
|
|
79
80
|
export async function classifyMarket(config, options) {
|
|
80
81
|
const model = options.llm.model ?? DEFAULT_MODELS[options.llm.provider];
|
|
81
|
-
const
|
|
82
|
+
const progress = options.progress ?? nullProgressEmitter();
|
|
83
|
+
const { entries, textByHash } = options.store
|
|
84
|
+
? await options.store.loadCaptureTexts()
|
|
85
|
+
: loadCaptureTexts(config.category, options.capturesDir);
|
|
82
86
|
if (entries.length === 0) {
|
|
83
87
|
throw new Error(`No captures for ${config.category} — run \`market capture\` first`);
|
|
84
88
|
}
|
|
@@ -92,6 +96,7 @@ export async function classifyMarket(config, options) {
|
|
|
92
96
|
const claimIds = config.claims.map((claim) => claim.id);
|
|
93
97
|
const observations = [];
|
|
94
98
|
const retriedVendorIds = [];
|
|
99
|
+
progress.stage(MARKET_CAPTURE_STAGES[2], 2, MARKET_CAPTURE_STAGES.length);
|
|
95
100
|
for (const [vendorIndex, vendorId] of vendorIds.entries()) {
|
|
96
101
|
try {
|
|
97
102
|
options.onVendor?.(vendorIndex, vendorIds.length, vendorId);
|
|
@@ -99,6 +104,7 @@ export async function classifyMarket(config, options) {
|
|
|
99
104
|
catch {
|
|
100
105
|
// progress is presentation-only
|
|
101
106
|
}
|
|
107
|
+
progress.note(vendorId);
|
|
102
108
|
const vendor = config.vendors.find((candidate) => candidate.id === vendorId);
|
|
103
109
|
if (!vendor)
|
|
104
110
|
throw new Error(`Unknown vendor "${vendorId}"`);
|
|
@@ -120,6 +126,7 @@ export async function classifyMarket(config, options) {
|
|
|
120
126
|
evidence: [],
|
|
121
127
|
});
|
|
122
128
|
}
|
|
129
|
+
progress.items(vendorIndex + 1, vendorIds.length);
|
|
123
130
|
continue;
|
|
124
131
|
}
|
|
125
132
|
const prompt = (feedback) => `${CLASSIFY_INSTRUCTIONS}\n\nSurface rule for this category:\n${config.surfaceRule ?? "(default rule above)"}\n\nClaims to classify (all of them):\n${claimsBlock(config.claims)}\n${feedback}\nVendor: ${vendor.name}\nCaptured pages:\n${dossier}`;
|
|
@@ -165,7 +172,9 @@ export async function classifyMarket(config, options) {
|
|
|
165
172
|
}
|
|
166
173
|
for (const reading of outcome.readings)
|
|
167
174
|
observations.push(toObservation(reading, vendorId));
|
|
175
|
+
progress.items(vendorIndex + 1, vendorIds.length);
|
|
168
176
|
}
|
|
177
|
+
progress.flush();
|
|
169
178
|
return {
|
|
170
179
|
set: {
|
|
171
180
|
id: `set_${config.category}_${options.runLabel}`,
|
package/dist/marketTaxonomy.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { type LlmCallOptions } from "./llm.ts";
|
|
2
|
-
import { type FetchPage, type MarketConfig } from "./market.ts";
|
|
2
|
+
import { type FetchPage, type MarketConfig, type MarketStore } from "./market.ts";
|
|
3
3
|
/**
|
|
4
4
|
* Cold-start taxonomy bootstrap. `market init` writes a stub for a human
|
|
5
5
|
* analyst to fill in; the self-serve hosted map has no analyst in the loop, so
|
|
@@ -30,6 +30,11 @@ export type SuggestTaxonomyOptions = {
|
|
|
30
30
|
/** Test injectables. */
|
|
31
31
|
fetchPage?: FetchPage;
|
|
32
32
|
capturesDir?: string;
|
|
33
|
+
/**
|
|
34
|
+
* Storage seam for the bootstrap captures; defaults to the file layout.
|
|
35
|
+
* Pass createMemoryMarketStore for throwaway grounding (hosted flow).
|
|
36
|
+
*/
|
|
37
|
+
store?: MarketStore;
|
|
33
38
|
now?: () => Date;
|
|
34
39
|
};
|
|
35
40
|
export type SuggestTaxonomyResult = {
|
package/dist/marketTaxonomy.js
CHANGED
|
@@ -147,8 +147,10 @@ export async function suggestMarketConfig(options) {
|
|
|
147
147
|
const anchorId = anchorSeed ? vendors[options.vendors.indexOf(anchorSeed)]?.id : undefined;
|
|
148
148
|
// Capture the seed homepages so the proposer only sees text we actually
|
|
149
149
|
// fetched (the SSRF guard in captureMarket applies to these user-supplied URLs).
|
|
150
|
-
await captureMarket({ category, vendors, claims: [] }, { dir: options.capturesDir, runLabel: "bootstrap", fetchPage: options.fetchPage, now: options.now });
|
|
151
|
-
const capture =
|
|
150
|
+
await captureMarket({ category, vendors, claims: [] }, { dir: options.capturesDir, store: options.store, runLabel: "bootstrap", fetchPage: options.fetchPage, now: options.now });
|
|
151
|
+
const capture = options.store
|
|
152
|
+
? await options.store.loadCaptureTexts()
|
|
153
|
+
: loadCaptureTexts(category, options.capturesDir);
|
|
152
154
|
const { dossier, unreadable } = buildDossier(vendors, capture, perVendorChars);
|
|
153
155
|
if (!dossier.trim()) {
|
|
154
156
|
throw new Error(`market init --auto: none of the ${vendors.length} seed pages returned readable text — check the URLs are public homepages.`);
|
package/dist/mcp-bin.js
CHANGED
|
@@ -1,4 +1,33 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
+
const args = process.argv.slice(2);
|
|
3
|
+
if (args.includes("--help") || args.includes("-h")) {
|
|
4
|
+
console.log(`Usage: fullstackgtm-mcp [--help] [--version]
|
|
5
|
+
|
|
6
|
+
Starts the fullstackgtm MCP stdio server.
|
|
7
|
+
|
|
8
|
+
The MCP server requires optional peer dependencies:
|
|
9
|
+
@modelcontextprotocol/sdk
|
|
10
|
+
zod
|
|
11
|
+
|
|
12
|
+
Zero-install with peers:
|
|
13
|
+
npx -y -p fullstackgtm -p @modelcontextprotocol/sdk -p zod fullstackgtm-mcp`);
|
|
14
|
+
process.exit(0);
|
|
15
|
+
}
|
|
16
|
+
if (args.includes("--version") || args.includes("-v")) {
|
|
17
|
+
console.log(await readPackageVersion());
|
|
18
|
+
process.exit(0);
|
|
19
|
+
}
|
|
20
|
+
async function readPackageVersion() {
|
|
21
|
+
try {
|
|
22
|
+
const { readFileSync } = await import("node:fs");
|
|
23
|
+
const raw = readFileSync(new URL("../package.json", import.meta.url), "utf8");
|
|
24
|
+
const parsed = JSON.parse(raw);
|
|
25
|
+
return parsed.version ?? "unknown";
|
|
26
|
+
}
|
|
27
|
+
catch {
|
|
28
|
+
return "unknown";
|
|
29
|
+
}
|
|
30
|
+
}
|
|
2
31
|
// The MCP server needs the optional peer dependencies. Import dynamically so
|
|
3
32
|
// a missing peer produces install guidance instead of a module-load stack
|
|
4
33
|
// trace — `npx fullstackgtm-mcp` alone never installs optional peers.
|
package/dist/mcp.js
CHANGED
|
@@ -52,6 +52,8 @@ import { createStripeConnector } from "./connectors/stripe.js";
|
|
|
52
52
|
import { getCredential, resolveHubspotAccessToken, resolveSalesforceConnection, } from "./credentials.js";
|
|
53
53
|
import { generateDemoSnapshot } from "./demo.js";
|
|
54
54
|
import { formatPatchPlanRun, patchPlanToMarkdown } from "./format.js";
|
|
55
|
+
import { verifyApprovalDigests } from "./integrity.js";
|
|
56
|
+
import { createFilePlanStore } from "./planStore.js";
|
|
55
57
|
import { builtinAuditRules } from "./rules.js";
|
|
56
58
|
import { sampleSnapshot } from "./sampleData.js";
|
|
57
59
|
import { normalizeTranscript, parseCall } from "./calls.js";
|
|
@@ -270,7 +272,11 @@ const toolDefinitions = [
|
|
|
270
272
|
title: "Apply Approved Patch Operations",
|
|
271
273
|
description: "Apply explicitly approved operations from a patch plan through a provider " +
|
|
272
274
|
"connector. Operations not listed in approvedOperationIds are never written, " +
|
|
273
|
-
"and requires_human_* placeholders need a value override."
|
|
275
|
+
"and requires_human_* placeholders need a value override. When the plan is in " +
|
|
276
|
+
"the local plan store (saved via `audit --save`), approvals are verified against " +
|
|
277
|
+
"the store's HMAC approval digests — ids not approved with `plans approve` are " +
|
|
278
|
+
"refused — and the run is recorded onto the stored plan so `plans show` and " +
|
|
279
|
+
"`audit-log export` include it.",
|
|
274
280
|
inputSchema: {
|
|
275
281
|
provider: z.enum(["hubspot", "salesforce"]),
|
|
276
282
|
planPath: z.string(),
|
|
@@ -280,12 +286,107 @@ const toolDefinitions = [
|
|
|
280
286
|
},
|
|
281
287
|
},
|
|
282
288
|
handler: async ({ provider, planPath, approvedOperationIds, valueOverrides, output }) => {
|
|
283
|
-
|
|
289
|
+
// The file may be a raw PatchPlan (audit --out) or a StoredPlan envelope
|
|
290
|
+
// (a file straight out of the plan-store directory).
|
|
291
|
+
const parsed = JSON.parse(readFileSync(resolve(process.cwd(), planPath), "utf8"));
|
|
292
|
+
const filePlan = isStoredPlanFile(parsed) ? parsed.plan : parsed;
|
|
293
|
+
if (!filePlan || !Array.isArray(filePlan.operations)) {
|
|
294
|
+
throw new Error(`${planPath} is not a patch plan (expected { id, operations: [...] }).`);
|
|
295
|
+
}
|
|
296
|
+
// Governance parity with CLI apply: when this plan id exists in the local
|
|
297
|
+
// plan store, the STORE is the source of truth — the approved-id set must
|
|
298
|
+
// come from `plans approve` (which HMAC-signed each approved op), the
|
|
299
|
+
// signatures are re-verified here, and the run is recorded back onto the
|
|
300
|
+
// plan so audit-log export sees MCP applies exactly like CLI applies.
|
|
301
|
+
const store = createFilePlanStore();
|
|
302
|
+
const stored = typeof filePlan.id === "string" && /^[\w.-]+$/.test(filePlan.id)
|
|
303
|
+
? await store.get(filePlan.id)
|
|
304
|
+
: null;
|
|
305
|
+
let plan;
|
|
306
|
+
let effectiveOverrides;
|
|
307
|
+
if (stored) {
|
|
308
|
+
if (stored.status !== "approved") {
|
|
309
|
+
throw new Error(`Plan ${filePlan.id} is ${stored.status}; approve operations first with ` +
|
|
310
|
+
`\`fullstackgtm plans approve ${filePlan.id} --operations <ids|all>\`.`);
|
|
311
|
+
}
|
|
312
|
+
// Downgrade guard (same as CLI apply): an approved plan with no
|
|
313
|
+
// signatures either predates 0.26 or had approvalDigests stripped to
|
|
314
|
+
// skip the integrity check. Refuse rather than trust the file.
|
|
315
|
+
if (stored.approvedOperationIds.length > 0 && !stored.approvalDigests) {
|
|
316
|
+
throw new Error(`Refusing to apply plan ${filePlan.id}: it was approved without integrity signatures ` +
|
|
317
|
+
"(approved before 0.26.0, or its signatures were removed). Re-approve it with " +
|
|
318
|
+
`\`fullstackgtm plans approve ${filePlan.id} --operations <ids|all>\`.`);
|
|
319
|
+
}
|
|
320
|
+
// Never widen: every id the tool wants applied must already be
|
|
321
|
+
// store-approved. (A subset is fine — narrowing never writes more.)
|
|
322
|
+
const storeApproved = new Set(stored.approvedOperationIds);
|
|
323
|
+
const unapproved = approvedOperationIds.filter((id) => !storeApproved.has(id));
|
|
324
|
+
if (unapproved.length > 0) {
|
|
325
|
+
throw new Error(`Refusing to apply plan ${filePlan.id}: operation(s) ${unapproved.join(", ")} were never ` +
|
|
326
|
+
"approved in the plan store. Approve them first with " +
|
|
327
|
+
`\`fullstackgtm plans approve ${filePlan.id} --operations <ids>\`.`);
|
|
328
|
+
}
|
|
329
|
+
// Integrity gate: verify against the EFFECTIVE overrides (stored ∪ tool
|
|
330
|
+
// args), so a tool-supplied value that changes what the human approved
|
|
331
|
+
// is treated as tamper, not a live override — what gets written must
|
|
332
|
+
// equal what was signed.
|
|
333
|
+
effectiveOverrides = { ...stored.valueOverrides, ...(valueOverrides ?? {}) };
|
|
334
|
+
const verification = verifyApprovalDigests(stored.plan.operations, stored.approvedOperationIds, effectiveOverrides, stored.approvalDigests);
|
|
335
|
+
if (!verification.ok) {
|
|
336
|
+
const detail = verification.reason === "no_key"
|
|
337
|
+
? "the plan-signing key is missing (was this plan approved on another machine?). Re-approve it here with `fullstackgtm plans approve`."
|
|
338
|
+
: `these operations differ from what was approved: ${verification.tampered.join(", ")}. ` +
|
|
339
|
+
"If you want a different value, set it at approval (`plans approve --value <op>=<v>`) and re-approve; " +
|
|
340
|
+
"otherwise the plan was edited after approval — review and re-approve.";
|
|
341
|
+
throw new Error(`Refusing to apply plan ${filePlan.id}: ${detail}`);
|
|
342
|
+
}
|
|
343
|
+
plan = stored.plan; // apply what was signed, not what the file says now
|
|
344
|
+
}
|
|
345
|
+
else {
|
|
346
|
+
// External plan file, not in the store: no digests exist to verify, but
|
|
347
|
+
// approvals must still reference real operations in the plan content.
|
|
348
|
+
plan = filePlan;
|
|
349
|
+
const known = new Set(plan.operations.map((operation) => operation.id));
|
|
350
|
+
const unknown = approvedOperationIds.filter((id) => !known.has(id));
|
|
351
|
+
if (unknown.length > 0) {
|
|
352
|
+
throw new Error(`Plan ${planPath} has no operation(s) ${unknown.join(", ")} — approvedOperationIds ` +
|
|
353
|
+
"must reference operations in the plan.");
|
|
354
|
+
}
|
|
355
|
+
effectiveOverrides = valueOverrides ?? {};
|
|
356
|
+
}
|
|
284
357
|
const run = await applyPatchPlan(await connectorFor(provider), plan, {
|
|
285
358
|
approvedOperationIds,
|
|
286
|
-
valueOverrides,
|
|
359
|
+
valueOverrides: effectiveOverrides,
|
|
287
360
|
});
|
|
288
|
-
|
|
361
|
+
// Persist the run (same data the CLI records) so runs[] and plan status
|
|
362
|
+
// update and audit-log export includes MCP applies. External plan files
|
|
363
|
+
// have no store entry to update — say so instead of silently dropping it.
|
|
364
|
+
let runRecorded = false;
|
|
365
|
+
if (stored) {
|
|
366
|
+
await store.recordRun(plan.id, run);
|
|
367
|
+
runRecorded = true;
|
|
368
|
+
}
|
|
369
|
+
const governance = {
|
|
370
|
+
planSource: stored ? "store" : "external",
|
|
371
|
+
approvalDigestsVerified: stored !== null,
|
|
372
|
+
runRecorded,
|
|
373
|
+
...(stored
|
|
374
|
+
? {}
|
|
375
|
+
: {
|
|
376
|
+
note: "Plan file is not in the local plan store: approvals came from tool arguments " +
|
|
377
|
+
"(verified against the plan content only — no approval-digest check) and the run " +
|
|
378
|
+
"was not recorded. Use `fullstackgtm audit --save` + `plans approve` for " +
|
|
379
|
+
"store-backed governance.",
|
|
380
|
+
}),
|
|
381
|
+
};
|
|
382
|
+
if (output === "markdown") {
|
|
383
|
+
return content(`${formatPatchPlanRun(run)}\n\nGovernance: plan source ${governance.planSource}; ` +
|
|
384
|
+
(runRecorded
|
|
385
|
+
? "approval digests verified; run recorded on the stored plan (audit-log export includes it)."
|
|
386
|
+
: governance.note));
|
|
387
|
+
}
|
|
388
|
+
// Backward compatible: the run's own fields stay top-level; governance is additive.
|
|
389
|
+
return content({ ...run, governance });
|
|
289
390
|
},
|
|
290
391
|
},
|
|
291
392
|
{
|
|
@@ -392,6 +493,18 @@ export async function startMcpServer() {
|
|
|
392
493
|
const transport = new StdioServerTransport();
|
|
393
494
|
await server.connect(transport);
|
|
394
495
|
}
|
|
496
|
+
/**
|
|
497
|
+
* A plan-store file is a StoredPlan envelope ({ plan, status, runs, ... });
|
|
498
|
+
* `audit --out` writes a bare PatchPlan. fullstackgtm_apply accepts either.
|
|
499
|
+
*/
|
|
500
|
+
function isStoredPlanFile(value) {
|
|
501
|
+
if (typeof value !== "object" || value === null || !("plan" in value))
|
|
502
|
+
return false;
|
|
503
|
+
const plan = value.plan;
|
|
504
|
+
return (typeof plan === "object" &&
|
|
505
|
+
plan !== null &&
|
|
506
|
+
Array.isArray(plan.operations));
|
|
507
|
+
}
|
|
395
508
|
function loadMarketConfigOrHint(path) {
|
|
396
509
|
try {
|
|
397
510
|
return loadMarketConfig(path);
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The shared progress-event vocabulary — ONE set of semantics for every
|
|
3
|
+
* long-running process, rendered natively by each surface:
|
|
4
|
+
*
|
|
5
|
+
* CLI → ui.ts (spinner / checklist / apply-ticker; TTY-only styling)
|
|
6
|
+
* app → Convex heartbeat patches → reactive ProgressChip / StageTimeline
|
|
7
|
+
* both → optional broker streaming (a paired CLI POSTs heartbeats to the
|
|
8
|
+
* hosted app under its clientRunId, so long local runs tick live in
|
|
9
|
+
* the org's activity feed)
|
|
10
|
+
*
|
|
11
|
+
* Renderer-agnostic by contract: no event assumes persistence (the CLI drops
|
|
12
|
+
* them after painting) or ANSI (the app can't use it). Listeners are
|
|
13
|
+
* best-effort — a throwing renderer must never fail the work — and the
|
|
14
|
+
* throttle policy lives HERE so both surfaces feel identical.
|
|
15
|
+
*
|
|
16
|
+
* Standing convention: if a verb loops, it emits. New long-running verbs wire
|
|
17
|
+
* an emitter at the package layer; surfaces get feedback for free.
|
|
18
|
+
*/
|
|
19
|
+
export type ProgressEvent = {
|
|
20
|
+
kind: "stage";
|
|
21
|
+
stage: string;
|
|
22
|
+
stageIndex: number;
|
|
23
|
+
stageCount: number;
|
|
24
|
+
} | {
|
|
25
|
+
kind: "items";
|
|
26
|
+
done: number;
|
|
27
|
+
total?: number;
|
|
28
|
+
} | {
|
|
29
|
+
kind: "note";
|
|
30
|
+
note: string;
|
|
31
|
+
} | {
|
|
32
|
+
kind: "opResult";
|
|
33
|
+
opId: string;
|
|
34
|
+
status: "applied" | "skipped" | "failed" | "conflict";
|
|
35
|
+
detail?: string;
|
|
36
|
+
} | {
|
|
37
|
+
kind: "meter";
|
|
38
|
+
spent: number;
|
|
39
|
+
budget: number;
|
|
40
|
+
unit: string;
|
|
41
|
+
};
|
|
42
|
+
/** Accumulated state — what a heartbeat persists / a chip renders. */
|
|
43
|
+
export type ProgressSnapshot = {
|
|
44
|
+
stage?: string;
|
|
45
|
+
stageIndex?: number;
|
|
46
|
+
stageCount?: number;
|
|
47
|
+
itemsDone?: number;
|
|
48
|
+
itemsTotal?: number;
|
|
49
|
+
note?: string;
|
|
50
|
+
opsApplied: number;
|
|
51
|
+
opsSkipped: number;
|
|
52
|
+
opsFailed: number;
|
|
53
|
+
updatedAt: number;
|
|
54
|
+
};
|
|
55
|
+
export type ProgressListener = (event: ProgressEvent, snapshot: ProgressSnapshot) => void;
|
|
56
|
+
export type ProgressEmitter = {
|
|
57
|
+
stage(stage: string, stageIndex: number, stageCount: number): void;
|
|
58
|
+
items(done: number, total?: number): void;
|
|
59
|
+
note(note: string): void;
|
|
60
|
+
opResult(opId: string, status: "applied" | "skipped" | "failed" | "conflict", detail?: string): void;
|
|
61
|
+
meter(spent: number, budget: number, unit: string): void;
|
|
62
|
+
/** Current accumulated state (always fresh, ignoring the throttle). */
|
|
63
|
+
snapshot(): ProgressSnapshot;
|
|
64
|
+
/** Force-deliver the latest state (call at stage ends / completion). */
|
|
65
|
+
flush(): void;
|
|
66
|
+
};
|
|
67
|
+
/**
|
|
68
|
+
* Throttled emitter: `items` heartbeats coalesce to at most one delivery per
|
|
69
|
+
* throttle window (terminal `done === total` always delivers); `stage`,
|
|
70
|
+
* `opResult`, `note`, and `meter` are structural and always deliver.
|
|
71
|
+
*/
|
|
72
|
+
export declare function createProgressEmitter(listener: ProgressListener, options?: {
|
|
73
|
+
throttleMs?: number;
|
|
74
|
+
now?: () => number;
|
|
75
|
+
}): ProgressEmitter;
|
|
76
|
+
/**
|
|
77
|
+
* Fan one event stream out to several listeners (e.g. the CLI's local renderer
|
|
78
|
+
* plus the broker heartbeat streamer). Each listener is isolated: one throwing
|
|
79
|
+
* never starves the others, matching the emitter's own best-effort contract.
|
|
80
|
+
*/
|
|
81
|
+
export declare function composeListeners(...listeners: Array<ProgressListener | undefined | null>): ProgressListener;
|
|
82
|
+
export declare const CRM_SYNC_STAGES: readonly ["owners", "accounts", "contacts", "deals", "counters", "health"];
|
|
83
|
+
/** A connector snapshot pull — the CRM-sync stages that happen CLI-side. */
|
|
84
|
+
export declare const SNAPSHOT_PULL_STAGES: readonly ["owners", "accounts", "contacts", "deals"];
|
|
85
|
+
/** The Stripe connector's snapshot pull (billing systems have no owners). */
|
|
86
|
+
export declare const STRIPE_SNAPSHOT_STAGES: readonly ["customers", "subscriptions"];
|
|
87
|
+
export declare const CALL_SYNC_STAGES: readonly ["notes", "transcripts", "insights"];
|
|
88
|
+
export declare const BACKFILL_STRIPE_STAGES: readonly ["invoices", "snapshot", "matching", "plan"];
|
|
89
|
+
/** `backfill runs` — replaying local plan runs + health history to the broker. */
|
|
90
|
+
export declare const RUNS_REPLAY_STAGES: readonly ["runs", "health"];
|
|
91
|
+
export declare const APPLY_STAGES: readonly ["preflight", "operations", "results"];
|
|
92
|
+
/** `enrich acquire` — routing sourced candidate rows into a create plan. */
|
|
93
|
+
export declare const ACQUIRE_STAGES: readonly ["candidates"];
|
|
94
|
+
export declare const MARKET_CAPTURE_STAGES: readonly ["sources", "capture", "classify", "persist"];
|
|
95
|
+
/** No-op emitter for callers that don't care (keeps signatures simple). */
|
|
96
|
+
export declare function nullProgressEmitter(): ProgressEmitter;
|