fullstackgtm 0.44.0 → 0.46.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (149) hide show
  1. package/CHANGELOG.md +293 -1
  2. package/INSTALL_FOR_AGENTS.md +13 -11
  3. package/README.md +45 -24
  4. package/dist/audit.d.ts +3 -1
  5. package/dist/audit.js +29 -2
  6. package/dist/backfill.d.ts +86 -0
  7. package/dist/backfill.js +251 -0
  8. package/dist/bin.js +4 -1
  9. package/dist/cli/audit.d.ts +15 -0
  10. package/dist/cli/audit.js +392 -0
  11. package/dist/cli/auth.d.ts +82 -0
  12. package/dist/cli/auth.js +607 -0
  13. package/dist/cli/backfill.d.ts +1 -0
  14. package/dist/cli/backfill.js +125 -0
  15. package/dist/cli/backfillRuns.d.ts +1 -0
  16. package/dist/cli/backfillRuns.js +187 -0
  17. package/dist/cli/call.d.ts +1 -0
  18. package/dist/cli/call.js +387 -0
  19. package/dist/cli/capabilities.d.ts +30 -0
  20. package/dist/cli/capabilities.js +197 -0
  21. package/dist/cli/draft.d.ts +7 -0
  22. package/dist/cli/draft.js +87 -0
  23. package/dist/cli/enrich.d.ts +8 -0
  24. package/dist/cli/enrich.js +806 -0
  25. package/dist/cli/fix.d.ts +33 -0
  26. package/dist/cli/fix.js +346 -0
  27. package/dist/cli/help.d.ts +25 -0
  28. package/dist/cli/help.js +646 -0
  29. package/dist/cli/icp.d.ts +7 -0
  30. package/dist/cli/icp.js +229 -0
  31. package/dist/cli/init.d.ts +7 -0
  32. package/dist/cli/init.js +58 -0
  33. package/dist/cli/market.d.ts +1 -0
  34. package/dist/cli/market.js +391 -0
  35. package/dist/cli/plans.d.ts +5 -0
  36. package/dist/cli/plans.js +456 -0
  37. package/dist/cli/schedule.d.ts +8 -0
  38. package/dist/cli/schedule.js +479 -0
  39. package/dist/cli/shared.d.ts +71 -0
  40. package/dist/cli/shared.js +342 -0
  41. package/dist/cli/signals.d.ts +7 -0
  42. package/dist/cli/signals.js +412 -0
  43. package/dist/cli/suggest.d.ts +49 -0
  44. package/dist/cli/suggest.js +135 -0
  45. package/dist/cli/tam.d.ts +9 -0
  46. package/dist/cli/tam.js +387 -0
  47. package/dist/cli/ui.d.ts +142 -0
  48. package/dist/cli/ui.js +427 -0
  49. package/dist/cli.d.ts +1 -57
  50. package/dist/cli.js +123 -5157
  51. package/dist/connector.d.ts +23 -0
  52. package/dist/connector.js +47 -0
  53. package/dist/connectors/hubspot.d.ts +10 -1
  54. package/dist/connectors/hubspot.js +244 -10
  55. package/dist/connectors/hubspotAuth.js +5 -1
  56. package/dist/connectors/linkedin.js +14 -14
  57. package/dist/connectors/salesforce.d.ts +10 -1
  58. package/dist/connectors/salesforce.js +39 -6
  59. package/dist/connectors/signalSources.js +7 -59
  60. package/dist/connectors/stripe.d.ts +37 -0
  61. package/dist/connectors/stripe.js +103 -31
  62. package/dist/enrich.d.ts +7 -0
  63. package/dist/enrich.js +7 -0
  64. package/dist/health.d.ts +11 -69
  65. package/dist/health.js +4 -134
  66. package/dist/healthScore.d.ts +71 -0
  67. package/dist/healthScore.js +143 -0
  68. package/dist/icp.d.ts +15 -11
  69. package/dist/icp.js +19 -36
  70. package/dist/index.d.ts +3 -1
  71. package/dist/index.js +3 -1
  72. package/dist/judge.d.ts +2 -0
  73. package/dist/judge.js +6 -0
  74. package/dist/llm.d.ts +29 -0
  75. package/dist/llm.js +206 -0
  76. package/dist/market.d.ts +39 -1
  77. package/dist/market.js +116 -44
  78. package/dist/marketClassify.d.ts +11 -1
  79. package/dist/marketClassify.js +17 -2
  80. package/dist/marketTaxonomy.d.ts +6 -1
  81. package/dist/marketTaxonomy.js +4 -2
  82. package/dist/mcp-bin.js +31 -2
  83. package/dist/mcp.js +372 -166
  84. package/dist/progress.d.ts +96 -0
  85. package/dist/progress.js +142 -0
  86. package/dist/runReport.d.ts +24 -0
  87. package/dist/runReport.js +139 -4
  88. package/dist/schedule.d.ts +80 -2
  89. package/dist/schedule.js +254 -1
  90. package/dist/spoolFiles.d.ts +44 -0
  91. package/dist/spoolFiles.js +114 -0
  92. package/dist/types.d.ts +29 -1
  93. package/docs/api.md +75 -8
  94. package/docs/architecture.md +2 -0
  95. package/docs/linkedin-connector-spec.md +1 -1
  96. package/docs/signal-spool-format.md +13 -0
  97. package/llms.txt +18 -9
  98. package/package.json +10 -3
  99. package/skills/fullstackgtm/SKILL.md +2 -1
  100. package/src/audit.ts +27 -1
  101. package/src/backfill.ts +340 -0
  102. package/src/bin.ts +5 -1
  103. package/src/cli/audit.ts +447 -0
  104. package/src/cli/auth.ts +669 -0
  105. package/src/cli/backfill.ts +156 -0
  106. package/src/cli/backfillRuns.ts +198 -0
  107. package/src/cli/call.ts +414 -0
  108. package/src/cli/capabilities.ts +215 -0
  109. package/src/cli/draft.ts +101 -0
  110. package/src/cli/enrich.ts +908 -0
  111. package/src/cli/fix.ts +373 -0
  112. package/src/cli/help.ts +702 -0
  113. package/src/cli/icp.ts +265 -0
  114. package/src/cli/init.ts +65 -0
  115. package/src/cli/market.ts +423 -0
  116. package/src/cli/plans.ts +524 -0
  117. package/src/cli/schedule.ts +526 -0
  118. package/src/cli/shared.ts +392 -0
  119. package/src/cli/signals.ts +434 -0
  120. package/src/cli/suggest.ts +151 -0
  121. package/src/cli/tam.ts +435 -0
  122. package/src/cli/ui.ts +497 -0
  123. package/src/cli.ts +122 -5855
  124. package/src/connector.ts +66 -0
  125. package/src/connectors/hubspot.ts +273 -9
  126. package/src/connectors/hubspotAuth.ts +5 -1
  127. package/src/connectors/linkedin.ts +14 -14
  128. package/src/connectors/salesforce.ts +51 -3
  129. package/src/connectors/signalSources.ts +7 -56
  130. package/src/connectors/stripe.ts +154 -34
  131. package/src/enrich.ts +13 -0
  132. package/src/health.ts +14 -213
  133. package/src/healthScore.ts +223 -0
  134. package/src/icp.ts +19 -35
  135. package/src/index.ts +28 -1
  136. package/src/judge.ts +7 -0
  137. package/src/llm.ts +239 -0
  138. package/src/market.ts +157 -44
  139. package/src/marketClassify.ts +26 -2
  140. package/src/marketTaxonomy.ts +10 -2
  141. package/src/mcp-bin.ts +34 -2
  142. package/src/mcp.ts +270 -63
  143. package/src/progress.ts +197 -0
  144. package/src/runReport.ts +159 -4
  145. package/src/schedule.ts +310 -3
  146. package/src/spoolFiles.ts +116 -0
  147. package/src/types.ts +32 -2
  148. package/docs/dx-punch-list.md +0 -87
  149. package/docs/roadmap-to-1.0.md +0 -211
@@ -19,10 +19,11 @@
19
19
  * try/catch idiom of atsBoards/prospectSources.
20
20
  */
21
21
 
22
- import { readFileSync, readdirSync, statSync } from "node:fs";
23
- import { join, resolve } from "node:path";
22
+ import { readFileSync, statSync } from "node:fs";
23
+ import { resolve } from "node:path";
24
24
  import type { SignalBucket, StagedSignalRow } from "../signals.ts";
25
25
  import { SIGNAL_BUCKETS } from "../signals.ts";
26
+ import { parseSpoolText, spoolFilesIn } from "../spoolFiles.ts";
26
27
 
27
28
  export type SignalSourceShape = "pull" | "push";
28
29
  export type SignalSourceAuth = "none" | "api_key" | "oauth";
@@ -99,6 +100,9 @@ export const fileSource: SignalSourceConnector = {
99
100
  } catch {
100
101
  return []; // missing path: an empty source, not a crash
101
102
  }
103
+ // Container parsing lives in the shared spool reader (src/spoolFiles.ts) so
104
+ // `signals fetch --from`, `enrich ingest`, and `market observe --from` read
105
+ // the same convention; the "file source" label keeps error text unchanged.
102
106
  const files = isDir ? spoolFilesIn(abs) : [abs];
103
107
  const rows: Record<string, unknown>[] = [];
104
108
  for (const file of files) {
@@ -108,65 +112,12 @@ export const fileSource: SignalSourceConnector = {
108
112
  } catch {
109
113
  continue; // one unreadable file in a spool dir must not sink the rest
110
114
  }
111
- const trimmed = raw.trim();
112
- if (!trimmed) continue;
113
- const parsed = trimmed.startsWith("[") ? parseJsonArray(trimmed, file) : parseJsonl(trimmed, file);
114
- rows.push(...parsed);
115
+ rows.push(...parseSpoolText(raw, file, "file source"));
115
116
  }
116
117
  return rows.map((row) => coerceRow(row, this.bucket));
117
118
  },
118
119
  };
119
120
 
120
- /** Spool files in a landing-zone directory: `*.jsonl` / `*.json`, name-sorted. */
121
- function spoolFilesIn(dir: string): string[] {
122
- let names: string[];
123
- try {
124
- names = readdirSync(dir);
125
- } catch {
126
- return [];
127
- }
128
- return names
129
- .filter((name) => name.endsWith(".jsonl") || name.endsWith(".json"))
130
- .sort()
131
- .map((name) => join(dir, name));
132
- }
133
-
134
- function parseJsonArray(raw: string, path: string): Record<string, unknown>[] {
135
- let parsed: unknown;
136
- try {
137
- parsed = JSON.parse(raw);
138
- } catch (error) {
139
- throw new Error(`file source ${path}: not valid JSON (${error instanceof Error ? error.message : String(error)}).`);
140
- }
141
- if (!Array.isArray(parsed)) throw new Error(`file source ${path}: expected a JSON array of staged signal rows.`);
142
- return parsed.map((entry, index) => {
143
- if (!entry || typeof entry !== "object" || Array.isArray(entry)) {
144
- throw new Error(`file source ${path}: row ${index} is not an object.`);
145
- }
146
- return entry as Record<string, unknown>;
147
- });
148
- }
149
-
150
- function parseJsonl(raw: string, path: string): Record<string, unknown>[] {
151
- const out: Record<string, unknown>[] = [];
152
- const lines = raw.split("\n");
153
- lines.forEach((line, index) => {
154
- const t = line.trim();
155
- if (!t) return;
156
- let parsed: unknown;
157
- try {
158
- parsed = JSON.parse(t);
159
- } catch (error) {
160
- throw new Error(`file source ${path}: line ${index} is not valid JSON (${error instanceof Error ? error.message : String(error)}).`);
161
- }
162
- if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
163
- throw new Error(`file source ${path}: line ${index} is not a JSON object.`);
164
- }
165
- out.push(parsed as Record<string, unknown>);
166
- });
167
- return out;
168
- }
169
-
170
121
  // ---------------------------------------------------------------------------
171
122
  // serpapi-news — funding/company signals from a news REST query (API key)
172
123
 
@@ -7,6 +7,7 @@ import type {
7
7
  PatchOperation,
8
8
  PatchOperationResult,
9
9
  } from "../types.ts";
10
+ import { STRIPE_SNAPSHOT_STAGES, type ProgressEmitter } from "../progress.ts";
10
11
 
11
12
  const DEFAULT_API_BASE_URL = "https://api.stripe.com";
12
13
 
@@ -16,8 +17,59 @@ export type StripeConnectorOptions = {
16
17
  apiBaseUrl?: string;
17
18
  /** Injectable fetch for testing. */
18
19
  fetchImpl?: typeof fetch;
20
+ /**
21
+ * Shared progress vocabulary (src/progress.ts): the snapshot pull emits a
22
+ * `stage` per collection (customers, subscriptions) and an `items`
23
+ * heartbeat per page. Presentation-only — errors are swallowed.
24
+ */
25
+ progress?: ProgressEmitter;
19
26
  };
20
27
 
28
+ // Module-scope request/list (parameterized by options) so the connector
29
+ // factory and fetchStripePaidInvoices share one implementation.
30
+ async function stripeRequest(options: StripeConnectorOptions, path: string): Promise<any> {
31
+ const baseUrl = (options.apiBaseUrl ?? DEFAULT_API_BASE_URL).replace(/\/$/, "");
32
+ const fetchImpl = options.fetchImpl ?? fetch;
33
+ const apiKey = await options.getApiKey();
34
+ const response = await fetchImpl(`${baseUrl}${path}`, {
35
+ headers: { Authorization: `Bearer ${apiKey}` },
36
+ });
37
+ if (!response.ok) {
38
+ // Status line only — the body can echo request details bound to a live
39
+ // billing key, and these errors land in scheduled-run records.
40
+ await response.text().catch(() => undefined);
41
+ throw new Error(`Stripe API error ${response.status}. Check the restricted key and request.`);
42
+ }
43
+ return response.json();
44
+ }
45
+
46
+ /** Stripe list pagination: follow `has_more` with `starting_after=<last id>`. */
47
+ async function stripeList(
48
+ options: StripeConnectorOptions,
49
+ path: string,
50
+ onPage?: (fetched: number) => void,
51
+ ): Promise<any[]> {
52
+ const results: any[] = [];
53
+ let startingAfter: string | undefined;
54
+ do {
55
+ const separator = path.includes("?") ? "&" : "?";
56
+ const data = await stripeRequest(
57
+ options,
58
+ `${path}${startingAfter ? `${separator}starting_after=${encodeURIComponent(startingAfter)}` : ""}`,
59
+ );
60
+ const page: any[] = data.data ?? [];
61
+ results.push(...page);
62
+ try {
63
+ onPage?.(results.length);
64
+ } catch {
65
+ // progress is presentation-only; never let it fail a pull
66
+ }
67
+ startingAfter =
68
+ data.has_more === true && page.length > 0 ? String(page.at(-1).id) : undefined;
69
+ } while (startingAfter);
70
+ return results;
71
+ }
72
+
21
73
  /**
22
74
  * Read-only billing connector for Stripe — the first non-CRM connector,
23
75
  * proving the GtmConnector contract generalizes to billing systems.
@@ -37,46 +89,24 @@ export type StripeConnectorOptions = {
37
89
  * collections are always empty.
38
90
  */
39
91
  export function createStripeConnector(options: StripeConnectorOptions): GtmConnector {
40
- const baseUrl = (options.apiBaseUrl ?? DEFAULT_API_BASE_URL).replace(/\/$/, "");
41
- const fetchImpl = options.fetchImpl ?? fetch;
42
-
43
- async function request(path: string): Promise<any> {
44
- const apiKey = await options.getApiKey();
45
- const response = await fetchImpl(`${baseUrl}${path}`, {
46
- headers: { Authorization: `Bearer ${apiKey}` },
47
- });
48
- if (!response.ok) {
49
- // Status line only the body can echo request details bound to a live
50
- // billing key, and these errors land in scheduled-run records.
51
- await response.text().catch(() => undefined);
52
- throw new Error(`Stripe API error ${response.status}. Check the restricted key and request.`);
53
- }
54
- return response.json();
55
- }
56
-
57
- /** Stripe list pagination: follow `has_more` with `starting_after=<last id>`. */
58
- async function list(path: string): Promise<any[]> {
59
- const results: any[] = [];
60
- let startingAfter: string | undefined;
61
- do {
62
- const separator = path.includes("?") ? "&" : "?";
63
- const data = await request(
64
- `${path}${startingAfter ? `${separator}starting_after=${encodeURIComponent(startingAfter)}` : ""}`,
65
- );
66
- const page: any[] = data.data ?? [];
67
- results.push(...page);
68
- startingAfter =
69
- data.has_more === true && page.length > 0 ? String(page.at(-1).id) : undefined;
70
- } while (startingAfter);
71
- return results;
72
- }
92
+ const list = (path: string, onPage?: (fetched: number) => void) =>
93
+ stripeList(options, path, onPage);
94
+ // Shared progress vocabulary: stage per collection, items per page.
95
+ const pullStage = (stage: (typeof STRIPE_SNAPSHOT_STAGES)[number]) =>
96
+ options.progress?.stage(
97
+ stage,
98
+ STRIPE_SNAPSHOT_STAGES.indexOf(stage),
99
+ STRIPE_SNAPSHOT_STAGES.length,
100
+ );
101
+ const pullItems = (fetched: number) => options.progress?.items(fetched);
73
102
 
74
103
  async function assembleSnapshot(createdGte?: number): Promise<CanonicalGtmSnapshot> {
75
104
  // Stripe filters list endpoints on the integer `created` field via
76
105
  // created[gte]=<unix seconds>; omitted on a full snapshot.
77
106
  const createdFilter =
78
107
  createdGte === undefined ? "" : `&created[gte]=${createdGte}`;
79
- const customers = await list(`/v1/customers?limit=100${createdFilter}`);
108
+ pullStage("customers");
109
+ const customers = await list(`/v1/customers?limit=100${createdFilter}`, pullItems);
80
110
  const accounts: CanonicalAccount[] = [];
81
111
  const contacts: CanonicalContact[] = [];
82
112
  for (const customer of customers) {
@@ -111,8 +141,10 @@ export function createStripeConnector(options: StripeConnectorOptions): GtmConne
111
141
  }
112
142
  }
113
143
 
144
+ pullStage("subscriptions");
114
145
  const subscriptions = await list(
115
146
  `/v1/subscriptions?status=all&limit=100${createdFilter}`,
147
+ pullItems,
116
148
  );
117
149
  const deals: CanonicalDeal[] = subscriptions
118
150
  .filter((subscription) => subscription.id)
@@ -161,6 +193,9 @@ export function createStripeConnector(options: StripeConnectorOptions): GtmConne
161
193
  };
162
194
  });
163
195
 
196
+ // Deliver any throttled trailing items heartbeat before the pull returns.
197
+ options.progress?.flush();
198
+
164
199
  return {
165
200
  generatedAt: new Date().toISOString(),
166
201
  provider: "stripe",
@@ -205,6 +240,91 @@ export function createStripeConnector(options: StripeConnectorOptions): GtmConne
205
240
  };
206
241
  }
207
242
 
243
+ /**
244
+ * One PAID Stripe invoice, normalized for the backfill plan builder
245
+ * (`backfill stripe`). Amounts are major units (Stripe returns cents);
246
+ * `paidAt` is the payment date (YYYY-MM-DD) from status_transitions.paid_at.
247
+ */
248
+ export type StripePaidInvoice = {
249
+ id: string;
250
+ number?: string;
251
+ customerId?: string;
252
+ customerName?: string;
253
+ customerEmail?: string;
254
+ /** Lowercased domain of the customer's billing email — the CRM match key. */
255
+ customerDomain?: string;
256
+ /** Major currency units (amount_paid / 100). */
257
+ amountPaid: number;
258
+ currency?: string;
259
+ /** YYYY-MM-DD from status_transitions.paid_at (unix seconds). */
260
+ paidAt?: string;
261
+ description?: string;
262
+ };
263
+
264
+ /**
265
+ * Read-only pull of PAID invoices (GET /v1/invoices?status=paid with the
266
+ * customer expanded), the revenue source of truth the backfill plan builder
267
+ * turns into closed-won deal proposals. Invoices with a zero amount_paid or
268
+ * no id are skipped — a $0 invoice is not a won deal.
269
+ */
270
+ export async function fetchStripePaidInvoices(
271
+ options: StripeConnectorOptions,
272
+ opts: { sinceIso?: string; onPage?: (fetched: number) => void } = {},
273
+ ): Promise<StripePaidInvoice[]> {
274
+ let createdFilter = "";
275
+ if (opts.sinceIso !== undefined) {
276
+ const sinceMs = Date.parse(opts.sinceIso);
277
+ if (!Number.isFinite(sinceMs)) throw new Error(`Invalid since timestamp: ${opts.sinceIso}`);
278
+ createdFilter = `&created[gte]=${Math.floor(sinceMs / 1000)}`;
279
+ }
280
+ const invoices = await stripeList(
281
+ options,
282
+ `/v1/invoices?status=paid&limit=100&expand[]=data.customer${createdFilter}`,
283
+ opts.onPage,
284
+ );
285
+ const paid: StripePaidInvoice[] = [];
286
+ for (const invoice of invoices) {
287
+ if (!invoice?.id) continue;
288
+ const amountPaidCents = numberOrUndefined(invoice.amount_paid);
289
+ if (amountPaidCents === undefined || amountPaidCents <= 0) continue;
290
+ // customer is the expanded object when expand[]=data.customer was honored,
291
+ // else the plain id string — handle both.
292
+ const customer = invoice.customer;
293
+ const customerId =
294
+ typeof customer === "string" ? customer : stringOrUndefined(customer?.id);
295
+ // Invoices also carry flat customer_name/customer_email copies — use them
296
+ // when the customer object wasn't expanded (or lacks the field).
297
+ const customerName =
298
+ (typeof customer === "object" && customer !== null
299
+ ? stringOrUndefined(customer.name)
300
+ : undefined) ?? stringOrUndefined(invoice.customer_name);
301
+ const customerEmail =
302
+ typeof customer === "object" && customer !== null
303
+ ? stringOrUndefined(customer.email) ?? stringOrUndefined(invoice.customer_email)
304
+ : stringOrUndefined(invoice.customer_email);
305
+ const customerDomain = customerEmail?.includes("@")
306
+ ? customerEmail.split("@").at(-1)!.toLowerCase()
307
+ : undefined;
308
+ const paidAtSeconds = numberOrUndefined(invoice.status_transitions?.paid_at);
309
+ paid.push({
310
+ id: String(invoice.id),
311
+ number: stringOrUndefined(invoice.number),
312
+ customerId,
313
+ customerName,
314
+ customerEmail,
315
+ customerDomain,
316
+ amountPaid: amountPaidCents / 100,
317
+ currency: stringOrUndefined(invoice.currency)?.toUpperCase(),
318
+ paidAt:
319
+ paidAtSeconds !== undefined
320
+ ? new Date(paidAtSeconds * 1000).toISOString().split("T")[0]
321
+ : undefined,
322
+ description: stringOrUndefined(invoice.description),
323
+ });
324
+ }
325
+ return paid;
326
+ }
327
+
208
328
  function stringOrUndefined(value: unknown): string | undefined {
209
329
  if (value === undefined || value === null || value === "") return undefined;
210
330
  return String(value);
package/src/enrich.ts CHANGED
@@ -3,6 +3,7 @@ import { join } from "node:path";
3
3
  import { credentialsDir, ensureSecureHomeDir, writeSecureFile } from "./credentials.ts";
4
4
  import { HUBSPOT_DEFAULT_FIELD_MAPPINGS } from "./mappings.ts";
5
5
  import type { AcquireBudget } from "./acquireMeter.ts";
6
+ import type { ProgressEmitter } from "./progress.ts";
6
7
  import { parseAssignmentPolicy, resolveAssignment } from "./assign.ts";
7
8
  import type { AssignmentContext, AssignmentPolicy } from "./assign.ts";
8
9
  import type {
@@ -1028,6 +1029,12 @@ export type BuildAcquirePlanOptions = {
1028
1029
  /** Meter ceiling: max create ops to emit. null/undefined = unlimited. */
1029
1030
  maxRecords?: number | null;
1030
1031
  now?: () => Date;
1032
+ /**
1033
+ * Shared progress vocabulary (src/progress.ts): `items` over the candidate
1034
+ * rows as they are routed, plus a `meter` reading of creates vs the meter
1035
+ * ceiling. Presentation-only — never changes what the plan proposes.
1036
+ */
1037
+ progress?: ProgressEmitter;
1031
1038
  };
1032
1039
 
1033
1040
  /** First non-empty value among several candidate paths in a source payload. */
@@ -1120,7 +1127,10 @@ export function buildAcquirePlan(options: BuildAcquirePlanOptions): AcquirePlanR
1120
1127
  }
1121
1128
  }
1122
1129
 
1130
+ let routed = 0;
1123
1131
  for (const record of records) {
1132
+ // Progress heartbeat over the candidate rows (throttled by the emitter).
1133
+ options.progress?.items((routed += 1), records.length);
1124
1134
  const createMap = acquire.create[record.objectType];
1125
1135
  const match = config.match[record.objectType];
1126
1136
  // No create mapping or no match config for this type — can neither create
@@ -1235,6 +1245,9 @@ export function buildAcquirePlan(options: BuildAcquirePlanOptions): AcquirePlanR
1235
1245
  counts.created += 1;
1236
1246
  estCostUsd += costPerRecord;
1237
1247
  }
1248
+ options.progress?.flush();
1249
+ // Meter reading: creates proposed this run against the meter's ceiling.
1250
+ if (cap !== null) options.progress?.meter(counts.created, cap, "creates");
1238
1251
 
1239
1252
  const plan: PatchPlan = {
1240
1253
  id: `patch_plan_acq_${fnv1a(`${source}:${runLabel}:${nowIso}`)}`,
package/src/health.ts CHANGED
@@ -2,7 +2,20 @@ import { appendFileSync, mkdirSync, readFileSync } from "node:fs";
2
2
  import { join } from "node:path";
3
3
 
4
4
  import { activeProfile, credentialsDir, ensureSecureHomeDir, writeSecureFile } from "./credentials.ts";
5
- import type { AuditFindingSeverity, CanonicalGtmSnapshot, PatchPlan } from "./types.ts";
5
+ import type { CanonicalGtmSnapshot } from "./types.ts";
6
+ import type { HealthEntry } from "./healthScore.ts";
7
+
8
+ // Pure scoring lives in healthScore.ts (node-free, importable from V8 runtimes
9
+ // like the hosted app's Convex queries); this module owns the fs-backed
10
+ // profile timeline. Re-exported so existing imports keep working.
11
+ export {
12
+ computeHealth,
13
+ summarizeHealth,
14
+ healthToMarkdown,
15
+ type HealthEntry,
16
+ type HealthRuleDelta,
17
+ type HealthRollup,
18
+ } from "./healthScore.ts";
6
19
 
7
20
  /**
8
21
  * The engagement workspace: a per-client health timeline that accrues from the
@@ -15,218 +28,6 @@ import type { AuditFindingSeverity, CanonicalGtmSnapshot, PatchPlan } from "./ty
15
28
  * so it is stable in CI and comparable run-over-run.
16
29
  */
17
30
 
18
- // Severity weights: a critical finding costs ~3× a warning, ~10× an info.
19
- const SEVERITY_WEIGHT: Record<AuditFindingSeverity, number> = { info: 1, warning: 3, critical: 10 };
20
-
21
- export type HealthEntry = {
22
- /** ISO timestamp of the audit that produced this entry. */
23
- at: string;
24
- /** The saved plan this entry summarizes (correlates to plans/ and snapshots/). */
25
- planId: string;
26
- /** Deterministic 0–100 hygiene score (higher is healthier). */
27
- score: number;
28
- /** Total finding count across all rules. */
29
- findings: number;
30
- /** Severity-weighted finding total (drives the score). */
31
- weightedFindings: number;
32
- /** Record counts at audit time — the denominator that normalizes the score. */
33
- records: { accounts: number; contacts: number; deals: number; total: number };
34
- /** Finding count per rule id. */
35
- byRule: Record<string, number>;
36
- /** Finding count per severity. */
37
- severityCounts: Record<AuditFindingSeverity, number>;
38
- /**
39
- * Per-object-type breakdown — so "is my contact data clean but my pipeline
40
- * messy?" is answerable without re-auditing. Each type carries its own
41
- * record-normalized score (same curve as the overall score, scoped to that
42
- * type's records + findings).
43
- */
44
- byObjectType: Record<"account" | "contact" | "deal", {
45
- records: number;
46
- findings: number;
47
- weightedFindings: number;
48
- score: number;
49
- }>;
50
- };
51
-
52
- export type HealthRuleDelta = {
53
- ruleId: string;
54
- current: number;
55
- previous: number;
56
- /** current − previous: positive = more findings (worse), negative = fewer (better). */
57
- delta: number;
58
- };
59
-
60
- export type HealthRollup = {
61
- profile: string;
62
- auditCount: number;
63
- first: string;
64
- latest: string;
65
- current: HealthEntry;
66
- previous: HealthEntry | null;
67
- /** current.score − previous.score: positive = improving. null on the first audit. */
68
- scoreDelta: number | null;
69
- ruleDeltas: HealthRuleDelta[];
70
- history: Array<{ at: string; score: number; findings: number }>;
71
- };
72
-
73
- /**
74
- * Deterministically score a saved audit. score = 100 / (1 + weighted findings
75
- * per record): 0 findings → 100; one weighted finding per record → 50; the
76
- * curve is bounded (0, 100], monotonic, and needs no clamping or magic
77
- * constants. A messy 50-record CRM scores worse than the same finding count
78
- * across 5,000 records, which is the point.
79
- */
80
- export function computeHealth(plan: PatchPlan, snapshot: CanonicalGtmSnapshot, at: string): HealthEntry {
81
- const byRule: Record<string, number> = {};
82
- const severityCounts: Record<AuditFindingSeverity, number> = { info: 0, warning: 0, critical: 0 };
83
- const typeTally: Record<"account" | "contact" | "deal", { findings: number; weightedFindings: number }> = {
84
- account: { findings: 0, weightedFindings: 0 },
85
- contact: { findings: 0, weightedFindings: 0 },
86
- deal: { findings: 0, weightedFindings: 0 },
87
- };
88
- let weightedFindings = 0;
89
- for (const finding of plan.findings) {
90
- byRule[finding.ruleId] = (byRule[finding.ruleId] ?? 0) + 1;
91
- severityCounts[finding.severity] += 1;
92
- weightedFindings += SEVERITY_WEIGHT[finding.severity];
93
- const t = finding.objectType;
94
- if (t === "account" || t === "contact" || t === "deal") {
95
- typeTally[t].findings += 1;
96
- typeTally[t].weightedFindings += SEVERITY_WEIGHT[finding.severity];
97
- }
98
- }
99
-
100
- const records = {
101
- accounts: snapshot.accounts.length,
102
- contacts: snapshot.contacts.length,
103
- deals: snapshot.deals.length,
104
- total: snapshot.accounts.length + snapshot.contacts.length + snapshot.deals.length,
105
- };
106
-
107
- const scoreFor = (weighted: number, recordCount: number) =>
108
- Math.round(100 / (1 + weighted / Math.max(recordCount, 1)));
109
- const recordsByType = { account: records.accounts, contact: records.contacts, deal: records.deals };
110
- const byObjectType = (["account", "contact", "deal"] as const).reduce(
111
- (acc, t) => {
112
- acc[t] = {
113
- records: recordsByType[t],
114
- findings: typeTally[t].findings,
115
- weightedFindings: typeTally[t].weightedFindings,
116
- score: scoreFor(typeTally[t].weightedFindings, recordsByType[t]),
117
- };
118
- return acc;
119
- },
120
- {} as HealthEntry["byObjectType"],
121
- );
122
-
123
- const score = scoreFor(weightedFindings, records.total);
124
-
125
- return {
126
- at,
127
- planId: plan.id,
128
- score,
129
- findings: plan.findings.length,
130
- weightedFindings,
131
- records,
132
- byRule,
133
- severityCounts,
134
- byObjectType,
135
- };
136
- }
137
-
138
- /** Roll a timeline up into the current state, the change since last time, and per-rule deltas. */
139
- export function summarizeHealth(entries: HealthEntry[], profile: string): HealthRollup | null {
140
- if (entries.length === 0) return null;
141
- // Oldest → newest, so `current` is the last entry.
142
- const sorted = [...entries].sort((a, b) => a.at.localeCompare(b.at));
143
- const current = sorted[sorted.length - 1];
144
- const previous = sorted.length > 1 ? sorted[sorted.length - 2] : null;
145
-
146
- const ruleIds = new Set<string>([
147
- ...Object.keys(current.byRule),
148
- ...(previous ? Object.keys(previous.byRule) : []),
149
- ]);
150
- const ruleDeltas: HealthRuleDelta[] = [...ruleIds]
151
- .map((ruleId) => {
152
- const cur = current.byRule[ruleId] ?? 0;
153
- const prev = previous?.byRule[ruleId] ?? 0;
154
- return { ruleId, current: cur, previous: prev, delta: cur - prev };
155
- })
156
- .sort((a, b) => b.current - a.current || a.ruleId.localeCompare(b.ruleId));
157
-
158
- return {
159
- profile,
160
- auditCount: sorted.length,
161
- first: sorted[0].at,
162
- latest: current.at,
163
- current,
164
- previous,
165
- scoreDelta: previous ? current.score - previous.score : null,
166
- ruleDeltas,
167
- history: sorted.map((entry) => ({ at: entry.at, score: entry.score, findings: entry.findings })),
168
- };
169
- }
170
-
171
- // Sign-based and uniform: ▲ = the number went up, ▼ = down. The reader supplies
172
- // the meaning (score up = good; findings up = bad) — mixing arrow direction with
173
- // good/bad reads as contradictory ("▲ -3").
174
- function arrow(delta: number): string {
175
- if (delta === 0) return "·";
176
- return delta > 0 ? "▲" : "▼";
177
- }
178
-
179
- /** Human-readable rollup: headline score + trend, then per-rule change since last audit. */
180
- export function healthToMarkdown(rollup: HealthRollup): string {
181
- const { current, scoreDelta } = rollup;
182
- const lines = [`# GTM health — profile \`${rollup.profile}\``, ""];
183
-
184
- const deltaText =
185
- scoreDelta === null
186
- ? "(first audit — no prior reading)"
187
- : `${arrow(scoreDelta)} ${scoreDelta >= 0 ? "+" : ""}${scoreDelta} since the previous audit`;
188
- lines.push(
189
- `Score: **${current.score}/100** ${deltaText}`,
190
- `Audits: ${rollup.auditCount} over ${shortDate(rollup.first)} → ${shortDate(rollup.latest)}`,
191
- `Findings: ${current.findings} (${current.severityCounts.critical} critical, ${current.severityCounts.warning} warning, ${current.severityCounts.info} info)`,
192
- `Records: ${current.records.accounts} accounts, ${current.records.contacts} contacts, ${current.records.deals} deals`,
193
- "",
194
- );
195
-
196
- lines.push("## By object type", "", "| Type | Score | Findings | Records |", "| --- | --- | --- | --- |");
197
- for (const t of ["account", "contact", "deal"] as const) {
198
- const b = current.byObjectType[t];
199
- lines.push(`| ${t} | ${b.score}/100 | ${b.findings} | ${b.records} |`);
200
- }
201
- lines.push("");
202
-
203
- if (rollup.history.length > 1) {
204
- lines.push("## Trend", "");
205
- for (const point of rollup.history) {
206
- const marker = point.at === rollup.latest ? " ← latest" : "";
207
- lines.push(`- ${shortDate(point.at)} score ${point.score} (${point.findings} findings)${marker}`);
208
- }
209
- lines.push("");
210
- }
211
-
212
- if (rollup.ruleDeltas.length > 0) {
213
- lines.push("## By rule (Δ since previous audit)", "", "| Rule | Findings | Δ |", "| --- | --- | --- |");
214
- for (const rule of rollup.ruleDeltas) {
215
- const sign = rule.delta > 0 ? `+${rule.delta}` : `${rule.delta}`;
216
- const cell = rule.delta === 0 ? "·" : `${arrow(rule.delta)} ${sign}`;
217
- lines.push(`| ${rule.ruleId} | ${rule.current} | ${cell} |`);
218
- }
219
- lines.push("");
220
- }
221
-
222
- lines.push("> Score is deterministic: 100 / (1 + severity-weighted findings per record). Lower findings or more clean records ⇒ higher score.");
223
- return `${lines.join("\n")}\n`;
224
- }
225
-
226
- function shortDate(iso: string): string {
227
- return iso.slice(0, 10);
228
- }
229
-
230
31
  // ── Profile-scoped storage ──────────────────────────────────────────────────
231
32
 
232
33
  /** `$FSGTM_HOME[/profiles/<name>]/health.jsonl` — append-only, one entry per audit-save. */