fullstackgtm 0.40.0 → 0.42.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +57 -0
- package/README.md +30 -1
- package/dist/cli.js +615 -6
- package/dist/connector.js +34 -0
- package/dist/connectors/atsBoards.d.ts +60 -0
- package/dist/connectors/atsBoards.js +179 -0
- package/dist/connectors/hubspot.js +124 -0
- package/dist/connectors/prospectSources.d.ts +7 -5
- package/dist/connectors/prospectSources.js +82 -47
- package/dist/connectors/salesforce.js +112 -0
- package/dist/draft.d.ts +182 -0
- package/dist/draft.js +333 -0
- package/dist/index.d.ts +5 -0
- package/dist/index.js +5 -0
- package/dist/judge.d.ts +209 -0
- package/dist/judge.js +490 -0
- package/dist/judgeEval.d.ts +141 -0
- package/dist/judgeEval.js +331 -0
- package/dist/llm.d.ts +13 -0
- package/dist/llm.js +18 -2
- package/dist/schedule.js +11 -1
- package/dist/signals.d.ts +197 -0
- package/dist/signals.js +515 -0
- package/dist/types.d.ts +11 -0
- package/docs/api.md +8 -1
- package/package.json +1 -1
- package/src/cli.ts +696 -7
- package/src/connector.ts +37 -0
- package/src/connectors/atsBoards.ts +242 -0
- package/src/connectors/hubspot.ts +125 -0
- package/src/connectors/prospectSources.ts +94 -43
- package/src/connectors/salesforce.ts +113 -0
- package/src/draft.ts +463 -0
- package/src/index.ts +94 -0
- package/src/judge.ts +661 -0
- package/src/judgeEval.ts +459 -0
- package/src/llm.ts +31 -2
- package/src/schedule.ts +11 -1
- package/src/signals.ts +685 -0
- package/src/types.ts +11 -0
package/src/cli.ts
CHANGED
|
@@ -148,6 +148,46 @@ import {
|
|
|
148
148
|
INTERVIEW_SPEC,
|
|
149
149
|
type Icp,
|
|
150
150
|
} from "./icp.ts";
|
|
151
|
+
import {
|
|
152
|
+
buildSignalsFromAts,
|
|
153
|
+
computeWeights,
|
|
154
|
+
createFileSignalStore,
|
|
155
|
+
DEFAULT_SIGNALS_CONFIG,
|
|
156
|
+
dedupeSignals,
|
|
157
|
+
loadSignalsConfig,
|
|
158
|
+
makeOutcome,
|
|
159
|
+
normalizeAccountDomain,
|
|
160
|
+
SIGNAL_BUCKETS,
|
|
161
|
+
signalRunId,
|
|
162
|
+
signalId,
|
|
163
|
+
type Signal,
|
|
164
|
+
type SignalBucket,
|
|
165
|
+
type SignalOutcomeResult,
|
|
166
|
+
type SignalsConfig,
|
|
167
|
+
} from "./signals.ts";
|
|
168
|
+
import { fetchAtsJobs, type AtsBoardSource, type AtsJob } from "./connectors/atsBoards.ts";
|
|
169
|
+
import {
|
|
170
|
+
createFileJudgeStore,
|
|
171
|
+
DEFAULT_JUDGE_PROMPT,
|
|
172
|
+
judgeRunId,
|
|
173
|
+
judgeSignals,
|
|
174
|
+
} from "./judge.ts";
|
|
175
|
+
import {
|
|
176
|
+
authorOpeners,
|
|
177
|
+
DEFAULT_DRAFT_PROMPT,
|
|
178
|
+
DRAFT_CHANNELS,
|
|
179
|
+
draft,
|
|
180
|
+
type DraftChannel,
|
|
181
|
+
} from "./draft.ts";
|
|
182
|
+
import {
|
|
183
|
+
DEFAULT_GOLDEN_SET,
|
|
184
|
+
DEFAULT_MIN_ACCURACY,
|
|
185
|
+
defaultJudgeFn,
|
|
186
|
+
gradeAgainstOutcomes,
|
|
187
|
+
gradeJudge,
|
|
188
|
+
parseGoldenSet,
|
|
189
|
+
} from "./judgeEval.ts";
|
|
190
|
+
import type { LlmCallOptions } from "./llm.ts";
|
|
151
191
|
import {
|
|
152
192
|
apolloPullKeysForAppend,
|
|
153
193
|
apolloPullKeysForRefresh,
|
|
@@ -250,6 +290,29 @@ Usage:
|
|
|
250
290
|
fill-blanks-only patch plan through the normal dry-run →
|
|
251
291
|
approve → apply gate. refresh re-checks stale stamped fields
|
|
252
292
|
and proposes updates only where the source value changed.
|
|
293
|
+
fullstackgtm signals fetch [--bucket job,…] [--source greenhouse,lever,ashby] [--watchlist <path|crm:seg>] [--keywords …] [--from <file.json>] [--save]
|
|
294
|
+
fullstackgtm signals list [--since 7d] [--bucket b] [--account d] [--unjudged]
|
|
295
|
+
fullstackgtm signals outcome --account <d> [--touch <id>] --result replied|meeting|bounced|no_reply
|
|
296
|
+
fullstackgtm signals weights [--explain]
|
|
297
|
+
detect fresh buying triggers (ATS hiring scrapes + staged
|
|
298
|
+
funding/company/social ingest), rank them, persist a local
|
|
299
|
+
signal ledger. READ-ONLY re: CRM — fetch NEVER emits a plan;
|
|
300
|
+
--save persists only the signal ledger. outcome feeds the
|
|
301
|
+
learned per-bucket weights.
|
|
302
|
+
fullstackgtm icp interview | set <answers.json> | show
|
|
303
|
+
fullstackgtm icp judge [--signals-from latest|<label>] [--with-history] [--prompt <path>] [--min-score 0] [--save]
|
|
304
|
+
fullstackgtm icp eval [--golden <path>|default] [--against-outcomes] [--min-accuracy 0.8] [--json]
|
|
305
|
+
build the ICP, then judge fresh signals into
|
|
306
|
+
send/nurture/skip (timing × fit × memory). Deterministic
|
|
307
|
+
baseline; LLM why-now + play with a key (gated verbatim).
|
|
308
|
+
eval grades the judge and exits 2 below the bar — the
|
|
309
|
+
probabilistic-judgment gate. Read-only; --save writes only
|
|
310
|
+
the local judge store.
|
|
311
|
+
fullstackgtm draft [--from-judge latest|<label>] [--min-score 80] [--prompt <path>] [--channel email|linkedin|task] [--save]
|
|
312
|
+
author one trigger-grounded opener per hot judge decision as
|
|
313
|
+
a governed create_task plan. First line must contain a
|
|
314
|
+
verbatim span of the why-now or the draft is rejected. Never
|
|
315
|
+
sends — --save stages a needs_approval plan for approve → apply.
|
|
253
316
|
fullstackgtm bulk-update <account|contact|deal> --where <expr> [--where …] (--set <field>=<value> [--set …] | --archive [--force-archive-duplicates] | --create-task <text>) [--require <field>=<value> …] [--guard <object>:<where>[;<where>]:<none|some> …] [source options] [--save] [--json] [--out <path>]
|
|
254
317
|
governed generic writes: filter the snapshot
|
|
255
318
|
(field=value, field!=value, field~substr, field!~substr,
|
|
@@ -622,11 +685,48 @@ const HELP: Record<string, HelpEntry> = {
|
|
|
622
685
|
"Capture vendor pages (content-addressed), classify intensity per claim (LLM bring-your-own-key, or fill the worksheet with any agent), then compute deterministic front states and drift. Every quoted span is verified verbatim against the stored capture before it's accepted.",
|
|
623
686
|
seeAlso: [],
|
|
624
687
|
},
|
|
688
|
+
|
|
689
|
+
// Outbound intelligence — signals → judge → draft (the GTM brain)
|
|
690
|
+
signals: {
|
|
691
|
+
summary: "detect fresh buying triggers (ATS hiring + staged ingest), ranked",
|
|
692
|
+
phase: "Detect",
|
|
693
|
+
synopsis: [
|
|
694
|
+
"fullstackgtm signals fetch [--bucket job,…] [--source greenhouse,lever,ashby] [--watchlist <path|crm:seg>] [--keywords …] [--from <file.json>] [--save]",
|
|
695
|
+
"fullstackgtm signals list [--since 7d] [--bucket b] [--account d] [--unjudged]",
|
|
696
|
+
"fullstackgtm signals outcome --account <d> [--touch <id>] --result replied|meeting|bounced|no_reply",
|
|
697
|
+
"fullstackgtm signals weights [--explain]",
|
|
698
|
+
],
|
|
699
|
+
detail:
|
|
700
|
+
"Read-only re: CRM — `fetch` NEVER emits a patch plan; `--save` persists only the local signal ledger. ATS adapters are no-auth. `outcome` feeds the learned per-bucket `weights`.",
|
|
701
|
+
seeAlso: ["icp", "draft"],
|
|
702
|
+
},
|
|
703
|
+
icp: {
|
|
704
|
+
summary: "build the ICP, then judge fresh signals into send/nurture/skip",
|
|
705
|
+
phase: "Detect",
|
|
706
|
+
synopsis: [
|
|
707
|
+
"fullstackgtm icp interview | set <answers.json> | show",
|
|
708
|
+
"fullstackgtm icp judge [--signals-from latest|<label>] [--with-history] [--prompt <path>] [--min-score 0] [--save]",
|
|
709
|
+
"fullstackgtm icp eval [--golden <path>|default] [--against-outcomes] [--min-accuracy 0.8] [--json]",
|
|
710
|
+
],
|
|
711
|
+
detail:
|
|
712
|
+
"`judge` ranks unjudged signals by timing × fit × memory (deterministic baseline; LLM why-now + play with a key, gated verbatim). `eval` is the probabilistic-judgment gate — exits 2 below the bar. Read-only; --save writes only the local judge store.",
|
|
713
|
+
seeAlso: ["signals", "draft", "enrich"],
|
|
714
|
+
},
|
|
715
|
+
draft: {
|
|
716
|
+
summary: "author one trigger-grounded opener per hot decision as a governed plan",
|
|
717
|
+
phase: "Remediate",
|
|
718
|
+
synopsis: [
|
|
719
|
+
"fullstackgtm draft [--from-judge latest|<label>] [--min-score 80] [--prompt <path>] [--channel email|linkedin|task] [--save]",
|
|
720
|
+
],
|
|
721
|
+
detail:
|
|
722
|
+
"Takes hot judge decisions (send, score>=min) and stages ONE create_task opener each. The first line must contain a verbatim span of the why-now or the draft is rejected. Never sends — --save stages a needs_approval plan for `plans approve` -> `apply`.",
|
|
723
|
+
seeAlso: ["icp", "plans", "apply"],
|
|
724
|
+
},
|
|
625
725
|
};
|
|
626
726
|
|
|
627
727
|
// Verbs that print their own richer multi-subcommand help; runCli routes their
|
|
628
728
|
// `--help` to themselves, so commandHelp() only renders these via `help <verb>`.
|
|
629
|
-
const BESPOKE_HELP = ["call", "market", "enrich", "bulk-update", "schedule"];
|
|
729
|
+
const BESPOKE_HELP = ["call", "market", "enrich", "bulk-update", "schedule", "signals", "icp", "draft"];
|
|
630
730
|
|
|
631
731
|
// Lifecycle-grouped front door. One line per verb, organized by the
|
|
632
732
|
// Prevent→Detect→Remediate→Verify loop so a new user sees ~6 jobs, not 22 verbs.
|
|
@@ -639,6 +739,7 @@ function shortUsage() {
|
|
|
639
739
|
["Calls → evidence", ["call"]],
|
|
640
740
|
["Govern — the plan/apply spine", ["suggest", "plans", "apply", "audit-log", "merge"]],
|
|
641
741
|
["Market intelligence", ["market"]],
|
|
742
|
+
["Outbound — signals → judge → draft", ["signals", "icp", "draft"]],
|
|
642
743
|
["Schedule — make it continuous", ["schedule"]],
|
|
643
744
|
];
|
|
644
745
|
const pad = Math.max(...Object.keys(HELP).map((k) => k.length)) + 2;
|
|
@@ -1365,9 +1466,13 @@ score always needs a key (scoring is LLM work).`);
|
|
|
1365
1466
|
*/
|
|
1366
1467
|
async function requireLlmCredential(
|
|
1367
1468
|
command: "parse" | "score" | "market classify" = "parse",
|
|
1368
|
-
): Promise<{ provider: LlmProvider; apiKey: string }> {
|
|
1469
|
+
): Promise<{ provider: LlmProvider; apiKey: string; anthropicBaseUrl?: string; openaiBaseUrl?: string }> {
|
|
1470
|
+
// Base-URL overrides are resolved here (not inside forcedToolCall) so the LLM
|
|
1471
|
+
// seam stays pure/injectable. Spread into LlmCallOptions at every call site
|
|
1472
|
+
// via `...credential`; unset env leaves the upstream defaults untouched.
|
|
1473
|
+
const baseUrls = resolveLlmBaseUrls();
|
|
1369
1474
|
const resolved = resolveLlmCredential();
|
|
1370
|
-
if (resolved) return resolved;
|
|
1475
|
+
if (resolved) return { ...resolved, ...baseUrls };
|
|
1371
1476
|
// Scoring is inherently LLM work — there is no keyword fallback to suggest.
|
|
1372
1477
|
const fallbackHint =
|
|
1373
1478
|
command === "parse"
|
|
@@ -1391,7 +1496,22 @@ async function requireLlmCredential(
|
|
|
1391
1496
|
const now = new Date().toISOString();
|
|
1392
1497
|
storeCredential(provider, { kind: "api_key", accessToken: apiKey, createdAt: now, updatedAt: now });
|
|
1393
1498
|
console.error(`Stored ${provider} key (${validation.detail}). Future runs use it automatically; remove with \`fullstackgtm logout ${provider}\`.\n`);
|
|
1394
|
-
return { provider, apiKey };
|
|
1499
|
+
return { provider, apiKey, ...baseUrls };
|
|
1500
|
+
}
|
|
1501
|
+
|
|
1502
|
+
/**
|
|
1503
|
+
* Read optional LLM base-URL overrides from env so the package can run against
|
|
1504
|
+
* an Anthropic/OpenAI-compatible gateway (GLM-5.2, z.ai, Ollama) without code
|
|
1505
|
+
* changes. Returns only the keys that are set; an empty object when neither is,
|
|
1506
|
+
* so spreading into LlmCallOptions is a no-op by default.
|
|
1507
|
+
*/
|
|
1508
|
+
function resolveLlmBaseUrls(
|
|
1509
|
+
env: Record<string, string | undefined> = process.env,
|
|
1510
|
+
): { anthropicBaseUrl?: string; openaiBaseUrl?: string } {
|
|
1511
|
+
const out: { anthropicBaseUrl?: string; openaiBaseUrl?: string } = {};
|
|
1512
|
+
if (env.ANTHROPIC_API_BASE_URL) out.anthropicBaseUrl = env.ANTHROPIC_API_BASE_URL;
|
|
1513
|
+
if (env.OPENAI_API_BASE_URL) out.openaiBaseUrl = env.OPENAI_API_BASE_URL;
|
|
1514
|
+
return out;
|
|
1395
1515
|
}
|
|
1396
1516
|
|
|
1397
1517
|
function renderScorecard(scorecard: CallScorecard, title?: string): string {
|
|
@@ -2319,6 +2439,407 @@ function loadIcp(args: string[]): Icp | undefined {
|
|
|
2319
2439
|
return parseIcp(readFileSync(path, "utf8"));
|
|
2320
2440
|
}
|
|
2321
2441
|
|
|
2442
|
+
/**
|
|
2443
|
+
* Resolve a signals config: explicit --config, else signals.config.json in cwd,
|
|
2444
|
+
* else the zero-config DEFAULT_SIGNALS_CONFIG (preset-first, like enrich).
|
|
2445
|
+
*/
|
|
2446
|
+
function resolveSignalsConfig(args: string[]): SignalsConfig {
|
|
2447
|
+
const explicit = option(args, "--config");
|
|
2448
|
+
if (explicit) return loadSignalsConfig(resolve(process.cwd(), explicit));
|
|
2449
|
+
const local = resolve(process.cwd(), "signals.config.json");
|
|
2450
|
+
if (existsSync(local)) return loadSignalsConfig(local);
|
|
2451
|
+
return DEFAULT_SIGNALS_CONFIG;
|
|
2452
|
+
}
|
|
2453
|
+
|
|
2454
|
+
/** A watchlist entry: an account domain plus optional per-source board tokens. */
|
|
2455
|
+
type WatchlistAccount = { domain: string; boards?: Partial<Record<AtsBoardSource, string>> };
|
|
2456
|
+
|
|
2457
|
+
/**
|
|
2458
|
+
* Resolve the watchlist of accounts to scan. Sources, in precedence order:
|
|
2459
|
+
* - a --watchlist file (JSON array of {domain, boards?} or bare domain strings),
|
|
2460
|
+
* - a `crm:<segment>` token (derive account domains from a CRM snapshot, scoped
|
|
2461
|
+
* by the segment's `field=value` where-clauses; reuses readSnapshot),
|
|
2462
|
+
* - config.watchlist.domains.
|
|
2463
|
+
* Board tokens come from the entry's own `boards`, else config.watchlist.boards.
|
|
2464
|
+
*/
|
|
2465
|
+
async function resolveWatchlist(
|
|
2466
|
+
args: string[],
|
|
2467
|
+
config: SignalsConfig,
|
|
2468
|
+
): Promise<WatchlistAccount[]> {
|
|
2469
|
+
const configBoards = config.watchlist.boards ?? {};
|
|
2470
|
+
const fromConfig = (domain: string): WatchlistAccount => {
|
|
2471
|
+
const normalized = normalizeAccountDomain(domain);
|
|
2472
|
+
const token = configBoards[normalized] ?? configBoards[domain];
|
|
2473
|
+
return token ? { domain: normalized, boards: { greenhouse: token, lever: token, ashby: token } } : { domain: normalized };
|
|
2474
|
+
};
|
|
2475
|
+
|
|
2476
|
+
const watchlistArg = option(args, "--watchlist") ?? config.watchlist.source;
|
|
2477
|
+
if (watchlistArg && watchlistArg.startsWith("crm:")) {
|
|
2478
|
+
const segment = watchlistArg.slice("crm:".length);
|
|
2479
|
+
const snapshot = await readSnapshot(args);
|
|
2480
|
+
// Segment is an optional `;`-separated list of `field=value` equality
|
|
2481
|
+
// clauses over snapshot accounts (the common case; richer filtering lives in
|
|
2482
|
+
// bulk-update). Empty segment = every account with a domain.
|
|
2483
|
+
const clauses = (segment ? segment.split(";").map((s) => s.trim()).filter(Boolean) : []).map((clause) => {
|
|
2484
|
+
const eq = clause.indexOf("=");
|
|
2485
|
+
if (eq === -1) throw new Error(`--watchlist crm:<segment>: clause "${clause}" must be field=value.`);
|
|
2486
|
+
return { field: clause.slice(0, eq).trim(), value: clause.slice(eq + 1).trim() };
|
|
2487
|
+
});
|
|
2488
|
+
const accounts = (snapshot.accounts ?? []).filter((acct) =>
|
|
2489
|
+
clauses.every((c) => {
|
|
2490
|
+
const raw = (acct as Record<string, unknown>)[c.field];
|
|
2491
|
+
return (raw == null ? "" : String(raw)).toLowerCase() === c.value.toLowerCase();
|
|
2492
|
+
}),
|
|
2493
|
+
);
|
|
2494
|
+
const domains = accounts
|
|
2495
|
+
.map((acct) => normalizeAccountDomain(acct.domain ?? ""))
|
|
2496
|
+
.filter((d): d is string => Boolean(d));
|
|
2497
|
+
return [...new Set(domains)].map(fromConfig);
|
|
2498
|
+
}
|
|
2499
|
+
if (watchlistArg) {
|
|
2500
|
+
const raw = JSON.parse(readFileSync(resolve(process.cwd(), watchlistArg), "utf8")) as unknown;
|
|
2501
|
+
if (!Array.isArray(raw)) throw new Error(`--watchlist ${watchlistArg}: expected a JSON array of domains or {domain, boards?} objects`);
|
|
2502
|
+
return raw.map((entry): WatchlistAccount => {
|
|
2503
|
+
if (typeof entry === "string") return fromConfig(entry);
|
|
2504
|
+
if (entry && typeof entry === "object" && typeof (entry as { domain?: unknown }).domain === "string") {
|
|
2505
|
+
const e = entry as { domain: string; boards?: Partial<Record<AtsBoardSource, string>> };
|
|
2506
|
+
const base = fromConfig(e.domain);
|
|
2507
|
+
return e.boards ? { domain: base.domain, boards: { ...base.boards, ...e.boards } } : base;
|
|
2508
|
+
}
|
|
2509
|
+
throw new Error(`--watchlist ${watchlistArg}: each entry must be a domain string or {domain, boards?} object`);
|
|
2510
|
+
});
|
|
2511
|
+
}
|
|
2512
|
+
return (config.watchlist.domains ?? []).map(fromConfig);
|
|
2513
|
+
}
|
|
2514
|
+
|
|
2515
|
+
/**
|
|
2516
|
+
* `signals` — detect fresh buying triggers (ATS job-board scrapes + staged
|
|
2517
|
+
* funding/company/social ingest), rank them, and persist a local signal ledger.
|
|
2518
|
+
* READ-ONLY re: CRM: NEVER emits a PatchPlan, never calls a recording connector.
|
|
2519
|
+
* `--save` persists only to the signal store, not a plan.
|
|
2520
|
+
*/
|
|
2521
|
+
async function signalsCommand(args: string[]) {
|
|
2522
|
+
const [sub, ...rest] = args;
|
|
2523
|
+
// Help-before-network: catch --help/-h BEFORE any config load or fetch —
|
|
2524
|
+
// anywhere in argv (`signals --help` and `signals fetch --help` both land here).
|
|
2525
|
+
if (!sub || args.includes("--help") || args.includes("-h")) {
|
|
2526
|
+
console.log(`Usage:
|
|
2527
|
+
fullstackgtm signals fetch [--bucket job,funding,...] [--source greenhouse,lever,ashby] [--watchlist <path|crm:segment>] [--keywords "growth,revops"] [--from <file.json>] [--config <path>] [--save]
|
|
2528
|
+
fullstackgtm signals list [--since 7d] [--bucket b] [--account d] [--unjudged]
|
|
2529
|
+
fullstackgtm signals outcome --account <domain> [--touch <id>] --result replied|meeting|bounced|no_reply
|
|
2530
|
+
fullstackgtm signals weights [--explain]
|
|
2531
|
+
|
|
2532
|
+
Detect fresh buying triggers and rank them. \`fetch\` is read-only re: CRM — it
|
|
2533
|
+
NEVER emits a patch plan; --save persists only the local signal ledger (used by
|
|
2534
|
+
\`icp judge\`). ATS adapters are no-auth, so no credential flags exist.`);
|
|
2535
|
+
return;
|
|
2536
|
+
}
|
|
2537
|
+
|
|
2538
|
+
if (sub === "fetch") {
|
|
2539
|
+
const config = resolveSignalsConfig(rest);
|
|
2540
|
+
// Merge --keywords into the job bucket; filter buckets + job sources.
|
|
2541
|
+
const keywordsArg = option(rest, "--keywords");
|
|
2542
|
+
if (keywordsArg) {
|
|
2543
|
+
const merged = keywordsArg.split(",").map((k) => k.trim()).filter(Boolean);
|
|
2544
|
+
config.buckets.job = { ...config.buckets.job, keywords: merged };
|
|
2545
|
+
}
|
|
2546
|
+
const bucketFilter = option(rest, "--bucket");
|
|
2547
|
+
const buckets: SignalBucket[] = bucketFilter
|
|
2548
|
+
? (bucketFilter.split(",").map((b) => b.trim()).filter(Boolean) as SignalBucket[])
|
|
2549
|
+
: SIGNAL_BUCKETS.filter((b) => (config.buckets[b]?.sources.length ?? 0) > 0);
|
|
2550
|
+
for (const b of buckets) {
|
|
2551
|
+
if (!SIGNAL_BUCKETS.includes(b)) throw new Error(`Unknown bucket: ${b} (one of ${SIGNAL_BUCKETS.join(", ")})`);
|
|
2552
|
+
}
|
|
2553
|
+
const sourceFilter = option(rest, "--source");
|
|
2554
|
+
const allJobSources: AtsBoardSource[] = ["greenhouse", "lever", "ashby"];
|
|
2555
|
+
const jobSources = sourceFilter
|
|
2556
|
+
? (sourceFilter.split(",").map((s) => s.trim()).filter(Boolean) as AtsBoardSource[]).filter((s) =>
|
|
2557
|
+
allJobSources.includes(s),
|
|
2558
|
+
)
|
|
2559
|
+
: (config.buckets.job.sources.filter((s) => allJobSources.includes(s as AtsBoardSource)) as AtsBoardSource[]);
|
|
2560
|
+
const now = new Date();
|
|
2561
|
+
|
|
2562
|
+
const store = createFileSignalStore();
|
|
2563
|
+
const priorSignals = await store.allSignals();
|
|
2564
|
+
const candidates: Signal[] = [];
|
|
2565
|
+
|
|
2566
|
+
// Job bucket: scan ATS boards for each watchlist account x job source.
|
|
2567
|
+
if (buckets.includes("job")) {
|
|
2568
|
+
const watchlist = await resolveWatchlist(rest, config);
|
|
2569
|
+
for (const account of watchlist) {
|
|
2570
|
+
const rawJobs: Array<AtsJob & { source: AtsBoardSource }> = [];
|
|
2571
|
+
for (const source of jobSources) {
|
|
2572
|
+
const boardToken = account.boards?.[source] ?? account.domain;
|
|
2573
|
+
const jobs = await fetchAtsJobs({
|
|
2574
|
+
source,
|
|
2575
|
+
boardToken,
|
|
2576
|
+
accountDomain: account.domain,
|
|
2577
|
+
keywords: config.buckets.job.keywords,
|
|
2578
|
+
});
|
|
2579
|
+
for (const job of jobs) rawJobs.push({ ...job, source });
|
|
2580
|
+
}
|
|
2581
|
+
const accountSignals = buildSignalsFromAts(rawJobs, { domain: account.domain }, config, {
|
|
2582
|
+
now,
|
|
2583
|
+
priorSignals,
|
|
2584
|
+
});
|
|
2585
|
+
candidates.push(...accountSignals);
|
|
2586
|
+
}
|
|
2587
|
+
}
|
|
2588
|
+
|
|
2589
|
+
// Staged ingest (funding/company/social): --from <file.json>.
|
|
2590
|
+
const fromFile = option(rest, "--from");
|
|
2591
|
+
if (fromFile) {
|
|
2592
|
+
const ingested = readStagedSignals(resolve(process.cwd(), fromFile), buckets, now);
|
|
2593
|
+
candidates.push(...ingested);
|
|
2594
|
+
}
|
|
2595
|
+
|
|
2596
|
+
const fetched = candidates.length;
|
|
2597
|
+
const { fresh, deduped } = dedupeSignals(candidates, priorSignals, config.dedupWindowDays, now);
|
|
2598
|
+
const ranked = [...fresh].sort((a, b) => b.weight - a.weight || a.accountDomain.localeCompare(b.accountDomain));
|
|
2599
|
+
|
|
2600
|
+
// Ranked fresh signals to stdout; guidance to stderr.
|
|
2601
|
+
console.log(JSON.stringify(ranked, null, 2));
|
|
2602
|
+
console.error(
|
|
2603
|
+
`Fetched ${fetched} candidate signal(s); ${fresh.length} fresh, ${deduped.length} deduped ` +
|
|
2604
|
+
`(window ${config.dedupWindowDays}d). NO plan emitted — signals are read-only re: CRM.`,
|
|
2605
|
+
);
|
|
2606
|
+
|
|
2607
|
+
const save = rest.includes("--save");
|
|
2608
|
+
if (save) {
|
|
2609
|
+
const runLabel = option(rest, "--label") ?? `signals-${now.toISOString().slice(0, 10)}`;
|
|
2610
|
+
const startedAt = now.toISOString();
|
|
2611
|
+
await store.appendRun({
|
|
2612
|
+
id: signalRunId(runLabel),
|
|
2613
|
+
runLabel,
|
|
2614
|
+
startedAt,
|
|
2615
|
+
completedAt: new Date().toISOString(),
|
|
2616
|
+
buckets,
|
|
2617
|
+
counts: { fetched, new: fresh.length, deduped: deduped.length },
|
|
2618
|
+
signals: ranked,
|
|
2619
|
+
});
|
|
2620
|
+
console.error(`Saved signal run "${runLabel}" (${fresh.length} fresh). Next: \`fullstackgtm icp judge --save\`.`);
|
|
2621
|
+
} else {
|
|
2622
|
+
console.error("(not saved — re-run with --save to persist this run to the signal ledger)");
|
|
2623
|
+
}
|
|
2624
|
+
return;
|
|
2625
|
+
}
|
|
2626
|
+
|
|
2627
|
+
if (sub === "list") {
|
|
2628
|
+
const store = createFileSignalStore();
|
|
2629
|
+
const all = await store.allSignals();
|
|
2630
|
+
const sinceArg = option(rest, "--since");
|
|
2631
|
+
const sinceMs = sinceArg ? parseSinceWindowMs(sinceArg) : undefined;
|
|
2632
|
+
const now = Date.now();
|
|
2633
|
+
const bucket = option(rest, "--bucket");
|
|
2634
|
+
const accountArg = option(rest, "--account");
|
|
2635
|
+
const account = accountArg ? normalizeAccountDomain(accountArg) : undefined;
|
|
2636
|
+
const unjudgedOnly = rest.includes("--unjudged");
|
|
2637
|
+
const filtered = all.filter((s) => {
|
|
2638
|
+
if (sinceMs !== undefined) {
|
|
2639
|
+
const seen = Date.parse(s.firstSeen);
|
|
2640
|
+
if (!Number.isFinite(seen) || now - seen > sinceMs) return false;
|
|
2641
|
+
}
|
|
2642
|
+
if (bucket && s.bucket !== bucket) return false;
|
|
2643
|
+
if (account && s.accountDomain !== account) return false;
|
|
2644
|
+
if (unjudgedOnly && s.judgedBy) return false;
|
|
2645
|
+
return true;
|
|
2646
|
+
});
|
|
2647
|
+
const ranked = filtered.sort((a, b) => b.weight - a.weight || a.accountDomain.localeCompare(b.accountDomain));
|
|
2648
|
+
console.log(JSON.stringify(ranked, null, 2));
|
|
2649
|
+
console.error(`${ranked.length} signal(s)${unjudgedOnly ? " (unjudged)" : ""}.`);
|
|
2650
|
+
return;
|
|
2651
|
+
}
|
|
2652
|
+
|
|
2653
|
+
if (sub === "outcome") {
|
|
2654
|
+
const accountArg = option(rest, "--account");
|
|
2655
|
+
if (!accountArg) throw new Error("signals outcome: --account <domain> is required.");
|
|
2656
|
+
const result = option(rest, "--result");
|
|
2657
|
+
const valid: SignalOutcomeResult[] = ["replied", "meeting", "bounced", "no_reply"];
|
|
2658
|
+
if (!result || !valid.includes(result as SignalOutcomeResult)) {
|
|
2659
|
+
throw new Error(`signals outcome: --result must be one of ${valid.join(", ")}.`);
|
|
2660
|
+
}
|
|
2661
|
+
const touchId = option(rest, "--touch") ?? undefined;
|
|
2662
|
+
const outcome = makeOutcome({ accountDomain: accountArg, touchId, result: result as SignalOutcomeResult });
|
|
2663
|
+
await createFileSignalStore().appendOutcome(outcome);
|
|
2664
|
+
console.error(
|
|
2665
|
+
`Recorded outcome "${outcome.result}" for ${outcome.accountDomain} (${outcome.id}). ` +
|
|
2666
|
+
`Re-run \`fullstackgtm signals weights\` to see the learned shift.`,
|
|
2667
|
+
);
|
|
2668
|
+
return;
|
|
2669
|
+
}
|
|
2670
|
+
|
|
2671
|
+
if (sub === "weights") {
|
|
2672
|
+
const config = resolveSignalsConfig(rest);
|
|
2673
|
+
const store = createFileSignalStore();
|
|
2674
|
+
const outcomes = await store.listOutcomes();
|
|
2675
|
+
const signalsById = new Map((await store.allSignals()).map((s) => [s.id, s]));
|
|
2676
|
+
const weights = computeWeights(config, outcomes, signalsById);
|
|
2677
|
+
console.log(JSON.stringify(weights, null, 2));
|
|
2678
|
+
if (rest.includes("--explain")) {
|
|
2679
|
+
// Per-bucket config default vs learned + booked/total over credited signals.
|
|
2680
|
+
const booked = new Map<SignalBucket, number>();
|
|
2681
|
+
const total = new Map<SignalBucket, number>();
|
|
2682
|
+
for (const outcome of outcomes) {
|
|
2683
|
+
for (const id of outcome.creditedSignals) {
|
|
2684
|
+
const signal = signalsById.get(id);
|
|
2685
|
+
if (!signal) continue;
|
|
2686
|
+
total.set(signal.bucket, (total.get(signal.bucket) ?? 0) + 1);
|
|
2687
|
+
if (outcome.result === "meeting") booked.set(signal.bucket, (booked.get(signal.bucket) ?? 0) + 1);
|
|
2688
|
+
}
|
|
2689
|
+
}
|
|
2690
|
+
console.error("Per-bucket weight (config default vs learned, booked/total credited):");
|
|
2691
|
+
for (const b of SIGNAL_BUCKETS) {
|
|
2692
|
+
console.error(
|
|
2693
|
+
` ${b.padEnd(8)} default ${config.buckets[b].weight.toFixed(2)} -> learned ${weights[b].toFixed(4)} ` +
|
|
2694
|
+
`(${booked.get(b) ?? 0}/${total.get(b) ?? 0} booked)`,
|
|
2695
|
+
);
|
|
2696
|
+
}
|
|
2697
|
+
}
|
|
2698
|
+
return;
|
|
2699
|
+
}
|
|
2700
|
+
|
|
2701
|
+
throw new Error(`Unknown signals subcommand: ${sub} (try: fetch, list, outcome, weights)`);
|
|
2702
|
+
}
|
|
2703
|
+
|
|
2704
|
+
/** Parse a "7d"/"30d"/"12h" recency window into milliseconds. */
|
|
2705
|
+
function parseSinceWindowMs(value: string): number {
|
|
2706
|
+
const match = /^(\d+)\s*([dhwm])$/i.exec(value.trim());
|
|
2707
|
+
if (!match) throw new Error(`--since: expected a window like 7d, 24h, 2w (got "${value}").`);
|
|
2708
|
+
const n = Number(match[1]);
|
|
2709
|
+
const unit = match[2].toLowerCase();
|
|
2710
|
+
const ms = unit === "h" ? 3600_000 : unit === "w" ? 7 * 86_400_000 : unit === "m" ? 30 * 86_400_000 : 86_400_000;
|
|
2711
|
+
return n * ms;
|
|
2712
|
+
}
|
|
2713
|
+
|
|
2714
|
+
/**
|
|
2715
|
+
* Read staged funding/company/social signals from a JSON file (array of partial
|
|
2716
|
+
* signals). Each row is validated against the signal schema + the quote gate
|
|
2717
|
+
* (a non-empty verbatim quote is required); source = "ingest".
|
|
2718
|
+
*/
|
|
2719
|
+
function readStagedSignals(path: string, buckets: SignalBucket[], now: Date): Signal[] {
|
|
2720
|
+
const raw = JSON.parse(readFileSync(path, "utf8")) as unknown;
|
|
2721
|
+
if (!Array.isArray(raw)) throw new Error(`--from ${path}: expected a JSON array of staged signals.`);
|
|
2722
|
+
const nowIso = now.toISOString();
|
|
2723
|
+
const out: Signal[] = [];
|
|
2724
|
+
raw.forEach((entry, index) => {
|
|
2725
|
+
if (!entry || typeof entry !== "object") throw new Error(`--from ${path}: row ${index} is not an object.`);
|
|
2726
|
+
const e = entry as Record<string, unknown>;
|
|
2727
|
+
const bucket = String(e.bucket ?? "");
|
|
2728
|
+
if (!SIGNAL_BUCKETS.includes(bucket as SignalBucket)) {
|
|
2729
|
+
throw new Error(`--from ${path}: row ${index} has unknown bucket "${bucket}" (one of ${SIGNAL_BUCKETS.join(", ")}).`);
|
|
2730
|
+
}
|
|
2731
|
+
if (buckets.length && !buckets.includes(bucket as SignalBucket)) return; // filtered out by --bucket
|
|
2732
|
+
const accountDomain = normalizeAccountDomain(String(e.accountDomain ?? e.domain ?? ""));
|
|
2733
|
+
if (!accountDomain) throw new Error(`--from ${path}: row ${index} is missing accountDomain.`);
|
|
2734
|
+
const trigger = String(e.trigger ?? "").trim();
|
|
2735
|
+
if (!trigger) throw new Error(`--from ${path}: row ${index} is missing trigger.`);
|
|
2736
|
+
const quote = String(e.quote ?? "").trim();
|
|
2737
|
+
if (!quote) throw new Error(`--from ${path}: row ${index} is missing the verbatim quote (the evidence anchor).`);
|
|
2738
|
+
const base = { accountDomain, bucket: bucket as SignalBucket, trigger };
|
|
2739
|
+
const firstSeen = typeof e.firstSeen === "string" && e.firstSeen ? e.firstSeen : nowIso;
|
|
2740
|
+
out.push({
|
|
2741
|
+
id: signalId(base),
|
|
2742
|
+
accountDomain,
|
|
2743
|
+
bucket: bucket as SignalBucket,
|
|
2744
|
+
trigger,
|
|
2745
|
+
quote,
|
|
2746
|
+
sourceUrl: String(e.sourceUrl ?? ""),
|
|
2747
|
+
firstSeen,
|
|
2748
|
+
weight: typeof e.weight === "number" ? e.weight : DEFAULT_SIGNALS_CONFIG.buckets[bucket as SignalBucket].weight,
|
|
2749
|
+
source: "ingest",
|
|
2750
|
+
judgedBy: null,
|
|
2751
|
+
});
|
|
2752
|
+
});
|
|
2753
|
+
return out;
|
|
2754
|
+
}
|
|
2755
|
+
|
|
2756
|
+
/**
|
|
2757
|
+
* `draft` — author ONE trigger-grounded opener per hot judge decision as a
|
|
2758
|
+
* governed create_task plan. Structurally a proposal: never sends, never writes
|
|
2759
|
+
* a CRM record. With --save the plan is staged needs_approval for plans approve
|
|
2760
|
+
* -> apply.
|
|
2761
|
+
*/
|
|
2762
|
+
async function draftCommand(args: string[]) {
|
|
2763
|
+
// Help-before-network: catch --help/-h BEFORE any config/credential/LLM call.
|
|
2764
|
+
if (args.includes("--help") || args.includes("-h")) {
|
|
2765
|
+
console.log(`Usage:
|
|
2766
|
+
fullstackgtm draft [--from-judge latest|<runLabel>] [--min-score 80] [--prompt <path>] [--channel email|linkedin|task] [--save]
|
|
2767
|
+
|
|
2768
|
+
Author one trigger-grounded opener per hot judge decision (decision=send,
|
|
2769
|
+
score>=min) as a governed create_task plan. The opener's first line must contain
|
|
2770
|
+
a verbatim span of the why-now trigger or the draft is rejected. Never sends —
|
|
2771
|
+
--save stages a needs_approval plan for \`plans approve\` -> \`apply\`. Without an
|
|
2772
|
+
LLM key it emits a clearly-labeled stub rather than fake authored copy.`);
|
|
2773
|
+
return;
|
|
2774
|
+
}
|
|
2775
|
+
|
|
2776
|
+
const fromJudge = option(args, "--from-judge") ?? "latest";
|
|
2777
|
+
const minScore = numericOption(args, "--min-score") ?? 80;
|
|
2778
|
+
const promptPath = option(args, "--prompt");
|
|
2779
|
+
const channelArg = option(args, "--channel") ?? "task";
|
|
2780
|
+
if (!DRAFT_CHANNELS.includes(channelArg as DraftChannel)) {
|
|
2781
|
+
throw new Error(`--channel must be one of ${DRAFT_CHANNELS.join(", ")} (got "${channelArg}").`);
|
|
2782
|
+
}
|
|
2783
|
+
const channel = channelArg as DraftChannel;
|
|
2784
|
+
const save = args.includes("--save");
|
|
2785
|
+
|
|
2786
|
+
// Load the JudgeRun.
|
|
2787
|
+
const judgeStore = createFileJudgeStore();
|
|
2788
|
+
const judgeRun = fromJudge === "latest" ? await judgeStore.latestRun() : await judgeStore.getRun(fromJudge);
|
|
2789
|
+
if (!judgeRun) {
|
|
2790
|
+
throw new Error(`No judge run "${fromJudge}" — run \`fullstackgtm icp judge --save\` first.`);
|
|
2791
|
+
}
|
|
2792
|
+
|
|
2793
|
+
// Rebuild the signalsById map the openers ground in: from the referenced signal
|
|
2794
|
+
// run, falling back to every stored signal.
|
|
2795
|
+
const signalStore = createFileSignalStore();
|
|
2796
|
+
const signalRun = await signalStore.getRun(judgeRun.signalRunLabel);
|
|
2797
|
+
const signals = signalRun ? signalRun.signals : await signalStore.allSignals();
|
|
2798
|
+
const signalsById = new Map(signals.map((s) => [s.id, s]));
|
|
2799
|
+
|
|
2800
|
+
// LLM resolution (optional — no-key path is the honestly-degraded stub).
|
|
2801
|
+
const cred = resolveLlmCredential();
|
|
2802
|
+
const model = option(args, "--model") ?? undefined;
|
|
2803
|
+
const llm: LlmCallOptions | undefined = cred
|
|
2804
|
+
? { ...cred, ...resolveLlmBaseUrls(), ...(model ? { model } : {}) }
|
|
2805
|
+
: undefined;
|
|
2806
|
+
const promptTemplate = promptPath ? readFileSync(resolve(process.cwd(), promptPath), "utf8") : DEFAULT_DRAFT_PROMPT;
|
|
2807
|
+
|
|
2808
|
+
const openers = await authorOpeners({
|
|
2809
|
+
decisions: judgeRun.decisions,
|
|
2810
|
+
signalsById,
|
|
2811
|
+
minScore,
|
|
2812
|
+
promptTemplate,
|
|
2813
|
+
llm,
|
|
2814
|
+
});
|
|
2815
|
+
const { plan, drafts, rejected } = draft({
|
|
2816
|
+
decisions: judgeRun.decisions,
|
|
2817
|
+
signalsById,
|
|
2818
|
+
minScore,
|
|
2819
|
+
channel,
|
|
2820
|
+
openers,
|
|
2821
|
+
});
|
|
2822
|
+
|
|
2823
|
+
console.log(JSON.stringify({ drafts, rejected }, null, 2));
|
|
2824
|
+
const stale = drafts.filter((d) => d.staleTrigger);
|
|
2825
|
+
console.error(
|
|
2826
|
+
`${drafts.length} opener(s) staged as create_task proposals (channel ${channel}, min score ${minScore})` +
|
|
2827
|
+
`${rejected.length ? `; ${rejected.length} rejected (ungrounded first line)` : ""}` +
|
|
2828
|
+
`${stale.length ? `; ${stale.length} flagged staleTrigger` : ""}` +
|
|
2829
|
+
`${llm ? "" : " — DETERMINISTIC stub (no LLM key)"}. A draft is a proposal; nothing is sent.`,
|
|
2830
|
+
);
|
|
2831
|
+
|
|
2832
|
+
if (save) {
|
|
2833
|
+
await createFilePlanStore().save(plan);
|
|
2834
|
+
console.error(
|
|
2835
|
+
`Saved plan ${plan.id} (status ${plan.status}). Review: \`fullstackgtm plans show ${plan.id}\`, ` +
|
|
2836
|
+
`then \`fullstackgtm plans approve ${plan.id} --operations all\` and \`fullstackgtm apply --plan-id ${plan.id} --provider <name>\`.`,
|
|
2837
|
+
);
|
|
2838
|
+
} else {
|
|
2839
|
+
console.error("(not saved — re-run with --save to stage the plan for plans approve -> apply)");
|
|
2840
|
+
}
|
|
2841
|
+
}
|
|
2842
|
+
|
|
2322
2843
|
/**
|
|
2323
2844
|
* `icp` — develop and inspect the Ideal Customer Profile that targets acquire.
|
|
2324
2845
|
* The CLI can't run AskUserQuestion itself; `icp interview` emits the question
|
|
@@ -2327,16 +2848,22 @@ function loadIcp(args: string[]): Icp | undefined {
|
|
|
2327
2848
|
*/
|
|
2328
2849
|
async function icpCommand(args: string[]) {
|
|
2329
2850
|
const [sub, ...rest] = args;
|
|
2330
|
-
|
|
2851
|
+
// Help-before-network: catch --help/-h anywhere in argv (so `icp --help`,
|
|
2852
|
+
// `icp judge --help`, `icp eval --help` all print usage before any work).
|
|
2853
|
+
if (!sub || args.includes("--help") || args.includes("-h")) {
|
|
2331
2854
|
console.log(`Usage:
|
|
2332
2855
|
fullstackgtm icp interview emit the interview spec (an agent drives it with AskUserQuestion)
|
|
2333
2856
|
fullstackgtm icp set <answers.json> [--name <n>] [--out <path>] write icp.json from interview answers
|
|
2334
2857
|
fullstackgtm icp show [--icp <path>] render the ICP + the discovery filters it produces
|
|
2858
|
+
fullstackgtm icp judge [--signals-from latest|<label>] [--with-history] [--prompt <path>] [--min-score 0] [--save] rank unjudged signals into send/nurture/skip decisions (timing x fit x memory)
|
|
2859
|
+
fullstackgtm icp eval [--golden <path>|default] [--against-outcomes] [--min-accuracy 0.8] [--json] grade the judge: golden-set accuracy and/or hot>cold calibration; exits 2 below the bar
|
|
2335
2860
|
|
|
2336
2861
|
The ICP makes \`enrich acquire\` targeted, not random: it generates each
|
|
2337
2862
|
provider's discovery filters (Explorium, pipe0/Crustdata) AND scores every
|
|
2338
2863
|
discovered prospect for fit — only above-threshold leads become create_record
|
|
2339
|
-
ops. Develop one by interview, then \`enrich acquire\` picks up ./icp.json
|
|
2864
|
+
ops. Develop one by interview, then \`enrich acquire\` picks up ./icp.json.
|
|
2865
|
+
\`icp judge\` ranks fresh signals (from \`signals fetch --save\`) into send/nurture/skip;
|
|
2866
|
+
\`icp eval\` is the probabilistic-judgment gate before any apply.`);
|
|
2340
2867
|
return;
|
|
2341
2868
|
}
|
|
2342
2869
|
if (sub === "interview") {
|
|
@@ -2383,7 +2910,161 @@ ops. Develop one by interview, then \`enrich acquire\` picks up ./icp.json.`);
|
|
|
2383
2910
|
);
|
|
2384
2911
|
return;
|
|
2385
2912
|
}
|
|
2386
|
-
|
|
2913
|
+
if (sub === "judge") {
|
|
2914
|
+
// The parent icpCommand already caught --help/-h before any work, so we can
|
|
2915
|
+
// flag-parse then act (no network until judgeSignals, and only with a key).
|
|
2916
|
+
const signalsFrom = option(rest, "--signals-from") ?? "latest";
|
|
2917
|
+
const withHistory = rest.includes("--with-history");
|
|
2918
|
+
const promptPath = option(rest, "--prompt");
|
|
2919
|
+
const minScore = numericOption(rest, "--min-score") ?? 0;
|
|
2920
|
+
const save = rest.includes("--save");
|
|
2921
|
+
|
|
2922
|
+
// 1) Load signals from the signal store.
|
|
2923
|
+
const signalStore = createFileSignalStore();
|
|
2924
|
+
const signalRun =
|
|
2925
|
+
signalsFrom === "latest" ? await signalStore.latestRun() : await signalStore.getRun(signalsFrom);
|
|
2926
|
+
if (!signalRun) {
|
|
2927
|
+
throw new Error(`No signal run "${signalsFrom}" — run \`fullstackgtm signals fetch --save\` first.`);
|
|
2928
|
+
}
|
|
2929
|
+
const unjudged = signalRun.signals.filter((s) => !s.judgedBy);
|
|
2930
|
+
if (unjudged.length === 0) throw new Error(`Signal run "${signalRun.runLabel}" has no unjudged signals.`);
|
|
2931
|
+
|
|
2932
|
+
// 2) Optional inputs: ICP (may be undefined), snapshot (only --with-history),
|
|
2933
|
+
// outcomes + config from the signal store / DEFAULT_SIGNALS_CONFIG.
|
|
2934
|
+
const icp = loadIcp(rest);
|
|
2935
|
+
const config: SignalsConfig = DEFAULT_SIGNALS_CONFIG;
|
|
2936
|
+
const outcomes = await signalStore.listOutcomes();
|
|
2937
|
+
const snapshot = withHistory ? await readSnapshot(rest) : undefined;
|
|
2938
|
+
|
|
2939
|
+
// 3) Resolve the LLM seam ONLY if a key already exists (deterministic baseline
|
|
2940
|
+
// otherwise — never PROMPT here; judge must run key-free).
|
|
2941
|
+
const cred = resolveLlmCredential();
|
|
2942
|
+
const baseUrls = resolveLlmBaseUrls();
|
|
2943
|
+
const model = option(rest, "--model") ?? undefined;
|
|
2944
|
+
const llm: LlmCallOptions | undefined = cred
|
|
2945
|
+
? { ...cred, ...baseUrls, ...(model ? { model } : {}) }
|
|
2946
|
+
: undefined;
|
|
2947
|
+
const promptTemplate = llm
|
|
2948
|
+
? promptPath
|
|
2949
|
+
? readFileSync(resolve(process.cwd(), promptPath), "utf8")
|
|
2950
|
+
: DEFAULT_JUDGE_PROMPT
|
|
2951
|
+
: undefined;
|
|
2952
|
+
|
|
2953
|
+
// 4) Judge.
|
|
2954
|
+
const decisions = (
|
|
2955
|
+
await judgeSignals({
|
|
2956
|
+
signals: unjudged,
|
|
2957
|
+
outcomes,
|
|
2958
|
+
config,
|
|
2959
|
+
icp,
|
|
2960
|
+
snapshot,
|
|
2961
|
+
withHistory,
|
|
2962
|
+
promptTemplate,
|
|
2963
|
+
llm,
|
|
2964
|
+
})
|
|
2965
|
+
).filter((d) => d.score >= minScore);
|
|
2966
|
+
|
|
2967
|
+
// 5) Ranked decisions to stdout (JSON), guidance to stderr.
|
|
2968
|
+
console.log(JSON.stringify(decisions, null, 2));
|
|
2969
|
+
console.error(
|
|
2970
|
+
`Judged ${unjudged.length} signal(s) across ${new Set(decisions.map((d) => d.accountDomain)).size} account(s): ` +
|
|
2971
|
+
`${decisions.filter((d) => d.decision === "send").length} send, ` +
|
|
2972
|
+
`${decisions.filter((d) => d.decision === "nurture").length} nurture, ` +
|
|
2973
|
+
`${decisions.filter((d) => d.decision === "skip").length} skip` +
|
|
2974
|
+
`${llm ? "" : " (deterministic — no LLM key; run `login anthropic|openai` for why-now + play)"}.`,
|
|
2975
|
+
);
|
|
2976
|
+
|
|
2977
|
+
// 6) --save: persist a JudgeRun + stamp consumed signals judgedBy.
|
|
2978
|
+
if (save) {
|
|
2979
|
+
const runLabel = option(rest, "--label") ?? `judge-${new Date().toISOString().slice(0, 10)}`;
|
|
2980
|
+
const store = createFileJudgeStore();
|
|
2981
|
+
await store.appendRun({
|
|
2982
|
+
id: judgeRunId(runLabel),
|
|
2983
|
+
runLabel,
|
|
2984
|
+
signalRunLabel: signalRun.runLabel,
|
|
2985
|
+
createdAt: new Date().toISOString(),
|
|
2986
|
+
decisions,
|
|
2987
|
+
});
|
|
2988
|
+
const judgedIds = new Set(decisions.flatMap((d) => d.evidence));
|
|
2989
|
+
for (const s of signalRun.signals) if (judgedIds.has(s.id)) s.judgedBy = runLabel;
|
|
2990
|
+
await signalStore.updateRun(signalRun);
|
|
2991
|
+
console.error(
|
|
2992
|
+
`Saved judge run "${runLabel}" (${decisions.length} decisions); stamped ${judgedIds.size} signals judgedBy. ` +
|
|
2993
|
+
`Next: \`fullstackgtm draft --from-judge ${runLabel} --save\`.`,
|
|
2994
|
+
);
|
|
2995
|
+
} else {
|
|
2996
|
+
console.error("(not saved — re-run with --save to persist a judge run and stamp the consumed signals)");
|
|
2997
|
+
}
|
|
2998
|
+
return;
|
|
2999
|
+
}
|
|
3000
|
+
if (sub === "eval") {
|
|
3001
|
+
// The parent icpCommand already caught --help/-h before any work. The whole
|
|
3002
|
+
// command is read-only: it grades, never emits a plan, never touches a provider.
|
|
3003
|
+
const goldenArg = option(rest, "--golden") ?? "default";
|
|
3004
|
+
const againstOutcomes = rest.includes("--against-outcomes");
|
|
3005
|
+
const minAccuracy = numericOption(rest, "--min-accuracy") ?? DEFAULT_MIN_ACCURACY;
|
|
3006
|
+
const asJson = rest.includes("--json");
|
|
3007
|
+
|
|
3008
|
+
if (againstOutcomes) {
|
|
3009
|
+
// The empirical gate: hot decisions must book strictly more than cold.
|
|
3010
|
+
const judgeStore = createFileJudgeStore();
|
|
3011
|
+
const labelArg = option(rest, "--from-judge");
|
|
3012
|
+
const judgeRun = labelArg ? await judgeStore.getRun(labelArg) : await judgeStore.latestRun();
|
|
3013
|
+
if (!judgeRun) {
|
|
3014
|
+
throw new Error(
|
|
3015
|
+
`No judge run ${labelArg ? `"${labelArg}"` : "to grade"} — run \`fullstackgtm icp judge --save\` first.`,
|
|
3016
|
+
);
|
|
3017
|
+
}
|
|
3018
|
+
const outcomes = await createFileSignalStore().listOutcomes();
|
|
3019
|
+
const cal = gradeAgainstOutcomes(judgeRun.decisions, outcomes);
|
|
3020
|
+
if (asJson) {
|
|
3021
|
+
console.log(JSON.stringify(cal, null, 2));
|
|
3022
|
+
} else {
|
|
3023
|
+
console.log(
|
|
3024
|
+
`Outcome calibration (judge run "${judgeRun.runLabel}"):\n` +
|
|
3025
|
+
` hot book rate: ${(cal.hotBookRate * 100).toFixed(1)}% over ${cal.hotCount} hot account(s)\n` +
|
|
3026
|
+
` cold book rate: ${(cal.coldBookRate * 100).toFixed(1)}% over ${cal.coldCount} cold account(s)\n` +
|
|
3027
|
+
` calibrated: ${cal.calibrated ? "yes (hot > cold)" : "NO (hot did not strictly beat cold)"}`,
|
|
3028
|
+
);
|
|
3029
|
+
}
|
|
3030
|
+
console.error(
|
|
3031
|
+
cal.calibrated
|
|
3032
|
+
? "Calibrated: hot scores book strictly more than cold — the judge earns its interrupt."
|
|
3033
|
+
: "NOT calibrated: hot did not strictly beat cold. Tune the rubric before scheduling apply behind this gate.",
|
|
3034
|
+
);
|
|
3035
|
+
if (!cal.calibrated) process.exitCode = 2;
|
|
3036
|
+
return;
|
|
3037
|
+
}
|
|
3038
|
+
|
|
3039
|
+
// The golden-set gate (default). Resolve "default" BEFORE any file read.
|
|
3040
|
+
const rows =
|
|
3041
|
+
goldenArg === "default"
|
|
3042
|
+
? DEFAULT_GOLDEN_SET
|
|
3043
|
+
: parseGoldenSet(readFileSync(resolve(process.cwd(), goldenArg), "utf8"));
|
|
3044
|
+
const result = await gradeJudge(rows, defaultJudgeFn({ icp: loadIcp(rest), now: new Date() }));
|
|
3045
|
+
if (asJson) {
|
|
3046
|
+
console.log(JSON.stringify(result, null, 2));
|
|
3047
|
+
} else {
|
|
3048
|
+
const c = result.confusion;
|
|
3049
|
+
console.log(
|
|
3050
|
+
`Golden-set grade (${rows.length} row(s)):\n` +
|
|
3051
|
+
` accuracy: ${(result.accuracy * 100).toFixed(1)}% (bar ${(minAccuracy * 100).toFixed(0)}%)\n` +
|
|
3052
|
+
` precision: ${(result.precision * 100).toFixed(1)}%\n` +
|
|
3053
|
+
` recall: ${(result.recall * 100).toFixed(1)}%\n` +
|
|
3054
|
+
` confusion: tp=${c.truePositive} fp=${c.falsePositive} tn=${c.trueNegative} fn=${c.falseNegative}`,
|
|
3055
|
+
);
|
|
3056
|
+
}
|
|
3057
|
+
if (result.accuracy < minAccuracy) {
|
|
3058
|
+
console.error(
|
|
3059
|
+
`Judge accuracy ${(result.accuracy * 100).toFixed(1)}% is below the ${(minAccuracy * 100).toFixed(0)}% bar — exit 2.`,
|
|
3060
|
+
);
|
|
3061
|
+
process.exitCode = 2;
|
|
3062
|
+
} else {
|
|
3063
|
+
console.error(`Judge passes the golden-set gate (${(result.accuracy * 100).toFixed(1)}% >= ${(minAccuracy * 100).toFixed(0)}%).`);
|
|
3064
|
+
}
|
|
3065
|
+
return;
|
|
3066
|
+
}
|
|
3067
|
+
throw new Error(`Unknown icp subcommand: ${sub} (try: interview, set, show, judge, eval)`);
|
|
2387
3068
|
}
|
|
2388
3069
|
|
|
2389
3070
|
/**
|
|
@@ -4529,6 +5210,14 @@ export async function runCli(argv: string[]) {
|
|
|
4529
5210
|
await icpCommand(args);
|
|
4530
5211
|
return;
|
|
4531
5212
|
}
|
|
5213
|
+
if (command === "signals") {
|
|
5214
|
+
await signalsCommand(args);
|
|
5215
|
+
return;
|
|
5216
|
+
}
|
|
5217
|
+
if (command === "draft") {
|
|
5218
|
+
await draftCommand(args);
|
|
5219
|
+
return;
|
|
5220
|
+
}
|
|
4532
5221
|
if (command === "schedule") {
|
|
4533
5222
|
await scheduleCommand(args);
|
|
4534
5223
|
return;
|