fullstackgtm 0.39.0 → 0.41.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +74 -0
- package/README.md +42 -1
- package/dist/cli.js +615 -6
- package/dist/connectors/atsBoards.d.ts +60 -0
- package/dist/connectors/atsBoards.js +179 -0
- package/dist/connectors/prospectSources.d.ts +7 -5
- package/dist/connectors/prospectSources.js +82 -47
- package/dist/draft.d.ts +182 -0
- package/dist/draft.js +333 -0
- package/dist/index.d.ts +5 -0
- package/dist/index.js +5 -0
- package/dist/judge.d.ts +209 -0
- package/dist/judge.js +490 -0
- package/dist/judgeEval.d.ts +141 -0
- package/dist/judgeEval.js +331 -0
- package/dist/llm.d.ts +13 -0
- package/dist/llm.js +18 -2
- package/dist/schedule.js +11 -1
- package/dist/signals.d.ts +197 -0
- package/dist/signals.js +515 -0
- package/docs/api.md +2 -2
- package/package.json +1 -1
- package/src/cli.ts +696 -7
- package/src/connectors/atsBoards.ts +242 -0
- package/src/connectors/prospectSources.ts +94 -43
- package/src/draft.ts +463 -0
- package/src/index.ts +94 -0
- package/src/judge.ts +661 -0
- package/src/judgeEval.ts +459 -0
- package/src/llm.ts +31 -2
- package/src/schedule.ts +11 -1
- package/src/signals.ts +685 -0
package/src/signals.ts
ADDED
|
@@ -0,0 +1,685 @@
|
|
|
1
|
+
import { mkdirSync, readFileSync, readdirSync, appendFileSync, existsSync } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { credentialsDir, ensureSecureHomeDir, writeSecureFile } from "./credentials.ts";
|
|
4
|
+
import type { AtsBoardSource, AtsJob } from "./connectors/atsBoards.ts";
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* The signals layer: a deterministic, read-only timing ledger. Every other CLI
|
|
8
|
+
* layer answers "is this CRM record correct?"; signals answers "did something
|
|
9
|
+
* change at this account this week?" — the premise of signal-based outbound.
|
|
10
|
+
*
|
|
11
|
+
* `signals fetch` runs a watchlist of accounts through bucketed adapters (ATS
|
|
12
|
+
* job boards are the free, no-auth example), captures each detected change as a
|
|
13
|
+
* `Signal` carrying a VERBATIM source quote and URL, dedups against the store,
|
|
14
|
+
* stamps `firstSeen`, and ranks by a learned-then-defaulted weight. Outcomes
|
|
15
|
+
* feed back via `signals outcome`; `signals weights` drifts the bucket priors
|
|
16
|
+
* toward what actually books meetings.
|
|
17
|
+
*
|
|
18
|
+
* GOVERNANCE — structural, forever: signals is Detect-side. It produces no
|
|
19
|
+
* `PatchOperation`, opens no plan, touches no provider record. `fetch --save`
|
|
20
|
+
* persists a `SignalRun` to the LOCAL signal store only; the write side is
|
|
21
|
+
* downstream and human-gated (`draft` → `plans approve` → `apply`). State stays
|
|
22
|
+
* local — the CLI never writes signal custom properties into the customer's
|
|
23
|
+
* portal. Recurring execution is owned by the horizontal scheduler; signals
|
|
24
|
+
* knows nothing about cron.
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
// ---------------------------------------------------------------------------
|
|
28
|
+
// Types
|
|
29
|
+
|
|
30
|
+
export type SignalBucket = "demand" | "funding" | "job" | "company" | "social";
|
|
31
|
+
|
|
32
|
+
export const SIGNAL_BUCKETS: SignalBucket[] = ["demand", "funding", "job", "company", "social"];
|
|
33
|
+
|
|
34
|
+
export type Signal = {
|
|
35
|
+
/** sig_<fnv1a(accountDomain|bucket|trigger)> — stable per detected change. */
|
|
36
|
+
id: string;
|
|
37
|
+
/** Normalized domain (strip protocol/www/path), matching enrich's key logic. */
|
|
38
|
+
accountDomain: string;
|
|
39
|
+
bucket: SignalBucket;
|
|
40
|
+
/** Short human label, e.g. "reposted: Head of Growth" or "hiring: VP Sales". */
|
|
41
|
+
trigger: string;
|
|
42
|
+
/** VERBATIM source text — the evidence anchor reused by judge/drafter. */
|
|
43
|
+
quote: string;
|
|
44
|
+
sourceUrl: string;
|
|
45
|
+
/** ISO 8601. */
|
|
46
|
+
firstSeen: string;
|
|
47
|
+
/** Bucket weight × recency × repost boost, computed at fetch. */
|
|
48
|
+
weight: number;
|
|
49
|
+
reposted?: boolean;
|
|
50
|
+
/** "greenhouse" | "lever" | "ashby" | "ingest". */
|
|
51
|
+
source: string;
|
|
52
|
+
/** judge runLabel that consumed this signal (null until judged). */
|
|
53
|
+
judgedBy?: string | null;
|
|
54
|
+
/**
|
|
55
|
+
* Stable per-role identity within a board (provider id when present, else the
|
|
56
|
+
* title). Lets the reposted heuristic tell "same role, gone, back" (a new req
|
|
57
|
+
* id under the same title) from one continuously-open req. job-bucket only.
|
|
58
|
+
*/
|
|
59
|
+
roleKey?: string;
|
|
60
|
+
};
|
|
61
|
+
|
|
62
|
+
export type SignalOutcomeResult = "replied" | "meeting" | "bounced" | "no_reply";
|
|
63
|
+
|
|
64
|
+
export type SignalOutcome = {
|
|
65
|
+
id: string;
|
|
66
|
+
accountDomain: string;
|
|
67
|
+
touchId?: string;
|
|
68
|
+
result: SignalOutcomeResult;
|
|
69
|
+
recordedAt: string;
|
|
70
|
+
/** Signal ids credited (from the judge decision). */
|
|
71
|
+
creditedSignals: string[];
|
|
72
|
+
};
|
|
73
|
+
|
|
74
|
+
export type SignalBucketConfig = {
|
|
75
|
+
sources: string[];
|
|
76
|
+
keywords?: string[];
|
|
77
|
+
weight: number;
|
|
78
|
+
};
|
|
79
|
+
|
|
80
|
+
export type SignalsConfig = {
|
|
81
|
+
watchlist: {
|
|
82
|
+
/** "crm:<segment>" derives domains from a CRM segment at run time. */
|
|
83
|
+
source?: string;
|
|
84
|
+
domains?: string[];
|
|
85
|
+
/** domain -> ATS board token, for boards not resolvable from the domain. */
|
|
86
|
+
boards?: Record<string, string>;
|
|
87
|
+
};
|
|
88
|
+
buckets: Record<SignalBucket, SignalBucketConfig>;
|
|
89
|
+
/** Dedup + reposted-detection window. Default 30. */
|
|
90
|
+
dedupWindowDays: number;
|
|
91
|
+
/** Extra weight for a reposted role (the "first hire fell through" heuristic). */
|
|
92
|
+
repostedRoleBoost: number;
|
|
93
|
+
};
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* Zero-config preset. `signals fetch` works with no `signals.config.json`
|
|
97
|
+
* (preset-first, like enrich). The four free buckets carry the system; `demand`
|
|
98
|
+
* is schema-reserved (privileged source, unbuilt) so its `sources` stay empty.
|
|
99
|
+
*/
|
|
100
|
+
export const DEFAULT_SIGNALS_CONFIG: SignalsConfig = {
|
|
101
|
+
watchlist: {},
|
|
102
|
+
buckets: {
|
|
103
|
+
demand: { sources: [], weight: 2.0 },
|
|
104
|
+
funding: { sources: ["news"], weight: 1.6 },
|
|
105
|
+
job: { sources: ["greenhouse", "lever", "ashby"], keywords: [], weight: 1.0 },
|
|
106
|
+
company: { sources: ["web"], weight: 1.0 },
|
|
107
|
+
social: { sources: ["web"], weight: 0.4 },
|
|
108
|
+
},
|
|
109
|
+
dedupWindowDays: 30,
|
|
110
|
+
repostedRoleBoost: 0.5,
|
|
111
|
+
};
|
|
112
|
+
|
|
113
|
+
// ---------------------------------------------------------------------------
|
|
114
|
+
// Domain normalization — mirrors enrich's normalizeKeyValue("domain", ...)
|
|
115
|
+
// (strip protocol/www/path, lowercased) so a signal account key lines up with
|
|
116
|
+
// the canonical snapshot / enrich matcher.
|
|
117
|
+
|
|
118
|
+
export function normalizeAccountDomain(value: string | undefined | null): string {
|
|
119
|
+
const text = String(value ?? "").trim().toLowerCase();
|
|
120
|
+
if (!text) return "";
|
|
121
|
+
return text
|
|
122
|
+
.replace(/^https?:\/\//, "")
|
|
123
|
+
.replace(/^www\./, "")
|
|
124
|
+
.replace(/\/.*$/, "");
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
// ---------------------------------------------------------------------------
|
|
128
|
+
// Stable hashing (FNV-1a) — mirrors enrich.ts fnv1a, duplicated to keep this
|
|
129
|
+
// file importable without the audit engine (the market.ts/enrich.ts precedent).
|
|
130
|
+
|
|
131
|
+
function fnv1a(value: string): string {
|
|
132
|
+
let hash = 0x811c9dc5;
|
|
133
|
+
for (let i = 0; i < value.length; i += 1) {
|
|
134
|
+
hash ^= value.charCodeAt(i);
|
|
135
|
+
hash = Math.imul(hash, 0x01000193);
|
|
136
|
+
}
|
|
137
|
+
return (hash >>> 0).toString(16).padStart(8, "0");
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/** Dedup key: a signal is "the same change" iff (account, bucket, trigger) match. */
|
|
141
|
+
export function dedupKey(signal: Pick<Signal, "accountDomain" | "bucket" | "trigger">): string {
|
|
142
|
+
return `${signal.accountDomain}|${signal.bucket}|${signal.trigger}`;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
export function signalId(signal: Pick<Signal, "accountDomain" | "bucket" | "trigger">): string {
|
|
146
|
+
return `sig_${fnv1a(dedupKey(signal))}`;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
function outcomeId(o: Pick<SignalOutcome, "accountDomain" | "touchId" | "result" | "recordedAt">): string {
|
|
150
|
+
return `out_${fnv1a(`${o.accountDomain}|${o.touchId ?? ""}|${o.result}|${o.recordedAt}`)}`;
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
// ---------------------------------------------------------------------------
|
|
154
|
+
// Config parsing — strict, up-front, names the offending entry (enrich pattern).
|
|
155
|
+
|
|
156
|
+
function fail(message: string): never {
|
|
157
|
+
throw new Error(`signals config: ${message}`);
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
export function parseSignalsConfig(raw: string): SignalsConfig {
|
|
161
|
+
let parsed: unknown;
|
|
162
|
+
try {
|
|
163
|
+
parsed = JSON.parse(raw);
|
|
164
|
+
} catch (error) {
|
|
165
|
+
fail(`not valid JSON (${error instanceof Error ? error.message : String(error)})`);
|
|
166
|
+
}
|
|
167
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
168
|
+
fail("expected a JSON object with watchlist, buckets, dedupWindowDays, repostedRoleBoost");
|
|
169
|
+
}
|
|
170
|
+
const config = parsed as Partial<SignalsConfig>;
|
|
171
|
+
|
|
172
|
+
// watchlist
|
|
173
|
+
const watchlist = config.watchlist ?? {};
|
|
174
|
+
if (typeof watchlist !== "object" || Array.isArray(watchlist)) {
|
|
175
|
+
fail('"watchlist" must be an object (source / domains / boards)');
|
|
176
|
+
}
|
|
177
|
+
if (watchlist.source !== undefined && typeof watchlist.source !== "string") {
|
|
178
|
+
fail('watchlist.source must be a string (e.g. "crm:open-pipeline")');
|
|
179
|
+
}
|
|
180
|
+
if (watchlist.domains !== undefined) {
|
|
181
|
+
if (!Array.isArray(watchlist.domains) || watchlist.domains.some((d) => typeof d !== "string")) {
|
|
182
|
+
fail("watchlist.domains must be an array of domain strings");
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
if (watchlist.boards !== undefined) {
|
|
186
|
+
if (typeof watchlist.boards !== "object" || Array.isArray(watchlist.boards)) {
|
|
187
|
+
fail("watchlist.boards must be an object mapping domain -> board token");
|
|
188
|
+
}
|
|
189
|
+
for (const [domain, tok] of Object.entries(watchlist.boards)) {
|
|
190
|
+
if (typeof tok !== "string" || !tok) fail(`watchlist.boards["${domain}"] must be a non-empty board token`);
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
// buckets — fill missing buckets from the default so a partial config is valid.
|
|
195
|
+
const buckets = {} as Record<SignalBucket, SignalBucketConfig>;
|
|
196
|
+
const rawBuckets = (config.buckets ?? {}) as Partial<Record<SignalBucket, unknown>>;
|
|
197
|
+
if (typeof rawBuckets !== "object" || Array.isArray(rawBuckets)) {
|
|
198
|
+
fail('"buckets" must be an object keyed by bucket name');
|
|
199
|
+
}
|
|
200
|
+
for (const key of Object.keys(rawBuckets)) {
|
|
201
|
+
if (!SIGNAL_BUCKETS.includes(key as SignalBucket)) {
|
|
202
|
+
fail(`unknown bucket "${key}" (use: ${SIGNAL_BUCKETS.join(", ")})`);
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
for (const bucket of SIGNAL_BUCKETS) {
|
|
206
|
+
const provided = rawBuckets[bucket] as Partial<SignalBucketConfig> | undefined;
|
|
207
|
+
if (provided === undefined) {
|
|
208
|
+
buckets[bucket] = { ...DEFAULT_SIGNALS_CONFIG.buckets[bucket] };
|
|
209
|
+
continue;
|
|
210
|
+
}
|
|
211
|
+
if (typeof provided !== "object" || Array.isArray(provided)) fail(`buckets.${bucket} must be an object`);
|
|
212
|
+
if (!Array.isArray(provided.sources) || provided.sources.some((s) => typeof s !== "string")) {
|
|
213
|
+
fail(`buckets.${bucket}.sources must be an array of source strings`);
|
|
214
|
+
}
|
|
215
|
+
if (provided.weight !== undefined && (!Number.isFinite(provided.weight) || provided.weight < 0)) {
|
|
216
|
+
fail(`buckets.${bucket}.weight must be a non-negative number`);
|
|
217
|
+
}
|
|
218
|
+
if (provided.keywords !== undefined) {
|
|
219
|
+
if (!Array.isArray(provided.keywords) || provided.keywords.some((k) => typeof k !== "string")) {
|
|
220
|
+
fail(`buckets.${bucket}.keywords must be an array of strings`);
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
buckets[bucket] = {
|
|
224
|
+
sources: provided.sources,
|
|
225
|
+
weight: provided.weight ?? DEFAULT_SIGNALS_CONFIG.buckets[bucket].weight,
|
|
226
|
+
...(provided.keywords !== undefined ? { keywords: provided.keywords } : {}),
|
|
227
|
+
};
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
const dedupWindowDays = config.dedupWindowDays ?? DEFAULT_SIGNALS_CONFIG.dedupWindowDays;
|
|
231
|
+
if (!Number.isFinite(dedupWindowDays) || dedupWindowDays <= 0) {
|
|
232
|
+
fail(`dedupWindowDays must be a positive number (got ${JSON.stringify(config.dedupWindowDays)})`);
|
|
233
|
+
}
|
|
234
|
+
const repostedRoleBoost = config.repostedRoleBoost ?? DEFAULT_SIGNALS_CONFIG.repostedRoleBoost;
|
|
235
|
+
if (!Number.isFinite(repostedRoleBoost) || repostedRoleBoost < 0) {
|
|
236
|
+
fail(`repostedRoleBoost must be a non-negative number (got ${JSON.stringify(config.repostedRoleBoost)})`);
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
return {
|
|
240
|
+
watchlist: {
|
|
241
|
+
...(watchlist.source !== undefined ? { source: watchlist.source } : {}),
|
|
242
|
+
...(watchlist.domains !== undefined ? { domains: watchlist.domains } : {}),
|
|
243
|
+
...(watchlist.boards !== undefined ? { boards: watchlist.boards } : {}),
|
|
244
|
+
},
|
|
245
|
+
buckets,
|
|
246
|
+
dedupWindowDays,
|
|
247
|
+
repostedRoleBoost,
|
|
248
|
+
};
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
export const SIGNALS_CONFIG_FILE_NAME = "signals.config.json";
|
|
252
|
+
|
|
253
|
+
export function loadSignalsConfig(path: string): SignalsConfig {
|
|
254
|
+
let raw: string;
|
|
255
|
+
try {
|
|
256
|
+
raw = readFileSync(path, "utf8");
|
|
257
|
+
} catch {
|
|
258
|
+
throw new Error(
|
|
259
|
+
`No signals config at ${path}. Create ${SIGNALS_CONFIG_FILE_NAME} (watchlist/buckets/` +
|
|
260
|
+
"dedupWindowDays/repostedRoleBoost — see docs/signals.md) or run with the zero-config defaults.",
|
|
261
|
+
);
|
|
262
|
+
}
|
|
263
|
+
return parseSignalsConfig(raw);
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
// ---------------------------------------------------------------------------
|
|
267
|
+
// Recency + weight
|
|
268
|
+
|
|
269
|
+
const DAY_MS = 24 * 60 * 60 * 1000;
|
|
270
|
+
|
|
271
|
+
/**
|
|
272
|
+
* Recency decay over the dedup window: a signal seen today is 1.0; one at the
|
|
273
|
+
* window edge is ~`floor`. Linear, clamped, deterministic — no surprise from
|
|
274
|
+
* an exponential's tail. Older than the window → `floor`.
|
|
275
|
+
*/
|
|
276
|
+
export function recencyFactor(firstSeen: string, now: Date, windowDays: number): number {
|
|
277
|
+
const seen = Date.parse(firstSeen);
|
|
278
|
+
if (!Number.isFinite(seen)) return 1;
|
|
279
|
+
const ageDays = Math.max(0, (now.getTime() - seen) / DAY_MS);
|
|
280
|
+
const floor = 0.4;
|
|
281
|
+
if (windowDays <= 0) return 1;
|
|
282
|
+
const decayed = 1 - (1 - floor) * Math.min(1, ageDays / windowDays);
|
|
283
|
+
return Math.max(floor, decayed);
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
/**
|
|
287
|
+
* Final signal weight = bucketWeight × recency (+ repost boost when reposted).
|
|
288
|
+
* Rounded to 4 dp so the same inputs always serialize the same string (the
|
|
289
|
+
* deterministic-store property the tests and dedup ledger rely on).
|
|
290
|
+
*/
|
|
291
|
+
export function signalWeight(opts: {
|
|
292
|
+
bucketWeight: number;
|
|
293
|
+
firstSeen: string;
|
|
294
|
+
now: Date;
|
|
295
|
+
windowDays: number;
|
|
296
|
+
reposted?: boolean;
|
|
297
|
+
repostedRoleBoost: number;
|
|
298
|
+
}): number {
|
|
299
|
+
const base = opts.bucketWeight * recencyFactor(opts.firstSeen, opts.now, opts.windowDays);
|
|
300
|
+
const boosted = opts.reposted ? base + opts.repostedRoleBoost : base;
|
|
301
|
+
return Math.round(boosted * 10000) / 10000;
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
// ---------------------------------------------------------------------------
|
|
305
|
+
// Build signals from ATS jobs
|
|
306
|
+
|
|
307
|
+
/**
|
|
308
|
+
* Map a board's open roles to `job` signals for one account, computing weights
|
|
309
|
+
* and applying the reposted heuristic against `priorSignals` (the store's prior
|
|
310
|
+
* job signals for this account). A role is REPOSTED when the store saw a
|
|
311
|
+
* same-title job signal inside `dedupWindowDays` under a DIFFERENT role id (the
|
|
312
|
+
* old req closed, a new req opened with the same title) — the "first hire fell
|
|
313
|
+
* through, more pain now" signal the article calls out.
|
|
314
|
+
*
|
|
315
|
+
* A role only becomes a signal if its quote can carry a configured keyword
|
|
316
|
+
* verbatim (the evidence anchor). With no keywords configured, every open role
|
|
317
|
+
* qualifies and the quote is title + leading JD snippet.
|
|
318
|
+
*/
|
|
319
|
+
export function buildSignalsFromAts(
|
|
320
|
+
rawJobs: Array<AtsJob & { source: AtsBoardSource }>,
|
|
321
|
+
account: { domain: string },
|
|
322
|
+
config: SignalsConfig,
|
|
323
|
+
opts: { now?: Date; priorSignals?: Signal[] } = {},
|
|
324
|
+
): Signal[] {
|
|
325
|
+
const now = opts.now ?? new Date();
|
|
326
|
+
const nowIso = now.toISOString();
|
|
327
|
+
const windowDays = config.dedupWindowDays;
|
|
328
|
+
const bucketWeight = config.buckets.job.weight;
|
|
329
|
+
const keywords = config.buckets.job.keywords ?? [];
|
|
330
|
+
const accountDomain = normalizeAccountDomain(account.domain);
|
|
331
|
+
|
|
332
|
+
// Prior same-title job signals (within window) indexed by title key, with the
|
|
333
|
+
// set of role ids seen — a different role id under a known title = reposted.
|
|
334
|
+
const priorByTitle = new Map<string, { roleKeys: Set<string>; withinWindow: boolean }>();
|
|
335
|
+
for (const prior of opts.priorSignals ?? []) {
|
|
336
|
+
if (prior.bucket !== "job" || prior.accountDomain !== accountDomain) continue;
|
|
337
|
+
const titleKey = roleTitleKey(prior.trigger);
|
|
338
|
+
const entry = priorByTitle.get(titleKey) ?? { roleKeys: new Set<string>(), withinWindow: false };
|
|
339
|
+
if (prior.roleKey) entry.roleKeys.add(prior.roleKey);
|
|
340
|
+
if (withinWindow(prior.firstSeen, now, windowDays)) entry.withinWindow = true;
|
|
341
|
+
priorByTitle.set(titleKey, entry);
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
const signals: Signal[] = [];
|
|
345
|
+
for (const job of rawJobs) {
|
|
346
|
+
const title = job.title.trim();
|
|
347
|
+
if (!title) continue;
|
|
348
|
+
// Quote must carry a configured keyword verbatim; with no keywords, any role
|
|
349
|
+
// qualifies. Reject a role whose JD contains none of the keywords — the
|
|
350
|
+
// judge/drafter could not ground a why-now on it.
|
|
351
|
+
const quote = jobQuote(title, job.jdSnippet, keywords);
|
|
352
|
+
if (quote === null) continue;
|
|
353
|
+
|
|
354
|
+
const titleKey = roleTitleKey(title);
|
|
355
|
+
const prior = priorByTitle.get(titleKey);
|
|
356
|
+
const reposted = Boolean(
|
|
357
|
+
prior && prior.withinWindow && job.firstSeenKey != null && !prior.roleKeys.has(job.firstSeenKey),
|
|
358
|
+
);
|
|
359
|
+
const trigger = reposted ? `reposted: ${title}` : `hiring: ${title}`;
|
|
360
|
+
const base = { accountDomain, bucket: "job" as const, trigger };
|
|
361
|
+
signals.push({
|
|
362
|
+
id: signalId(base),
|
|
363
|
+
accountDomain,
|
|
364
|
+
bucket: "job",
|
|
365
|
+
trigger,
|
|
366
|
+
quote,
|
|
367
|
+
sourceUrl: job.url,
|
|
368
|
+
firstSeen: nowIso,
|
|
369
|
+
weight: signalWeight({
|
|
370
|
+
bucketWeight,
|
|
371
|
+
firstSeen: nowIso,
|
|
372
|
+
now,
|
|
373
|
+
windowDays,
|
|
374
|
+
reposted,
|
|
375
|
+
repostedRoleBoost: config.repostedRoleBoost,
|
|
376
|
+
}),
|
|
377
|
+
...(reposted ? { reposted: true } : {}),
|
|
378
|
+
source: job.source,
|
|
379
|
+
judgedBy: null,
|
|
380
|
+
roleKey: job.firstSeenKey,
|
|
381
|
+
});
|
|
382
|
+
}
|
|
383
|
+
return signals;
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
/** Normalize a role title for same-role matching (collapse spacing, lowercase). */
|
|
387
|
+
function roleTitleKey(triggerOrTitle: string): string {
|
|
388
|
+
return triggerOrTitle
|
|
389
|
+
.replace(/^(reposted|hiring):\s*/i, "")
|
|
390
|
+
.trim()
|
|
391
|
+
.toLowerCase()
|
|
392
|
+
.replace(/\s+/g, " ");
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
/**
|
|
396
|
+
* Build the verbatim evidence quote for a job signal. The keyword (if any) must
|
|
397
|
+
* appear verbatim in the title or JD snippet; returns null when keywords are
|
|
398
|
+
* configured but none match (so the role is dropped, not faked).
|
|
399
|
+
*/
|
|
400
|
+
function jobQuote(title: string, jdSnippet: string, keywords: string[]): string | null {
|
|
401
|
+
const snippet = jdSnippet.trim();
|
|
402
|
+
const composed = snippet ? `${title} — ${snippet}` : title;
|
|
403
|
+
const active = keywords.map((k) => k.trim()).filter(Boolean);
|
|
404
|
+
if (active.length === 0) return composed;
|
|
405
|
+
const hay = composed.toLowerCase();
|
|
406
|
+
const matched = active.some((k) => hay.includes(k.toLowerCase()));
|
|
407
|
+
return matched ? composed : null;
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
function withinWindow(iso: string, now: Date, windowDays: number): boolean {
|
|
411
|
+
const t = Date.parse(iso);
|
|
412
|
+
if (!Number.isFinite(t)) return false;
|
|
413
|
+
return now.getTime() - t <= windowDays * DAY_MS;
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
// ---------------------------------------------------------------------------
|
|
417
|
+
// Dedup
|
|
418
|
+
|
|
419
|
+
/**
|
|
420
|
+
* Split candidate signals into fresh vs. deduped against prior signals. A
|
|
421
|
+
* candidate is deduped when a prior signal shares its `dedupKey`
|
|
422
|
+
* (account|bucket|trigger) AND that prior was seen inside `windowDays` — the
|
|
423
|
+
* store's dedup ledger role. A reposted role carries a DIFFERENT trigger
|
|
424
|
+
* ("reposted: X" vs "hiring: X"), so it is correctly fresh. Deterministic.
|
|
425
|
+
*/
|
|
426
|
+
export function dedupeSignals(
|
|
427
|
+
candidates: Signal[],
|
|
428
|
+
priorSignals: Signal[],
|
|
429
|
+
windowDays: number,
|
|
430
|
+
now: Date,
|
|
431
|
+
): { fresh: Signal[]; deduped: Signal[] } {
|
|
432
|
+
const recentKeys = new Set<string>();
|
|
433
|
+
for (const prior of priorSignals) {
|
|
434
|
+
if (withinWindow(prior.firstSeen, now, windowDays)) recentKeys.add(dedupKey(prior));
|
|
435
|
+
}
|
|
436
|
+
const fresh: Signal[] = [];
|
|
437
|
+
const deduped: Signal[] = [];
|
|
438
|
+
const seenThisRun = new Set<string>();
|
|
439
|
+
for (const candidate of candidates) {
|
|
440
|
+
const key = dedupKey(candidate);
|
|
441
|
+
if (recentKeys.has(key) || seenThisRun.has(key)) {
|
|
442
|
+
deduped.push(candidate);
|
|
443
|
+
continue;
|
|
444
|
+
}
|
|
445
|
+
seenThisRun.add(key);
|
|
446
|
+
fresh.push(candidate);
|
|
447
|
+
}
|
|
448
|
+
return { fresh, deduped };
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
// ---------------------------------------------------------------------------
|
|
452
|
+
// Learn loop: bucket weights from outcomes (Beta-smoothed booked-meeting rate)
|
|
453
|
+
|
|
454
|
+
/**
|
|
455
|
+
* Recompute a per-bucket weight multiplier from the outcome ledger: the
|
|
456
|
+
* booked-meeting rate of touches credited to each bucket, Beta-smoothed against
|
|
457
|
+
* the declared default so a single reply does not swing a bucket. With NO
|
|
458
|
+
* outcomes, weights == config defaults (graceful cold start). Deterministic:
|
|
459
|
+
* same ledger → same weights.
|
|
460
|
+
*
|
|
461
|
+
* Mechanic: a bucket's declared default `w` acts as the prior mean. We observe
|
|
462
|
+
* `booked` booked meetings out of `total` credited touches and form a
|
|
463
|
+
* Beta(α,β)-style posterior mean with prior strength `priorStrength`, then scale
|
|
464
|
+
* the default by (posteriorRate / priorRate). priorRate is a fixed baseline
|
|
465
|
+
* booked-rate the default is calibrated against, so a bucket that books above
|
|
466
|
+
* baseline gets heavier and one that dies gets cut — the config value is never
|
|
467
|
+
* overwritten, only this overlay moves.
|
|
468
|
+
*/
|
|
469
|
+
export function computeWeights(
|
|
470
|
+
config: SignalsConfig,
|
|
471
|
+
outcomes: SignalOutcome[],
|
|
472
|
+
signalsById?: Map<string, Signal>,
|
|
473
|
+
): Record<SignalBucket, number> {
|
|
474
|
+
const PRIOR_STRENGTH = 4; // pseudo-touches; damps small samples
|
|
475
|
+
const PRIOR_RATE = 0.2; // baseline booked-meeting rate the defaults assume
|
|
476
|
+
|
|
477
|
+
// Per-bucket booked / total over credited touches. A touch books if its
|
|
478
|
+
// result is "meeting". Credit flows to the bucket of each credited signal.
|
|
479
|
+
const booked: Record<SignalBucket, number> = blankCounts();
|
|
480
|
+
const total: Record<SignalBucket, number> = blankCounts();
|
|
481
|
+
for (const outcome of outcomes) {
|
|
482
|
+
const buckets = creditedBuckets(outcome, signalsById);
|
|
483
|
+
const isBooked = outcome.result === "meeting" ? 1 : 0;
|
|
484
|
+
for (const bucket of buckets) {
|
|
485
|
+
total[bucket] += 1;
|
|
486
|
+
booked[bucket] += isBooked;
|
|
487
|
+
}
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
const weights = {} as Record<SignalBucket, number>;
|
|
491
|
+
for (const bucket of SIGNAL_BUCKETS) {
|
|
492
|
+
const declared = config.buckets[bucket].weight;
|
|
493
|
+
const n = total[bucket];
|
|
494
|
+
if (n === 0) {
|
|
495
|
+
weights[bucket] = declared; // cold start: exactly the declared default
|
|
496
|
+
continue;
|
|
497
|
+
}
|
|
498
|
+
const posteriorRate =
|
|
499
|
+
(booked[bucket] + PRIOR_STRENGTH * PRIOR_RATE) / (n + PRIOR_STRENGTH);
|
|
500
|
+
const scaled = declared * (posteriorRate / PRIOR_RATE);
|
|
501
|
+
weights[bucket] = Math.round(scaled * 10000) / 10000;
|
|
502
|
+
}
|
|
503
|
+
return weights;
|
|
504
|
+
}
|
|
505
|
+
|
|
506
|
+
/**
|
|
507
|
+
* Which buckets an outcome credits. When the credited signal objects are known
|
|
508
|
+
* (`signalsById`), use their real buckets; otherwise fall back to parsing the
|
|
509
|
+
* `sig_` ids' bucket is not recoverable, so credit nothing (the cold-start /
|
|
510
|
+
* unknown path leaves weights at defaults rather than mis-crediting).
|
|
511
|
+
*/
|
|
512
|
+
function creditedBuckets(outcome: SignalOutcome, signalsById?: Map<string, Signal>): SignalBucket[] {
|
|
513
|
+
if (!signalsById) return [];
|
|
514
|
+
const buckets = new Set<SignalBucket>();
|
|
515
|
+
for (const id of outcome.creditedSignals) {
|
|
516
|
+
const signal = signalsById.get(id);
|
|
517
|
+
if (signal) buckets.add(signal.bucket);
|
|
518
|
+
}
|
|
519
|
+
return [...buckets];
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
function blankCounts(): Record<SignalBucket, number> {
|
|
523
|
+
return { demand: 0, funding: 0, job: 0, company: 0, social: 0 };
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
// ---------------------------------------------------------------------------
|
|
527
|
+
// Store: per-label JSON runs + append-only outcomes.jsonl, profile-scoped.
|
|
528
|
+
|
|
529
|
+
export function signalsDir(baseDir?: string): string {
|
|
530
|
+
return join(baseDir ?? credentialsDir(), "signals");
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
export type SignalRun = {
|
|
534
|
+
id: string;
|
|
535
|
+
runLabel: string;
|
|
536
|
+
startedAt: string;
|
|
537
|
+
completedAt: string | null;
|
|
538
|
+
/** Buckets the run scanned. */
|
|
539
|
+
buckets: SignalBucket[];
|
|
540
|
+
counts: { fetched: number; new: number; deduped: number };
|
|
541
|
+
signals: Signal[];
|
|
542
|
+
};
|
|
543
|
+
|
|
544
|
+
export function signalRunId(runLabel: string): string {
|
|
545
|
+
return `sigr_${fnv1a(runLabel)}`;
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
export interface SignalStore {
|
|
549
|
+
/** Append a new run; refuses an existing label (runs are append-only). */
|
|
550
|
+
appendRun(run: SignalRun): Promise<SignalRun>;
|
|
551
|
+
/** Update an existing run in place (e.g. stamp judgedBy); id must match. */
|
|
552
|
+
updateRun(run: SignalRun): Promise<SignalRun>;
|
|
553
|
+
getRun(runLabel: string): Promise<SignalRun | null>;
|
|
554
|
+
listRuns(): Promise<SignalRun[]>;
|
|
555
|
+
latestRun(): Promise<SignalRun | null>;
|
|
556
|
+
/** Append-only outcome ledger (outcomes.jsonl). */
|
|
557
|
+
appendOutcome(outcome: SignalOutcome): Promise<SignalOutcome>;
|
|
558
|
+
listOutcomes(): Promise<SignalOutcome[]>;
|
|
559
|
+
/** Every signal across all runs, newest run last (the timing source). */
|
|
560
|
+
allSignals(): Promise<Signal[]>;
|
|
561
|
+
}
|
|
562
|
+
|
|
563
|
+
export function createFileSignalStore(directory?: string): SignalStore {
|
|
564
|
+
const dir = directory ?? signalsDir();
|
|
565
|
+
const runsDir = join(dir, "runs");
|
|
566
|
+
const outcomesPath = join(dir, "outcomes.jsonl");
|
|
567
|
+
|
|
568
|
+
function fileFor(runLabel: string): string {
|
|
569
|
+
if (!/^[\w.-]+$/.test(runLabel)) throw new Error(`Invalid run label: ${runLabel}`);
|
|
570
|
+
return join(runsDir, `${runLabel}.json`);
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
function read(runLabel: string): SignalRun | null {
|
|
574
|
+
try {
|
|
575
|
+
return JSON.parse(readFileSync(fileFor(runLabel), "utf8")) as SignalRun;
|
|
576
|
+
} catch {
|
|
577
|
+
return null;
|
|
578
|
+
}
|
|
579
|
+
}
|
|
580
|
+
|
|
581
|
+
function write(run: SignalRun): SignalRun {
|
|
582
|
+
// Run files carry account domains + source quotes/URLs; keep them owner-only
|
|
583
|
+
// like plan/enrich files (and lock the home down even before any login).
|
|
584
|
+
if (!directory) ensureSecureHomeDir();
|
|
585
|
+
mkdirSync(runsDir, { recursive: true, mode: 0o700 });
|
|
586
|
+
writeSecureFile(fileFor(run.runLabel), `${JSON.stringify(run, null, 2)}\n`);
|
|
587
|
+
return run;
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
function listRunsSync(): SignalRun[] {
|
|
591
|
+
let names: string[] = [];
|
|
592
|
+
try {
|
|
593
|
+
names = readdirSync(runsDir).filter((name) => name.endsWith(".json"));
|
|
594
|
+
} catch {
|
|
595
|
+
return [];
|
|
596
|
+
}
|
|
597
|
+
return names
|
|
598
|
+
.map((name) => read(name.slice(0, -".json".length)))
|
|
599
|
+
.filter((run): run is SignalRun => run !== null)
|
|
600
|
+
.sort((a, b) => a.startedAt.localeCompare(b.startedAt));
|
|
601
|
+
}
|
|
602
|
+
|
|
603
|
+
function readOutcomes(): SignalOutcome[] {
|
|
604
|
+
let raw: string;
|
|
605
|
+
try {
|
|
606
|
+
raw = readFileSync(outcomesPath, "utf8");
|
|
607
|
+
} catch {
|
|
608
|
+
return [];
|
|
609
|
+
}
|
|
610
|
+
const out: SignalOutcome[] = [];
|
|
611
|
+
for (const line of raw.split("\n")) {
|
|
612
|
+
const trimmed = line.trim();
|
|
613
|
+
if (!trimmed) continue;
|
|
614
|
+
try {
|
|
615
|
+
out.push(JSON.parse(trimmed) as SignalOutcome);
|
|
616
|
+
} catch {
|
|
617
|
+
// Skip a corrupt line rather than fail the whole ledger read.
|
|
618
|
+
}
|
|
619
|
+
}
|
|
620
|
+
return out;
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
return {
|
|
624
|
+
async appendRun(run) {
|
|
625
|
+
if (read(run.runLabel)) {
|
|
626
|
+
throw new Error(
|
|
627
|
+
`Signal run "${run.runLabel}" already exists — runs are append-only; use a new run label`,
|
|
628
|
+
);
|
|
629
|
+
}
|
|
630
|
+
return write(run);
|
|
631
|
+
},
|
|
632
|
+
async updateRun(run) {
|
|
633
|
+
const existing = read(run.runLabel);
|
|
634
|
+
if (!existing) throw new Error(`No signal run "${run.runLabel}" to update`);
|
|
635
|
+
if (existing.id !== run.id) {
|
|
636
|
+
throw new Error(`Signal run "${run.runLabel}" belongs to a different run id (${existing.id})`);
|
|
637
|
+
}
|
|
638
|
+
return write(run);
|
|
639
|
+
},
|
|
640
|
+
async getRun(runLabel) {
|
|
641
|
+
return read(runLabel);
|
|
642
|
+
},
|
|
643
|
+
async listRuns() {
|
|
644
|
+
return listRunsSync();
|
|
645
|
+
},
|
|
646
|
+
async latestRun() {
|
|
647
|
+
const runs = listRunsSync();
|
|
648
|
+
return runs.length ? runs[runs.length - 1] : null;
|
|
649
|
+
},
|
|
650
|
+
async appendOutcome(outcome) {
|
|
651
|
+
if (!directory) ensureSecureHomeDir();
|
|
652
|
+
mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
653
|
+
// Append-only ledger: one JSON object per line, never rewritten.
|
|
654
|
+
if (!existsSync(outcomesPath)) writeSecureFile(outcomesPath, "");
|
|
655
|
+
appendFileSync(outcomesPath, `${JSON.stringify(outcome)}\n`, { mode: 0o600 });
|
|
656
|
+
return outcome;
|
|
657
|
+
},
|
|
658
|
+
async listOutcomes() {
|
|
659
|
+
return readOutcomes();
|
|
660
|
+
},
|
|
661
|
+
async allSignals() {
|
|
662
|
+
return listRunsSync().flatMap((run) => run.signals);
|
|
663
|
+
},
|
|
664
|
+
};
|
|
665
|
+
}
|
|
666
|
+
|
|
667
|
+
/** Construct a SignalOutcome with a stable id and a recordedAt stamp. */
|
|
668
|
+
export function makeOutcome(input: {
|
|
669
|
+
accountDomain: string;
|
|
670
|
+
touchId?: string;
|
|
671
|
+
result: SignalOutcomeResult;
|
|
672
|
+
creditedSignals?: string[];
|
|
673
|
+
recordedAt?: string;
|
|
674
|
+
}): SignalOutcome {
|
|
675
|
+
const recordedAt = input.recordedAt ?? new Date().toISOString();
|
|
676
|
+
const accountDomain = normalizeAccountDomain(input.accountDomain);
|
|
677
|
+
return {
|
|
678
|
+
id: outcomeId({ accountDomain, touchId: input.touchId, result: input.result, recordedAt }),
|
|
679
|
+
accountDomain,
|
|
680
|
+
...(input.touchId !== undefined ? { touchId: input.touchId } : {}),
|
|
681
|
+
result: input.result,
|
|
682
|
+
recordedAt,
|
|
683
|
+
creditedSignals: input.creditedSignals ?? [],
|
|
684
|
+
};
|
|
685
|
+
}
|