@agent-pattern-labs/leads-rig 0.1.3 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.codex/config.toml +1 -1
- package/.cursor/rules/main.mdc +2 -2
- package/.opencode/skills/lead-harness.md +2 -2
- package/.opencode/skills/public-leads.md +2 -2
- package/.pi/iso-route.md +15 -0
- package/.pi/prompts/lead-harness.md +2 -2
- package/.pi/prompts/public-leads.md +2 -2
- package/.pi/settings.json +9 -0
- package/AGENTS.md +2 -2
- package/CLAUDE.md +2 -2
- package/README.md +23 -6
- package/bin/create-leads-harness.mjs +2 -2
- package/bin/lead-harness.mjs +75 -75
- package/config/profile.example.yml +8 -5
- package/docs/ARCHITECTURE.md +3 -3
- package/docs/CONSTRUCTION.md +2 -2
- package/docs/SETUP.md +23 -5
- package/iso/commands/lead-harness.md +2 -2
- package/iso/commands/public-leads.md +2 -2
- package/iso/instructions.md +2 -2
- package/lib/leadharness-crawler.mjs +17 -13
- package/lib/leadharness-ingest.mjs +23 -6
- package/lib/leadharness-leads.mjs +277 -4
- package/modes/_shared.md +4 -2
- package/modes/batch.md +1 -1
- package/modes/ingest.md +6 -3
- package/modes/pipeline.md +1 -1
- package/modes/setup.md +3 -2
- package/package.json +25 -25
- package/scripts/batch-orchestrator.mjs +1 -1
- package/scripts/ingest.mjs +2 -2
- package/scripts/pipeline.mjs +2 -2
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { createHash } from 'crypto';
|
|
2
2
|
import { lookup, resolveMx } from 'dns/promises';
|
|
3
3
|
import { setTimeout as sleep } from 'timers/promises';
|
|
4
|
-
import { cleanDomains, normalizeDomain } from './leadharness-leads.mjs';
|
|
4
|
+
import { cleanDomains, dedupeLeadRecords, isRejectedLeadEmail, mergeLeadRecords, normalizeDomain } from './leadharness-leads.mjs';
|
|
5
5
|
|
|
6
6
|
const DEFAULT_MAX_PAGES = 10;
|
|
7
7
|
const HARD_MAX_PAGES = 25;
|
|
@@ -51,16 +51,14 @@ export async function crawlDomains(inputs, options = {}) {
|
|
|
51
51
|
for (const lead of result.leads) {
|
|
52
52
|
const key = leadKey(lead);
|
|
53
53
|
const current = leadsByKey.get(key);
|
|
54
|
-
|
|
55
|
-
leadsByKey.set(key, lead);
|
|
56
|
-
}
|
|
54
|
+
leadsByKey.set(key, current ? mergeLeadRecords(current, lead) : lead);
|
|
57
55
|
}
|
|
58
56
|
} catch (error) {
|
|
59
57
|
errors.push(`${target.domain}: ${error instanceof Error ? error.message : String(error)}`);
|
|
60
58
|
}
|
|
61
59
|
}
|
|
62
60
|
|
|
63
|
-
const leads = [...leadsByKey.values()].sort(sortLead);
|
|
61
|
+
const leads = dedupeLeadRecords([...leadsByKey.values()]).sort(sortLead);
|
|
64
62
|
const domains = cleanDomains([
|
|
65
63
|
...targets.map((target) => target.domain),
|
|
66
64
|
...results.map((result) => result.domain),
|
|
@@ -173,9 +171,7 @@ async function crawlDomainTarget(target, options) {
|
|
|
173
171
|
if (!isRelevantLead(lead, { minConfidence, includeBlocked: options.includeBlocked })) continue;
|
|
174
172
|
const key = leadKey(lead);
|
|
175
173
|
const current = leadsByKey.get(key);
|
|
176
|
-
|
|
177
|
-
leadsByKey.set(key, lead);
|
|
178
|
-
}
|
|
174
|
+
leadsByKey.set(key, current ? mergeLeadRecords(current, lead) : lead);
|
|
179
175
|
}
|
|
180
176
|
|
|
181
177
|
if (page.hasForm && isContactLikeURL(page.url)) {
|
|
@@ -183,9 +179,7 @@ async function crawlDomainTarget(target, options) {
|
|
|
183
179
|
if (isRelevantLead(lead, { minConfidence, includeBlocked: options.includeBlocked })) {
|
|
184
180
|
const key = leadKey(lead);
|
|
185
181
|
const current = leadsByKey.get(key);
|
|
186
|
-
|
|
187
|
-
leadsByKey.set(key, lead);
|
|
188
|
-
}
|
|
182
|
+
leadsByKey.set(key, current ? mergeLeadRecords(current, lead) : lead);
|
|
189
183
|
}
|
|
190
184
|
}
|
|
191
185
|
|
|
@@ -199,7 +193,7 @@ async function crawlDomainTarget(target, options) {
|
|
|
199
193
|
await sleep(boundedInteger(options.delayMs, DEFAULT_DELAY_MS, 0, 10_000));
|
|
200
194
|
}
|
|
201
195
|
|
|
202
|
-
const leads = [...leadsByKey.values()].sort(sortLead);
|
|
196
|
+
const leads = dedupeLeadRecords([...leadsByKey.values()]).sort(sortLead);
|
|
203
197
|
const completedAt = new Date().toISOString();
|
|
204
198
|
if (pages.length === 0) {
|
|
205
199
|
warnings.push(`no pages could be crawled for ${target.domain}`);
|
|
@@ -372,7 +366,7 @@ function extractEmails(html) {
|
|
|
372
366
|
for (const value of values) {
|
|
373
367
|
for (const found of String(value || '').matchAll(EMAIL_RE)) {
|
|
374
368
|
const email = found[0].toLowerCase().replace(/^mailto:/, '').replace(/[.,;:!?()[\]{}<>"']+$/g, '');
|
|
375
|
-
if (email && !isLikelyAssetEmail(email)) emails.add(email);
|
|
369
|
+
if (email && !isLikelyAssetEmail(email) && !isRejectedLeadEmail(email)) emails.add(email);
|
|
376
370
|
}
|
|
377
371
|
}
|
|
378
372
|
return [...emails].sort();
|
|
@@ -502,6 +496,11 @@ async function buildEmailLead(email, companyDomain, websiteUrl, page) {
|
|
|
502
496
|
emailType,
|
|
503
497
|
sourceUrl,
|
|
504
498
|
sourceLabel: sourceLabel(sourceUrl),
|
|
499
|
+
sources: [{
|
|
500
|
+
url: sourceUrl,
|
|
501
|
+
label: sourceLabel(sourceUrl),
|
|
502
|
+
evidence: snippet,
|
|
503
|
+
}],
|
|
505
504
|
evidence: snippet,
|
|
506
505
|
extractionMethod: 'agentic_harness_public_page',
|
|
507
506
|
verificationStatus: emailType === 'blocked' ? 'blocked' : verificationStatus,
|
|
@@ -533,6 +532,11 @@ function buildContactPath(domain, websiteUrl, page) {
|
|
|
533
532
|
emailType: 'contact_path',
|
|
534
533
|
sourceUrl,
|
|
535
534
|
sourceLabel: sourceLabel(sourceUrl),
|
|
535
|
+
sources: [{
|
|
536
|
+
url: sourceUrl,
|
|
537
|
+
label: sourceLabel(sourceUrl),
|
|
538
|
+
evidence: contactPathEvidence(page),
|
|
539
|
+
}],
|
|
536
540
|
evidence: contactPathEvidence(page),
|
|
537
541
|
extractionMethod: 'agentic_harness_contact_form',
|
|
538
542
|
verificationStatus: 'not_applicable',
|
|
@@ -14,10 +14,18 @@ export function resolveIngestConfig(opts = {}, profile = loadProfileConfig()) {
|
|
|
14
14
|
const configuredOperatorEmail = opts.operatorEmail || env.PUBLIC_LEADS_OPERATOR_EMAIL || env.LEAD_HARNESS_OPERATOR_EMAIL || profile.operatorEmail;
|
|
15
15
|
const adminEmails = splitList(env.ADMIN_EMAILS);
|
|
16
16
|
const operatorEmail = configuredOperatorEmail || legacyOperatorEmail || adminEmails[0] || '';
|
|
17
|
-
const tokenEnv = opts.tokenEnv
|
|
17
|
+
const tokenEnv = opts.tokenEnv
|
|
18
|
+
|| profile.authTokenEnv
|
|
19
|
+
|| profile.adminTokenEnv
|
|
20
|
+
|| (env.PUBLIC_LEADS_API_TOKEN ? 'PUBLIC_LEADS_API_TOKEN' : '')
|
|
21
|
+
|| (env.LEAD_HARNESS_API_TOKEN ? 'LEAD_HARNESS_API_TOKEN' : '')
|
|
22
|
+
|| 'PUBLIC_LEADS_API_TOKEN';
|
|
23
|
+
const baseUrl = opts.api || env.PUBLIC_LEADS_API || env.LEAD_HARNESS_API || profile.baseUrl || '';
|
|
18
24
|
|
|
19
25
|
return {
|
|
20
|
-
endpoint:
|
|
26
|
+
endpoint: baseUrl
|
|
27
|
+
? `${stripTrailingSlash(baseUrl)}${normalizeEndpointPath(opts.ingestPath || env.PUBLIC_LEADS_INGEST_PATH || env.LEAD_HARNESS_INGEST_PATH || profile.ingestPath || '/api/lead-ingests')}`
|
|
28
|
+
: '',
|
|
21
29
|
operatorEmail,
|
|
22
30
|
operatorEmailHeader: opts.operatorEmailHeader
|
|
23
31
|
|| opts.adminEmailHeader
|
|
@@ -25,8 +33,7 @@ export function resolveIngestConfig(opts = {}, profile = loadProfileConfig()) {
|
|
|
25
33
|
|| env.LEAD_HARNESS_OPERATOR_EMAIL_HEADER
|
|
26
34
|
|| profile.operatorEmailHeader
|
|
27
35
|
|| profile.adminEmailHeader
|
|
28
|
-
||
|
|
29
|
-
|| (operatorEmail ? 'X-Operator-Email' : ''),
|
|
36
|
+
|| 'X-Admin-Email',
|
|
30
37
|
authHeader: opts.authHeader || env.PUBLIC_LEADS_AUTH_HEADER || env.LEAD_HARNESS_AUTH_HEADER || profile.authHeader || 'Authorization',
|
|
31
38
|
authScheme: opts.authScheme || env.PUBLIC_LEADS_AUTH_SCHEME || env.LEAD_HARNESS_AUTH_SCHEME || profile.authScheme || 'Bearer',
|
|
32
39
|
tokenEnv,
|
|
@@ -56,6 +63,10 @@ export async function ingestPayload(payload, opts = {}) {
|
|
|
56
63
|
errors: payload.errors,
|
|
57
64
|
};
|
|
58
65
|
|
|
66
|
+
if (!config.endpoint) {
|
|
67
|
+
throw new Error('ingest API base URL is required (--api, $PUBLIC_LEADS_API, $LEAD_HARNESS_API, or api.base_url)');
|
|
68
|
+
}
|
|
69
|
+
|
|
59
70
|
if (opts.dryRun) {
|
|
60
71
|
const output = {
|
|
61
72
|
status: 'DRY RUN',
|
|
@@ -69,10 +80,16 @@ export async function ingestPayload(payload, opts = {}) {
|
|
|
69
80
|
}
|
|
70
81
|
|
|
71
82
|
if (config.operatorEmailHeader && !config.operatorEmail) {
|
|
72
|
-
throw new Error('operator email is required when an operator email header is configured');
|
|
83
|
+
throw new Error('operator email is required when an operator email header is configured (--operator-email, $PUBLIC_LEADS_OPERATOR_EMAIL, or api.operator_email)');
|
|
73
84
|
}
|
|
74
85
|
if (!config.token) {
|
|
75
|
-
|
|
86
|
+
const tokenHints = [...new Set([
|
|
87
|
+
`$${config.tokenEnv}`,
|
|
88
|
+
'$PUBLIC_LEADS_API_TOKEN',
|
|
89
|
+
'$LEAD_HARNESS_API_TOKEN',
|
|
90
|
+
'$ADMIN_API_TOKEN',
|
|
91
|
+
])];
|
|
92
|
+
throw new Error(`API token is required (--token, ${tokenHints.slice(0, -1).join(', ')}, or ${tokenHints.at(-1)})`);
|
|
76
93
|
}
|
|
77
94
|
|
|
78
95
|
const headers = {
|
|
@@ -28,6 +28,37 @@ export const ALLOWED_VERIFICATION_STATUSES = new Set([
|
|
|
28
28
|
'blocked',
|
|
29
29
|
'unknown',
|
|
30
30
|
]);
|
|
31
|
+
export const REJECTED_EMAIL_LOCAL_PREFIXES = [
|
|
32
|
+
'admin',
|
|
33
|
+
'booking',
|
|
34
|
+
'bookings',
|
|
35
|
+
'business',
|
|
36
|
+
'contact',
|
|
37
|
+
'customerservice',
|
|
38
|
+
'customer-service',
|
|
39
|
+
'customersuccess',
|
|
40
|
+
'customer-success',
|
|
41
|
+
'events',
|
|
42
|
+
'hello',
|
|
43
|
+
'help',
|
|
44
|
+
'hi',
|
|
45
|
+
'info',
|
|
46
|
+
'inquiries',
|
|
47
|
+
'enquiries',
|
|
48
|
+
'media',
|
|
49
|
+
'office',
|
|
50
|
+
'outreach',
|
|
51
|
+
'partners',
|
|
52
|
+
'partnerships',
|
|
53
|
+
'press',
|
|
54
|
+
'sales',
|
|
55
|
+
'service',
|
|
56
|
+
'services',
|
|
57
|
+
'speaking',
|
|
58
|
+
'success',
|
|
59
|
+
'support',
|
|
60
|
+
'team',
|
|
61
|
+
];
|
|
31
62
|
|
|
32
63
|
const EMAIL_RE = /^[^\s@]+@[^\s@]+\.[^\s@]+$/;
|
|
33
64
|
|
|
@@ -94,7 +125,7 @@ export function normalizePayload(value, { now = new Date().toISOString(), source
|
|
|
94
125
|
throw new Error('lead artifact must be a JSON array, JSONL records, or an object with a leads array');
|
|
95
126
|
}
|
|
96
127
|
|
|
97
|
-
const leads = payload.leads.map((lead) => normalizeLead(lead, { now }));
|
|
128
|
+
const leads = dedupeLeadRecords(payload.leads.map((lead) => normalizeLead(lead, { now })));
|
|
98
129
|
const results = Array.isArray(payload.results)
|
|
99
130
|
? payload.results.map((result) => normalizeResult(result, { now }))
|
|
100
131
|
: [];
|
|
@@ -146,6 +177,11 @@ export function normalizeLead(input, { now = new Date().toISOString() } = {}) {
|
|
|
146
177
|
emailType,
|
|
147
178
|
sourceUrl,
|
|
148
179
|
sourceLabel: stringValue(input.sourceLabel) || sourceUrl,
|
|
180
|
+
sources: normalizeLeadSources(input.sources, {
|
|
181
|
+
sourceUrl,
|
|
182
|
+
sourceLabel: stringValue(input.sourceLabel) || sourceUrl,
|
|
183
|
+
evidence: compactWhitespace(stringValue(input.evidence)),
|
|
184
|
+
}),
|
|
149
185
|
evidence: compactWhitespace(stringValue(input.evidence)),
|
|
150
186
|
extractionMethod: stringValue(input.extractionMethod) || 'agentic_harness',
|
|
151
187
|
verificationStatus,
|
|
@@ -158,7 +194,7 @@ export function normalizeLead(input, { now = new Date().toISOString() } = {}) {
|
|
|
158
194
|
};
|
|
159
195
|
|
|
160
196
|
if (!lead.id) lead.id = stableLeadID(lead);
|
|
161
|
-
return lead;
|
|
197
|
+
return normalizeLeadRecord(lead);
|
|
162
198
|
}
|
|
163
199
|
|
|
164
200
|
export function normalizeResult(input, { now = new Date().toISOString() } = {}) {
|
|
@@ -169,7 +205,9 @@ export function normalizeResult(input, { now = new Date().toISOString() } = {})
|
|
|
169
205
|
return {
|
|
170
206
|
domain,
|
|
171
207
|
websiteUrl: stringValue(input.websiteUrl) || (domain ? `https://${domain}/` : ''),
|
|
172
|
-
leads: Array.isArray(input.leads)
|
|
208
|
+
leads: Array.isArray(input.leads)
|
|
209
|
+
? dedupeLeadRecords(input.leads.map((lead) => normalizeLead(lead, { now })))
|
|
210
|
+
: [],
|
|
173
211
|
pages: Array.isArray(input.pages) ? input.pages.map(normalizePageVisit) : [],
|
|
174
212
|
warnings: normalizeWarnings(input.warnings),
|
|
175
213
|
completedAt: normalizeDateTime(input.completedAt, now),
|
|
@@ -225,6 +263,9 @@ export function validatePayload(payload, { allowEmpty = false } = {}) {
|
|
|
225
263
|
|
|
226
264
|
results.forEach((result, index) => {
|
|
227
265
|
validateResult(result, `results[${index}]`, issues);
|
|
266
|
+
result.leads.forEach((lead, leadIndex) => {
|
|
267
|
+
validateLead(lead, `results[${index}].leads[${leadIndex}]`, issues);
|
|
268
|
+
});
|
|
228
269
|
});
|
|
229
270
|
|
|
230
271
|
const errors = issues.filter((item) => item.severity === 'error').length;
|
|
@@ -252,6 +293,9 @@ export function validateLead(lead, path, issues) {
|
|
|
252
293
|
if (lead.email && !EMAIL_RE.test(lead.email)) {
|
|
253
294
|
issues.push(issue('error', `${path}.email`, 'invalid_email', 'email must be a valid email address when present'));
|
|
254
295
|
}
|
|
296
|
+
if (lead.email && isRejectedLeadEmail(lead.email)) {
|
|
297
|
+
issues.push(issue('error', `${path}.email`, 'generic_inbox_rejected', 'generic company inboxes such as info@, hello@, or support@ are not accepted; submit named people instead'));
|
|
298
|
+
}
|
|
255
299
|
if (lead.emailType && !ALLOWED_EMAIL_TYPES.has(lead.emailType)) {
|
|
256
300
|
issues.push(issue('error', `${path}.emailType`, 'invalid_email_type', `emailType must be one of ${[...ALLOWED_EMAIL_TYPES].join(', ')}`));
|
|
257
301
|
}
|
|
@@ -299,6 +343,19 @@ export function validateLead(lead, path, issues) {
|
|
|
299
343
|
if (!Array.isArray(lead.warnings)) {
|
|
300
344
|
issues.push(issue('error', `${path}.warnings`, 'invalid_warnings', 'warnings must be an array'));
|
|
301
345
|
}
|
|
346
|
+
if (!Array.isArray(lead.sources)) {
|
|
347
|
+
issues.push(issue('error', `${path}.sources`, 'invalid_sources', 'sources must be an array'));
|
|
348
|
+
} else {
|
|
349
|
+
lead.sources.forEach((source, index) => {
|
|
350
|
+
if (!source || typeof source !== 'object' || Array.isArray(source)) {
|
|
351
|
+
issues.push(issue('error', `${path}.sources[${index}]`, 'invalid_source', 'sources entries must be objects'));
|
|
352
|
+
return;
|
|
353
|
+
}
|
|
354
|
+
if (!isHttpUrl(source.url)) {
|
|
355
|
+
issues.push(issue('error', `${path}.sources[${index}].url`, 'invalid_source_url', 'sources entries require an http(s) URL'));
|
|
356
|
+
}
|
|
357
|
+
});
|
|
358
|
+
}
|
|
302
359
|
if (lead.foundAt && Number.isNaN(Date.parse(lead.foundAt))) {
|
|
303
360
|
issues.push(issue('error', `${path}.foundAt`, 'invalid_found_at', 'foundAt must be ISO datetime'));
|
|
304
361
|
}
|
|
@@ -447,6 +504,127 @@ export function domainFromUrl(value) {
|
|
|
447
504
|
}
|
|
448
505
|
}
|
|
449
506
|
|
|
507
|
+
export function dedupeLeadRecords(leads) {
|
|
508
|
+
const byKey = new Map();
|
|
509
|
+
for (const rawLead of leads || []) {
|
|
510
|
+
const lead = normalizeLeadRecord(rawLead);
|
|
511
|
+
const key = stableLeadKey(lead);
|
|
512
|
+
const current = byKey.get(key);
|
|
513
|
+
byKey.set(key, current ? mergeLeadRecords(current, lead) : lead);
|
|
514
|
+
}
|
|
515
|
+
return [...byKey.values()];
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
function normalizeLeadRecord(lead) {
|
|
519
|
+
const sources = normalizeLeadSources(lead?.sources, {
|
|
520
|
+
sourceUrl: lead?.sourceUrl,
|
|
521
|
+
sourceLabel: lead?.sourceLabel,
|
|
522
|
+
evidence: lead?.evidence,
|
|
523
|
+
});
|
|
524
|
+
const normalized = {
|
|
525
|
+
...lead,
|
|
526
|
+
sourceUrl: sources[0]?.url || stringValue(lead?.sourceUrl),
|
|
527
|
+
sourceLabel: sources[0]?.label || stringValue(lead?.sourceLabel),
|
|
528
|
+
sources,
|
|
529
|
+
evidence: sources[0]?.evidence || compactWhitespace(lead?.evidence),
|
|
530
|
+
warnings: normalizeWarnings(lead?.warnings),
|
|
531
|
+
socialUrls: normalizeUrlList(lead?.socialUrls),
|
|
532
|
+
contactUrls: normalizeUrlList(lead?.contactUrls),
|
|
533
|
+
};
|
|
534
|
+
normalized.id = stableLeadID(normalized);
|
|
535
|
+
return normalized;
|
|
536
|
+
}
|
|
537
|
+
|
|
538
|
+
export function mergeLeadRecords(current, candidate) {
|
|
539
|
+
const left = normalizeLeadRecord(current);
|
|
540
|
+
const right = normalizeLeadRecord(candidate);
|
|
541
|
+
const primary = leadQualityScore(right) > leadQualityScore(left) ? right : left;
|
|
542
|
+
const secondary = primary === right ? left : right;
|
|
543
|
+
|
|
544
|
+
const merged = {
|
|
545
|
+
...primary,
|
|
546
|
+
company: primary.company || secondary.company,
|
|
547
|
+
domain: primary.domain || secondary.domain,
|
|
548
|
+
websiteUrl: primary.websiteUrl || secondary.websiteUrl,
|
|
549
|
+
contactName: primary.contactName || secondary.contactName,
|
|
550
|
+
title: primary.title || secondary.title,
|
|
551
|
+
email: primary.email || secondary.email,
|
|
552
|
+
emailType: primary.emailType || secondary.emailType,
|
|
553
|
+
extractionMethod: primary.extractionMethod || secondary.extractionMethod,
|
|
554
|
+
verificationStatus: verificationRank(secondary.verificationStatus) > verificationRank(primary.verificationStatus)
|
|
555
|
+
? secondary.verificationStatus
|
|
556
|
+
: primary.verificationStatus,
|
|
557
|
+
confidence: Math.max(primary.confidence || 0, secondary.confidence || 0),
|
|
558
|
+
phone: primary.phone || secondary.phone,
|
|
559
|
+
socialUrls: normalizeUrlList([...(primary.socialUrls || []), ...(secondary.socialUrls || [])]),
|
|
560
|
+
contactUrls: normalizeUrlList([...(primary.contactUrls || []), ...(secondary.contactUrls || [])]),
|
|
561
|
+
warnings: normalizeWarnings([...(primary.warnings || []), ...(secondary.warnings || [])]),
|
|
562
|
+
};
|
|
563
|
+
|
|
564
|
+
merged.foundAt = earliestDateTime(primary.foundAt, secondary.foundAt);
|
|
565
|
+
if ((secondary.evidence || '').length > (merged.evidence || '').length) {
|
|
566
|
+
merged.evidence = secondary.evidence;
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
merged.sources = mergeLeadSources(merged.sourceUrl, left.sources, right.sources);
|
|
570
|
+
merged.sourceUrl = merged.sources[0]?.url || merged.sourceUrl;
|
|
571
|
+
merged.sourceLabel = merged.sources[0]?.label || merged.sourceLabel;
|
|
572
|
+
if (!merged.evidence) {
|
|
573
|
+
merged.evidence = merged.sources[0]?.evidence || '';
|
|
574
|
+
}
|
|
575
|
+
merged.id = stableLeadID(merged);
|
|
576
|
+
return normalizeLeadRecord(merged);
|
|
577
|
+
}
|
|
578
|
+
|
|
579
|
+
function normalizeLeadSources(value, fallback = {}) {
|
|
580
|
+
const byUrl = new Map();
|
|
581
|
+
const ordered = [];
|
|
582
|
+
const addSource = (source) => {
|
|
583
|
+
const url = stringValue(source?.url);
|
|
584
|
+
if (!url) return;
|
|
585
|
+
|
|
586
|
+
const entry = {
|
|
587
|
+
url,
|
|
588
|
+
label: stringValue(source?.label) || defaultSourceLabel(url),
|
|
589
|
+
evidence: compactWhitespace(source?.evidence),
|
|
590
|
+
};
|
|
591
|
+
const current = byUrl.get(url);
|
|
592
|
+
if (!current) {
|
|
593
|
+
byUrl.set(url, entry);
|
|
594
|
+
ordered.push(url);
|
|
595
|
+
return;
|
|
596
|
+
}
|
|
597
|
+
byUrl.set(url, {
|
|
598
|
+
url,
|
|
599
|
+
label: current.label || entry.label,
|
|
600
|
+
evidence: entry.evidence.length > current.evidence.length ? entry.evidence : current.evidence,
|
|
601
|
+
});
|
|
602
|
+
};
|
|
603
|
+
|
|
604
|
+
if (Array.isArray(value)) {
|
|
605
|
+
value.forEach(addSource);
|
|
606
|
+
}
|
|
607
|
+
addSource({
|
|
608
|
+
url: fallback.sourceUrl,
|
|
609
|
+
label: fallback.sourceLabel,
|
|
610
|
+
evidence: fallback.evidence,
|
|
611
|
+
});
|
|
612
|
+
|
|
613
|
+
const preferredUrl = stringValue(fallback.sourceUrl);
|
|
614
|
+
return ordered
|
|
615
|
+
.map((url) => byUrl.get(url))
|
|
616
|
+
.sort((left, right) => sourceSortValue(right, preferredUrl) - sourceSortValue(left, preferredUrl) || left.label.localeCompare(right.label) || left.url.localeCompare(right.url));
|
|
617
|
+
}
|
|
618
|
+
|
|
619
|
+
function mergeLeadSources(preferredUrl, ...lists) {
|
|
620
|
+
return normalizeLeadSources(lists.flat(), { sourceUrl: preferredUrl });
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
function sourceSortValue(source, preferredUrl) {
|
|
624
|
+
if (preferredUrl && source.url === preferredUrl) return 10_000;
|
|
625
|
+
return pagePriority(source.url || '') * 100 + Math.min((source.evidence || '').length, 99);
|
|
626
|
+
}
|
|
627
|
+
|
|
450
628
|
export function stableLeadID(lead) {
|
|
451
629
|
const key = stableLeadKey(lead);
|
|
452
630
|
if (!key.replace(/\|/g, '').trim()) return `agentic-${Date.now()}`;
|
|
@@ -469,10 +647,105 @@ function stableLeadKey(lead) {
|
|
|
469
647
|
].join('|').toLowerCase();
|
|
470
648
|
}
|
|
471
649
|
|
|
650
|
+
function leadQualityScore(lead) {
|
|
651
|
+
return (lead?.confidence || 0) * 1000
|
|
652
|
+
+ verificationRank(lead?.verificationStatus) * 100
|
|
653
|
+
+ pagePriority(lead?.sourceUrl || '') * 10
|
|
654
|
+
+ (Array.isArray(lead?.sources) ? lead.sources.length : 0) * 5
|
|
655
|
+
+ (lead?.phone ? 25 : 0)
|
|
656
|
+
+ (Array.isArray(lead?.socialUrls) ? lead.socialUrls.length : 0) * 5
|
|
657
|
+
+ (Array.isArray(lead?.contactUrls) ? lead.contactUrls.length : 0) * 5
|
|
658
|
+
+ (lead?.contactName ? 20 : 0)
|
|
659
|
+
+ (lead?.title ? 10 : 0);
|
|
660
|
+
}
|
|
661
|
+
|
|
662
|
+
function verificationRank(value) {
|
|
663
|
+
const status = stringValue(value).toLowerCase();
|
|
664
|
+
if (status === 'verified' || status === 'mx_verified') return 4;
|
|
665
|
+
if (status === 'unverified') return 3;
|
|
666
|
+
if (status === 'not_applicable') return 2;
|
|
667
|
+
if (status === 'unknown') return 1;
|
|
668
|
+
return 0;
|
|
669
|
+
}
|
|
670
|
+
|
|
671
|
+
function earliestDateTime(...values) {
|
|
672
|
+
const dates = values
|
|
673
|
+
.map((value) => stringValue(value))
|
|
674
|
+
.filter(Boolean)
|
|
675
|
+
.map((value) => new Date(value))
|
|
676
|
+
.filter((value) => !Number.isNaN(value.getTime()))
|
|
677
|
+
.sort((left, right) => left.getTime() - right.getTime());
|
|
678
|
+
return dates[0]?.toISOString() || new Date().toISOString();
|
|
679
|
+
}
|
|
680
|
+
|
|
681
|
+
function defaultSourceLabel(sourceUrl) {
|
|
682
|
+
const path = safePath(sourceUrl).replace(/^\/+|\/+$/g, '');
|
|
683
|
+
if (!path) return 'Home';
|
|
684
|
+
const last = path.split('/').filter(Boolean).pop() || path;
|
|
685
|
+
return last.replace(/[-_]+/g, ' ').replace(/\b\w/g, (char) => char.toUpperCase());
|
|
686
|
+
}
|
|
687
|
+
|
|
688
|
+
function safePath(value) {
|
|
689
|
+
try {
|
|
690
|
+
return new URL(value).pathname.toLowerCase();
|
|
691
|
+
} catch {
|
|
692
|
+
return '';
|
|
693
|
+
}
|
|
694
|
+
}
|
|
695
|
+
|
|
696
|
+
function pagePriority(link) {
|
|
697
|
+
const path = safePath(link);
|
|
698
|
+
const weighted = [
|
|
699
|
+
['contact', 100],
|
|
700
|
+
['about', 90],
|
|
701
|
+
['team', 85],
|
|
702
|
+
['people', 85],
|
|
703
|
+
['leadership', 85],
|
|
704
|
+
['staff', 80],
|
|
705
|
+
['founder', 80],
|
|
706
|
+
['press', 65],
|
|
707
|
+
['media', 65],
|
|
708
|
+
['blog', 55],
|
|
709
|
+
['author', 55],
|
|
710
|
+
['career', 45],
|
|
711
|
+
['privacy', 25],
|
|
712
|
+
['legal', 20],
|
|
713
|
+
['impressum', 20],
|
|
714
|
+
];
|
|
715
|
+
for (const [needle, score] of weighted) {
|
|
716
|
+
if (path.includes(needle)) return score;
|
|
717
|
+
}
|
|
718
|
+
return path === '/' ? 50 : 0;
|
|
719
|
+
}
|
|
720
|
+
|
|
472
721
|
export function fileHash(path) {
|
|
473
722
|
return createHash('sha256').update(readFileSync(path)).digest('hex');
|
|
474
723
|
}
|
|
475
724
|
|
|
725
|
+
export function isRejectedLeadEmail(email) {
|
|
726
|
+
const local = String(email || '').trim().toLowerCase().split('@')[0] || '';
|
|
727
|
+
return isRejectedLeadLocalPart(local);
|
|
728
|
+
}
|
|
729
|
+
|
|
730
|
+
export function isRejectedLeadLocalPart(local) {
|
|
731
|
+
const value = normalizeRejectedLeadLocalPart(local);
|
|
732
|
+
if (!value) return false;
|
|
733
|
+
return REJECTED_EMAIL_LOCAL_PREFIXES.some((prefix) => {
|
|
734
|
+
if (value === prefix) return true;
|
|
735
|
+
if (!value.startsWith(prefix) || value.length === prefix.length) return false;
|
|
736
|
+
const next = value[prefix.length];
|
|
737
|
+
return next === '.' || next === '_' || next === '-' || next === '+' || /\d/.test(next);
|
|
738
|
+
});
|
|
739
|
+
}
|
|
740
|
+
|
|
741
|
+
function normalizeRejectedLeadLocalPart(local) {
|
|
742
|
+
let value = String(local || '').trim().toLowerCase();
|
|
743
|
+
for (const noise of ['mailto', 'u003e', 'email']) {
|
|
744
|
+
if (value.startsWith(noise)) value = value.slice(noise.length);
|
|
745
|
+
}
|
|
746
|
+
return value.replace(/^[>._+-]+/, '');
|
|
747
|
+
}
|
|
748
|
+
|
|
476
749
|
export function issue(severity, path, code, message) {
|
|
477
750
|
return { severity, path, code, message };
|
|
478
751
|
}
|
|
@@ -507,7 +780,7 @@ function normalizeVerificationStatus(value, emailType) {
|
|
|
507
780
|
|
|
508
781
|
function normalizeWarnings(value) {
|
|
509
782
|
if (!Array.isArray(value)) return [];
|
|
510
|
-
return value.map((item) => stringValue(item)).filter(Boolean);
|
|
783
|
+
return [...new Set(value.map((item) => stringValue(item)).filter(Boolean))];
|
|
511
784
|
}
|
|
512
785
|
|
|
513
786
|
function normalizeUrlList(value) {
|
package/modes/_shared.md
CHANGED
|
@@ -28,13 +28,14 @@ Required fields:
|
|
|
28
28
|
- Prioritize home, contact, about, team, leadership, people, press, blog, careers, and legal pages.
|
|
29
29
|
- Respect robots.txt, paywalls, login walls, and obvious anti-scraping notices.
|
|
30
30
|
- Do not infer email formats. Do not brute-force guessed addresses.
|
|
31
|
+
- Prefer named people over organizational inboxes. Do not emit generic catch-all emails such as `info@`, `hello@`, `contact@`, `support@`, `team@`, or similar aliases. If only a general contact path exists, emit `contact_path` instead.
|
|
31
32
|
- Keep contact forms as `emailType: "contact_path"` with empty `email`.
|
|
32
33
|
- Mark operational no-contact inboxes such as `noreply`, `abuse`, `security`, and `legal` as `blocked` with confidence `0`.
|
|
33
34
|
|
|
34
35
|
## Quality Heuristics
|
|
35
36
|
|
|
36
37
|
- `person`: named person or person-like email with source context.
|
|
37
|
-
- `role`: public role inbox
|
|
38
|
+
- `role`: public role inbox only when it is clearly role-specific and not a generic catch-all alias.
|
|
38
39
|
- `contact_path`: public form or page where no email is available.
|
|
39
40
|
- `blocked`: public address that should not be used for outbound.
|
|
40
41
|
|
|
@@ -45,6 +46,7 @@ Confidence should reflect source quality, person specificity, domain match, and
|
|
|
45
46
|
```bash
|
|
46
47
|
npx public-leads validate --input data/lead-results.json
|
|
47
48
|
npx public-leads manifest --input data/lead-results.json
|
|
48
|
-
npx public-leads ingest --input data/lead-results.json --dry-run
|
|
49
|
+
PUBLIC_LEADS_API=https://cold-agent-leads.example.com npx public-leads ingest --input data/lead-results.json --dry-run
|
|
50
|
+
npx public-leads pipeline --input data/domains.tsv --ingest
|
|
49
51
|
npx public-leads verify
|
|
50
52
|
```
|
package/modes/batch.md
CHANGED
|
@@ -10,7 +10,7 @@ Prefer:
|
|
|
10
10
|
batch/batch-runner.sh --parallel 2
|
|
11
11
|
```
|
|
12
12
|
|
|
13
|
-
The runner delegates to `scripts/batch-orchestrator.mjs`, which uses `@
|
|
13
|
+
The runner delegates to `scripts/batch-orchestrator.mjs`, which uses `@agent-pattern-labs/iso-orchestrator` for durable workflow records in `.public-leads-runs/`, bounded fan-out, and state updates.
|
|
14
14
|
|
|
15
15
|
## Input
|
|
16
16
|
|
package/modes/ingest.md
CHANGED
|
@@ -12,20 +12,23 @@ Use this when the user asks to store validated leads in a configured ingest API.
|
|
|
12
12
|
```bash
|
|
13
13
|
npx public-leads manifest --input <artifact>
|
|
14
14
|
```
|
|
15
|
-
3. Confirm ingest
|
|
15
|
+
3. Confirm upstream ingest settings are available through environment variables, flags, or `config/profile.yml`. Do not print token values.
|
|
16
|
+
- `PUBLIC_LEADS_API`
|
|
17
|
+
- `PUBLIC_LEADS_API_TOKEN`
|
|
18
|
+
- `PUBLIC_LEADS_OPERATOR_EMAIL`
|
|
16
19
|
|
|
17
20
|
## Dry Run
|
|
18
21
|
|
|
19
22
|
Use dry-run before live ingest unless the user explicitly requested live submission:
|
|
20
23
|
|
|
21
24
|
```bash
|
|
22
|
-
npx public-leads ingest --input <artifact> --dry-run --out data/ingest-response.json
|
|
25
|
+
PUBLIC_LEADS_API=https://cold-agent-leads.example.com npx public-leads ingest --input <artifact> --dry-run --out data/ingest-response.json
|
|
23
26
|
```
|
|
24
27
|
|
|
25
28
|
## Live Ingest
|
|
26
29
|
|
|
27
30
|
```bash
|
|
28
|
-
npx public-leads ingest --input <artifact> --
|
|
31
|
+
npx public-leads ingest --input <artifact> --out data/ingest-response.json
|
|
29
32
|
```
|
|
30
33
|
|
|
31
34
|
## Output
|
package/modes/pipeline.md
CHANGED
|
@@ -20,7 +20,7 @@ Read domains from the first available source:
|
|
|
20
20
|
5. For browser-heavy batch work, use `batch/batch-runner.sh --parallel 2`.
|
|
21
21
|
6. Validate every artifact with `npx public-leads validate`.
|
|
22
22
|
7. Merge validated artifacts into a manifest with `npx public-leads manifest --input <artifact>`.
|
|
23
|
-
8. Ingest with `npx public-leads pipeline --ingest
|
|
23
|
+
8. Ingest with `npx public-leads pipeline --ingest` only when upload is requested and upstream API env vars are configured.
|
|
24
24
|
|
|
25
25
|
## Output
|
|
26
26
|
|
package/modes/setup.md
CHANGED
|
@@ -9,7 +9,7 @@ Use this when the user asks to configure or inspect the lead harness.
|
|
|
9
9
|
- `data/domains.tsv`
|
|
10
10
|
- `data/pipeline.md`
|
|
11
11
|
- direct domains in the user request
|
|
12
|
-
3. Confirm the ingest API settings:
|
|
12
|
+
3. Confirm the upstream ingest API settings:
|
|
13
13
|
- `api.base_url`
|
|
14
14
|
- `api.ingest_path`
|
|
15
15
|
- `api.operator_email`
|
|
@@ -17,7 +17,8 @@ Use this when the user asks to configure or inspect the lead harness.
|
|
|
17
17
|
- `api.auth_header`
|
|
18
18
|
- `api.auth_scheme`
|
|
19
19
|
- `api.auth_token_env`
|
|
20
|
-
- `
|
|
20
|
+
- or `PUBLIC_LEADS_API`, `PUBLIC_LEADS_API_TOKEN`, and `PUBLIC_LEADS_OPERATOR_EMAIL`
|
|
21
|
+
- `api.target_project` only when intentionally reading credentials from a local Cold Agent Leads checkout
|
|
21
22
|
4. Run `npx public-leads verify`.
|
|
22
23
|
|
|
23
24
|
## Output
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@agent-pattern-labs/leads-rig",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.5",
|
|
4
4
|
"description": "Agentic public-web lead discovery harness with portable ingest artifacts",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"author": "Razroo",
|
|
@@ -155,32 +155,32 @@
|
|
|
155
155
|
"fast-uri": "3.1.2"
|
|
156
156
|
},
|
|
157
157
|
"dependencies": {
|
|
158
|
-
"@
|
|
159
|
-
"@
|
|
160
|
-
"@
|
|
161
|
-
"@
|
|
162
|
-
"@
|
|
163
|
-
"@
|
|
164
|
-
"@
|
|
165
|
-
"@
|
|
166
|
-
"@
|
|
167
|
-
"@
|
|
168
|
-
"@
|
|
169
|
-
"@
|
|
170
|
-
"@
|
|
171
|
-
"@
|
|
172
|
-
"@
|
|
173
|
-
"@
|
|
174
|
-
"@
|
|
175
|
-
"@
|
|
176
|
-
"@
|
|
158
|
+
"@agent-pattern-labs/iso-cache": "^0.1.1",
|
|
159
|
+
"@agent-pattern-labs/iso-canon": "^0.1.1",
|
|
160
|
+
"@agent-pattern-labs/iso-capabilities": "^0.1.1",
|
|
161
|
+
"@agent-pattern-labs/iso-context": "^0.1.1",
|
|
162
|
+
"@agent-pattern-labs/iso-contract": "^0.1.1",
|
|
163
|
+
"@agent-pattern-labs/iso-facts": "^0.1.1",
|
|
164
|
+
"@agent-pattern-labs/iso-guard": "^0.1.1",
|
|
165
|
+
"@agent-pattern-labs/iso-index": "^0.1.1",
|
|
166
|
+
"@agent-pattern-labs/iso-ledger": "^0.1.1",
|
|
167
|
+
"@agent-pattern-labs/iso-lineage": "^0.1.1",
|
|
168
|
+
"@agent-pattern-labs/iso-migrate": "^0.1.1",
|
|
169
|
+
"@agent-pattern-labs/iso-orchestrator": "^0.2.1",
|
|
170
|
+
"@agent-pattern-labs/iso-postflight": "^0.1.1",
|
|
171
|
+
"@agent-pattern-labs/iso-preflight": "^0.1.1",
|
|
172
|
+
"@agent-pattern-labs/iso-prioritize": "^0.1.1",
|
|
173
|
+
"@agent-pattern-labs/iso-redact": "^0.1.1",
|
|
174
|
+
"@agent-pattern-labs/iso-score": "^0.1.1",
|
|
175
|
+
"@agent-pattern-labs/iso-timeline": "^0.1.1",
|
|
176
|
+
"@agent-pattern-labs/iso-trace": "^0.5.1",
|
|
177
177
|
"playwright": "^1.58.1"
|
|
178
178
|
},
|
|
179
179
|
"devDependencies": {
|
|
180
|
-
"@
|
|
181
|
-
"@
|
|
182
|
-
"@
|
|
183
|
-
"@
|
|
184
|
-
"@
|
|
180
|
+
"@agent-pattern-labs/agentmd": "^0.3.1",
|
|
181
|
+
"@agent-pattern-labs/iso": "^0.3.2",
|
|
182
|
+
"@agent-pattern-labs/iso-eval": "^0.4.1",
|
|
183
|
+
"@agent-pattern-labs/iso-harness": "^0.8.1",
|
|
184
|
+
"@agent-pattern-labs/iso-route": "^0.6.1"
|
|
185
185
|
}
|
|
186
186
|
}
|