@agent-pattern-labs/leads-rig 0.1.3 → 0.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,7 +1,7 @@
1
1
  import { createHash } from 'crypto';
2
2
  import { lookup, resolveMx } from 'dns/promises';
3
3
  import { setTimeout as sleep } from 'timers/promises';
4
- import { cleanDomains, normalizeDomain } from './leadharness-leads.mjs';
4
+ import { cleanDomains, dedupeLeadRecords, isRejectedLeadEmail, mergeLeadRecords, normalizeDomain } from './leadharness-leads.mjs';
5
5
 
6
6
  const DEFAULT_MAX_PAGES = 10;
7
7
  const HARD_MAX_PAGES = 25;
@@ -51,16 +51,14 @@ export async function crawlDomains(inputs, options = {}) {
51
51
  for (const lead of result.leads) {
52
52
  const key = leadKey(lead);
53
53
  const current = leadsByKey.get(key);
54
- if (!current || isBetterLead(lead, current)) {
55
- leadsByKey.set(key, lead);
56
- }
54
+ leadsByKey.set(key, current ? mergeLeadRecords(current, lead) : lead);
57
55
  }
58
56
  } catch (error) {
59
57
  errors.push(`${target.domain}: ${error instanceof Error ? error.message : String(error)}`);
60
58
  }
61
59
  }
62
60
 
63
- const leads = [...leadsByKey.values()].sort(sortLead);
61
+ const leads = dedupeLeadRecords([...leadsByKey.values()]).sort(sortLead);
64
62
  const domains = cleanDomains([
65
63
  ...targets.map((target) => target.domain),
66
64
  ...results.map((result) => result.domain),
@@ -173,9 +171,7 @@ async function crawlDomainTarget(target, options) {
173
171
  if (!isRelevantLead(lead, { minConfidence, includeBlocked: options.includeBlocked })) continue;
174
172
  const key = leadKey(lead);
175
173
  const current = leadsByKey.get(key);
176
- if (!current || isBetterLead(lead, current)) {
177
- leadsByKey.set(key, lead);
178
- }
174
+ leadsByKey.set(key, current ? mergeLeadRecords(current, lead) : lead);
179
175
  }
180
176
 
181
177
  if (page.hasForm && isContactLikeURL(page.url)) {
@@ -183,9 +179,7 @@ async function crawlDomainTarget(target, options) {
183
179
  if (isRelevantLead(lead, { minConfidence, includeBlocked: options.includeBlocked })) {
184
180
  const key = leadKey(lead);
185
181
  const current = leadsByKey.get(key);
186
- if (!current || isBetterLead(lead, current)) {
187
- leadsByKey.set(key, lead);
188
- }
182
+ leadsByKey.set(key, current ? mergeLeadRecords(current, lead) : lead);
189
183
  }
190
184
  }
191
185
 
@@ -199,7 +193,7 @@ async function crawlDomainTarget(target, options) {
199
193
  await sleep(boundedInteger(options.delayMs, DEFAULT_DELAY_MS, 0, 10_000));
200
194
  }
201
195
 
202
- const leads = [...leadsByKey.values()].sort(sortLead);
196
+ const leads = dedupeLeadRecords([...leadsByKey.values()]).sort(sortLead);
203
197
  const completedAt = new Date().toISOString();
204
198
  if (pages.length === 0) {
205
199
  warnings.push(`no pages could be crawled for ${target.domain}`);
@@ -372,7 +366,7 @@ function extractEmails(html) {
372
366
  for (const value of values) {
373
367
  for (const found of String(value || '').matchAll(EMAIL_RE)) {
374
368
  const email = found[0].toLowerCase().replace(/^mailto:/, '').replace(/[.,;:!?()[\]{}<>"']+$/g, '');
375
- if (email && !isLikelyAssetEmail(email)) emails.add(email);
369
+ if (email && !isLikelyAssetEmail(email) && !isRejectedLeadEmail(email)) emails.add(email);
376
370
  }
377
371
  }
378
372
  return [...emails].sort();
@@ -502,6 +496,11 @@ async function buildEmailLead(email, companyDomain, websiteUrl, page) {
502
496
  emailType,
503
497
  sourceUrl,
504
498
  sourceLabel: sourceLabel(sourceUrl),
499
+ sources: [{
500
+ url: sourceUrl,
501
+ label: sourceLabel(sourceUrl),
502
+ evidence: snippet,
503
+ }],
505
504
  evidence: snippet,
506
505
  extractionMethod: 'agentic_harness_public_page',
507
506
  verificationStatus: emailType === 'blocked' ? 'blocked' : verificationStatus,
@@ -533,6 +532,11 @@ function buildContactPath(domain, websiteUrl, page) {
533
532
  emailType: 'contact_path',
534
533
  sourceUrl,
535
534
  sourceLabel: sourceLabel(sourceUrl),
535
+ sources: [{
536
+ url: sourceUrl,
537
+ label: sourceLabel(sourceUrl),
538
+ evidence: contactPathEvidence(page),
539
+ }],
536
540
  evidence: contactPathEvidence(page),
537
541
  extractionMethod: 'agentic_harness_contact_form',
538
542
  verificationStatus: 'not_applicable',
@@ -14,10 +14,18 @@ export function resolveIngestConfig(opts = {}, profile = loadProfileConfig()) {
14
14
  const configuredOperatorEmail = opts.operatorEmail || env.PUBLIC_LEADS_OPERATOR_EMAIL || env.LEAD_HARNESS_OPERATOR_EMAIL || profile.operatorEmail;
15
15
  const adminEmails = splitList(env.ADMIN_EMAILS);
16
16
  const operatorEmail = configuredOperatorEmail || legacyOperatorEmail || adminEmails[0] || '';
17
- const tokenEnv = opts.tokenEnv || profile.authTokenEnv || profile.adminTokenEnv || 'ADMIN_API_TOKEN';
17
+ const tokenEnv = opts.tokenEnv
18
+ || profile.authTokenEnv
19
+ || profile.adminTokenEnv
20
+ || (env.PUBLIC_LEADS_API_TOKEN ? 'PUBLIC_LEADS_API_TOKEN' : '')
21
+ || (env.LEAD_HARNESS_API_TOKEN ? 'LEAD_HARNESS_API_TOKEN' : '')
22
+ || 'PUBLIC_LEADS_API_TOKEN';
23
+ const baseUrl = opts.api || env.PUBLIC_LEADS_API || env.LEAD_HARNESS_API || profile.baseUrl || '';
18
24
 
19
25
  return {
20
- endpoint: `${stripTrailingSlash(opts.api || env.PUBLIC_LEADS_API || env.LEAD_HARNESS_API || profile.baseUrl || 'http://localhost:8080')}${normalizeEndpointPath(opts.ingestPath || env.PUBLIC_LEADS_INGEST_PATH || env.LEAD_HARNESS_INGEST_PATH || profile.ingestPath || '/api/lead-ingests')}`,
26
+ endpoint: baseUrl
27
+ ? `${stripTrailingSlash(baseUrl)}${normalizeEndpointPath(opts.ingestPath || env.PUBLIC_LEADS_INGEST_PATH || env.LEAD_HARNESS_INGEST_PATH || profile.ingestPath || '/api/lead-ingests')}`
28
+ : '',
21
29
  operatorEmail,
22
30
  operatorEmailHeader: opts.operatorEmailHeader
23
31
  || opts.adminEmailHeader
@@ -25,8 +33,7 @@ export function resolveIngestConfig(opts = {}, profile = loadProfileConfig()) {
25
33
  || env.LEAD_HARNESS_OPERATOR_EMAIL_HEADER
26
34
  || profile.operatorEmailHeader
27
35
  || profile.adminEmailHeader
28
- || (legacyOperatorEmail || adminEmails.length > 0 ? 'X-Admin-Email' : '')
29
- || (operatorEmail ? 'X-Operator-Email' : ''),
36
+ || 'X-Admin-Email',
30
37
  authHeader: opts.authHeader || env.PUBLIC_LEADS_AUTH_HEADER || env.LEAD_HARNESS_AUTH_HEADER || profile.authHeader || 'Authorization',
31
38
  authScheme: opts.authScheme || env.PUBLIC_LEADS_AUTH_SCHEME || env.LEAD_HARNESS_AUTH_SCHEME || profile.authScheme || 'Bearer',
32
39
  tokenEnv,
@@ -56,6 +63,10 @@ export async function ingestPayload(payload, opts = {}) {
56
63
  errors: payload.errors,
57
64
  };
58
65
 
66
+ if (!config.endpoint) {
67
+ throw new Error('ingest API base URL is required (--api, $PUBLIC_LEADS_API, $LEAD_HARNESS_API, or api.base_url)');
68
+ }
69
+
59
70
  if (opts.dryRun) {
60
71
  const output = {
61
72
  status: 'DRY RUN',
@@ -69,10 +80,16 @@ export async function ingestPayload(payload, opts = {}) {
69
80
  }
70
81
 
71
82
  if (config.operatorEmailHeader && !config.operatorEmail) {
72
- throw new Error('operator email is required when an operator email header is configured');
83
+ throw new Error('operator email is required when an operator email header is configured (--operator-email, $PUBLIC_LEADS_OPERATOR_EMAIL, or api.operator_email)');
73
84
  }
74
85
  if (!config.token) {
75
- throw new Error(`API token is required (--token or $${config.tokenEnv})`);
86
+ const tokenHints = [...new Set([
87
+ `$${config.tokenEnv}`,
88
+ '$PUBLIC_LEADS_API_TOKEN',
89
+ '$LEAD_HARNESS_API_TOKEN',
90
+ '$ADMIN_API_TOKEN',
91
+ ])];
92
+ throw new Error(`API token is required (--token, ${tokenHints.slice(0, -1).join(', ')}, or ${tokenHints.at(-1)})`);
76
93
  }
77
94
 
78
95
  const headers = {
@@ -28,6 +28,37 @@ export const ALLOWED_VERIFICATION_STATUSES = new Set([
28
28
  'blocked',
29
29
  'unknown',
30
30
  ]);
31
+ export const REJECTED_EMAIL_LOCAL_PREFIXES = [
32
+ 'admin',
33
+ 'booking',
34
+ 'bookings',
35
+ 'business',
36
+ 'contact',
37
+ 'customerservice',
38
+ 'customer-service',
39
+ 'customersuccess',
40
+ 'customer-success',
41
+ 'events',
42
+ 'hello',
43
+ 'help',
44
+ 'hi',
45
+ 'info',
46
+ 'inquiries',
47
+ 'enquiries',
48
+ 'media',
49
+ 'office',
50
+ 'outreach',
51
+ 'partners',
52
+ 'partnerships',
53
+ 'press',
54
+ 'sales',
55
+ 'service',
56
+ 'services',
57
+ 'speaking',
58
+ 'success',
59
+ 'support',
60
+ 'team',
61
+ ];
31
62
 
32
63
  const EMAIL_RE = /^[^\s@]+@[^\s@]+\.[^\s@]+$/;
33
64
 
@@ -94,7 +125,7 @@ export function normalizePayload(value, { now = new Date().toISOString(), source
94
125
  throw new Error('lead artifact must be a JSON array, JSONL records, or an object with a leads array');
95
126
  }
96
127
 
97
- const leads = payload.leads.map((lead) => normalizeLead(lead, { now }));
128
+ const leads = dedupeLeadRecords(payload.leads.map((lead) => normalizeLead(lead, { now })));
98
129
  const results = Array.isArray(payload.results)
99
130
  ? payload.results.map((result) => normalizeResult(result, { now }))
100
131
  : [];
@@ -146,6 +177,11 @@ export function normalizeLead(input, { now = new Date().toISOString() } = {}) {
146
177
  emailType,
147
178
  sourceUrl,
148
179
  sourceLabel: stringValue(input.sourceLabel) || sourceUrl,
180
+ sources: normalizeLeadSources(input.sources, {
181
+ sourceUrl,
182
+ sourceLabel: stringValue(input.sourceLabel) || sourceUrl,
183
+ evidence: compactWhitespace(stringValue(input.evidence)),
184
+ }),
149
185
  evidence: compactWhitespace(stringValue(input.evidence)),
150
186
  extractionMethod: stringValue(input.extractionMethod) || 'agentic_harness',
151
187
  verificationStatus,
@@ -158,7 +194,7 @@ export function normalizeLead(input, { now = new Date().toISOString() } = {}) {
158
194
  };
159
195
 
160
196
  if (!lead.id) lead.id = stableLeadID(lead);
161
- return lead;
197
+ return normalizeLeadRecord(lead);
162
198
  }
163
199
 
164
200
  export function normalizeResult(input, { now = new Date().toISOString() } = {}) {
@@ -169,7 +205,9 @@ export function normalizeResult(input, { now = new Date().toISOString() } = {})
169
205
  return {
170
206
  domain,
171
207
  websiteUrl: stringValue(input.websiteUrl) || (domain ? `https://${domain}/` : ''),
172
- leads: Array.isArray(input.leads) ? input.leads.map((lead) => normalizeLead(lead, { now })) : [],
208
+ leads: Array.isArray(input.leads)
209
+ ? dedupeLeadRecords(input.leads.map((lead) => normalizeLead(lead, { now })))
210
+ : [],
173
211
  pages: Array.isArray(input.pages) ? input.pages.map(normalizePageVisit) : [],
174
212
  warnings: normalizeWarnings(input.warnings),
175
213
  completedAt: normalizeDateTime(input.completedAt, now),
@@ -225,6 +263,9 @@ export function validatePayload(payload, { allowEmpty = false } = {}) {
225
263
 
226
264
  results.forEach((result, index) => {
227
265
  validateResult(result, `results[${index}]`, issues);
266
+ result.leads.forEach((lead, leadIndex) => {
267
+ validateLead(lead, `results[${index}].leads[${leadIndex}]`, issues);
268
+ });
228
269
  });
229
270
 
230
271
  const errors = issues.filter((item) => item.severity === 'error').length;
@@ -252,6 +293,9 @@ export function validateLead(lead, path, issues) {
252
293
  if (lead.email && !EMAIL_RE.test(lead.email)) {
253
294
  issues.push(issue('error', `${path}.email`, 'invalid_email', 'email must be a valid email address when present'));
254
295
  }
296
+ if (lead.email && isRejectedLeadEmail(lead.email)) {
297
+ issues.push(issue('error', `${path}.email`, 'generic_inbox_rejected', 'generic company inboxes such as info@, hello@, or support@ are not accepted; submit named people instead'));
298
+ }
255
299
  if (lead.emailType && !ALLOWED_EMAIL_TYPES.has(lead.emailType)) {
256
300
  issues.push(issue('error', `${path}.emailType`, 'invalid_email_type', `emailType must be one of ${[...ALLOWED_EMAIL_TYPES].join(', ')}`));
257
301
  }
@@ -299,6 +343,19 @@ export function validateLead(lead, path, issues) {
299
343
  if (!Array.isArray(lead.warnings)) {
300
344
  issues.push(issue('error', `${path}.warnings`, 'invalid_warnings', 'warnings must be an array'));
301
345
  }
346
+ if (!Array.isArray(lead.sources)) {
347
+ issues.push(issue('error', `${path}.sources`, 'invalid_sources', 'sources must be an array'));
348
+ } else {
349
+ lead.sources.forEach((source, index) => {
350
+ if (!source || typeof source !== 'object' || Array.isArray(source)) {
351
+ issues.push(issue('error', `${path}.sources[${index}]`, 'invalid_source', 'sources entries must be objects'));
352
+ return;
353
+ }
354
+ if (!isHttpUrl(source.url)) {
355
+ issues.push(issue('error', `${path}.sources[${index}].url`, 'invalid_source_url', 'sources entries require an http(s) URL'));
356
+ }
357
+ });
358
+ }
302
359
  if (lead.foundAt && Number.isNaN(Date.parse(lead.foundAt))) {
303
360
  issues.push(issue('error', `${path}.foundAt`, 'invalid_found_at', 'foundAt must be ISO datetime'));
304
361
  }
@@ -447,6 +504,127 @@ export function domainFromUrl(value) {
447
504
  }
448
505
  }
449
506
 
507
+ export function dedupeLeadRecords(leads) {
508
+ const byKey = new Map();
509
+ for (const rawLead of leads || []) {
510
+ const lead = normalizeLeadRecord(rawLead);
511
+ const key = stableLeadKey(lead);
512
+ const current = byKey.get(key);
513
+ byKey.set(key, current ? mergeLeadRecords(current, lead) : lead);
514
+ }
515
+ return [...byKey.values()];
516
+ }
517
+
518
+ function normalizeLeadRecord(lead) {
519
+ const sources = normalizeLeadSources(lead?.sources, {
520
+ sourceUrl: lead?.sourceUrl,
521
+ sourceLabel: lead?.sourceLabel,
522
+ evidence: lead?.evidence,
523
+ });
524
+ const normalized = {
525
+ ...lead,
526
+ sourceUrl: sources[0]?.url || stringValue(lead?.sourceUrl),
527
+ sourceLabel: sources[0]?.label || stringValue(lead?.sourceLabel),
528
+ sources,
529
+ evidence: sources[0]?.evidence || compactWhitespace(lead?.evidence),
530
+ warnings: normalizeWarnings(lead?.warnings),
531
+ socialUrls: normalizeUrlList(lead?.socialUrls),
532
+ contactUrls: normalizeUrlList(lead?.contactUrls),
533
+ };
534
+ normalized.id = stableLeadID(normalized);
535
+ return normalized;
536
+ }
537
+
538
+ export function mergeLeadRecords(current, candidate) {
539
+ const left = normalizeLeadRecord(current);
540
+ const right = normalizeLeadRecord(candidate);
541
+ const primary = leadQualityScore(right) > leadQualityScore(left) ? right : left;
542
+ const secondary = primary === right ? left : right;
543
+
544
+ const merged = {
545
+ ...primary,
546
+ company: primary.company || secondary.company,
547
+ domain: primary.domain || secondary.domain,
548
+ websiteUrl: primary.websiteUrl || secondary.websiteUrl,
549
+ contactName: primary.contactName || secondary.contactName,
550
+ title: primary.title || secondary.title,
551
+ email: primary.email || secondary.email,
552
+ emailType: primary.emailType || secondary.emailType,
553
+ extractionMethod: primary.extractionMethod || secondary.extractionMethod,
554
+ verificationStatus: verificationRank(secondary.verificationStatus) > verificationRank(primary.verificationStatus)
555
+ ? secondary.verificationStatus
556
+ : primary.verificationStatus,
557
+ confidence: Math.max(primary.confidence || 0, secondary.confidence || 0),
558
+ phone: primary.phone || secondary.phone,
559
+ socialUrls: normalizeUrlList([...(primary.socialUrls || []), ...(secondary.socialUrls || [])]),
560
+ contactUrls: normalizeUrlList([...(primary.contactUrls || []), ...(secondary.contactUrls || [])]),
561
+ warnings: normalizeWarnings([...(primary.warnings || []), ...(secondary.warnings || [])]),
562
+ };
563
+
564
+ merged.foundAt = earliestDateTime(primary.foundAt, secondary.foundAt);
565
+ if ((secondary.evidence || '').length > (merged.evidence || '').length) {
566
+ merged.evidence = secondary.evidence;
567
+ }
568
+
569
+ merged.sources = mergeLeadSources(merged.sourceUrl, left.sources, right.sources);
570
+ merged.sourceUrl = merged.sources[0]?.url || merged.sourceUrl;
571
+ merged.sourceLabel = merged.sources[0]?.label || merged.sourceLabel;
572
+ if (!merged.evidence) {
573
+ merged.evidence = merged.sources[0]?.evidence || '';
574
+ }
575
+ merged.id = stableLeadID(merged);
576
+ return normalizeLeadRecord(merged);
577
+ }
578
+
579
+ function normalizeLeadSources(value, fallback = {}) {
580
+ const byUrl = new Map();
581
+ const ordered = [];
582
+ const addSource = (source) => {
583
+ const url = stringValue(source?.url);
584
+ if (!url) return;
585
+
586
+ const entry = {
587
+ url,
588
+ label: stringValue(source?.label) || defaultSourceLabel(url),
589
+ evidence: compactWhitespace(source?.evidence),
590
+ };
591
+ const current = byUrl.get(url);
592
+ if (!current) {
593
+ byUrl.set(url, entry);
594
+ ordered.push(url);
595
+ return;
596
+ }
597
+ byUrl.set(url, {
598
+ url,
599
+ label: current.label || entry.label,
600
+ evidence: entry.evidence.length > current.evidence.length ? entry.evidence : current.evidence,
601
+ });
602
+ };
603
+
604
+ if (Array.isArray(value)) {
605
+ value.forEach(addSource);
606
+ }
607
+ addSource({
608
+ url: fallback.sourceUrl,
609
+ label: fallback.sourceLabel,
610
+ evidence: fallback.evidence,
611
+ });
612
+
613
+ const preferredUrl = stringValue(fallback.sourceUrl);
614
+ return ordered
615
+ .map((url) => byUrl.get(url))
616
+ .sort((left, right) => sourceSortValue(right, preferredUrl) - sourceSortValue(left, preferredUrl) || left.label.localeCompare(right.label) || left.url.localeCompare(right.url));
617
+ }
618
+
619
+ function mergeLeadSources(preferredUrl, ...lists) {
620
+ return normalizeLeadSources(lists.flat(), { sourceUrl: preferredUrl });
621
+ }
622
+
623
+ function sourceSortValue(source, preferredUrl) {
624
+ if (preferredUrl && source.url === preferredUrl) return 10_000;
625
+ return pagePriority(source.url || '') * 100 + Math.min((source.evidence || '').length, 99);
626
+ }
627
+
450
628
  export function stableLeadID(lead) {
451
629
  const key = stableLeadKey(lead);
452
630
  if (!key.replace(/\|/g, '').trim()) return `agentic-${Date.now()}`;
@@ -469,10 +647,105 @@ function stableLeadKey(lead) {
469
647
  ].join('|').toLowerCase();
470
648
  }
471
649
 
650
+ function leadQualityScore(lead) {
651
+ return (lead?.confidence || 0) * 1000
652
+ + verificationRank(lead?.verificationStatus) * 100
653
+ + pagePriority(lead?.sourceUrl || '') * 10
654
+ + (Array.isArray(lead?.sources) ? lead.sources.length : 0) * 5
655
+ + (lead?.phone ? 25 : 0)
656
+ + (Array.isArray(lead?.socialUrls) ? lead.socialUrls.length : 0) * 5
657
+ + (Array.isArray(lead?.contactUrls) ? lead.contactUrls.length : 0) * 5
658
+ + (lead?.contactName ? 20 : 0)
659
+ + (lead?.title ? 10 : 0);
660
+ }
661
+
662
+ function verificationRank(value) {
663
+ const status = stringValue(value).toLowerCase();
664
+ if (status === 'verified' || status === 'mx_verified') return 4;
665
+ if (status === 'unverified') return 3;
666
+ if (status === 'not_applicable') return 2;
667
+ if (status === 'unknown') return 1;
668
+ return 0;
669
+ }
670
+
671
+ function earliestDateTime(...values) {
672
+ const dates = values
673
+ .map((value) => stringValue(value))
674
+ .filter(Boolean)
675
+ .map((value) => new Date(value))
676
+ .filter((value) => !Number.isNaN(value.getTime()))
677
+ .sort((left, right) => left.getTime() - right.getTime());
678
+ return dates[0]?.toISOString() || new Date().toISOString();
679
+ }
680
+
681
+ function defaultSourceLabel(sourceUrl) {
682
+ const path = safePath(sourceUrl).replace(/^\/+|\/+$/g, '');
683
+ if (!path) return 'Home';
684
+ const last = path.split('/').filter(Boolean).pop() || path;
685
+ return last.replace(/[-_]+/g, ' ').replace(/\b\w/g, (char) => char.toUpperCase());
686
+ }
687
+
688
+ function safePath(value) {
689
+ try {
690
+ return new URL(value).pathname.toLowerCase();
691
+ } catch {
692
+ return '';
693
+ }
694
+ }
695
+
696
+ function pagePriority(link) {
697
+ const path = safePath(link);
698
+ const weighted = [
699
+ ['contact', 100],
700
+ ['about', 90],
701
+ ['team', 85],
702
+ ['people', 85],
703
+ ['leadership', 85],
704
+ ['staff', 80],
705
+ ['founder', 80],
706
+ ['press', 65],
707
+ ['media', 65],
708
+ ['blog', 55],
709
+ ['author', 55],
710
+ ['career', 45],
711
+ ['privacy', 25],
712
+ ['legal', 20],
713
+ ['impressum', 20],
714
+ ];
715
+ for (const [needle, score] of weighted) {
716
+ if (path.includes(needle)) return score;
717
+ }
718
+ return path === '/' ? 50 : 0;
719
+ }
720
+
472
721
  export function fileHash(path) {
473
722
  return createHash('sha256').update(readFileSync(path)).digest('hex');
474
723
  }
475
724
 
725
+ export function isRejectedLeadEmail(email) {
726
+ const local = String(email || '').trim().toLowerCase().split('@')[0] || '';
727
+ return isRejectedLeadLocalPart(local);
728
+ }
729
+
730
+ export function isRejectedLeadLocalPart(local) {
731
+ const value = normalizeRejectedLeadLocalPart(local);
732
+ if (!value) return false;
733
+ return REJECTED_EMAIL_LOCAL_PREFIXES.some((prefix) => {
734
+ if (value === prefix) return true;
735
+ if (!value.startsWith(prefix) || value.length === prefix.length) return false;
736
+ const next = value[prefix.length];
737
+ return next === '.' || next === '_' || next === '-' || next === '+' || /\d/.test(next);
738
+ });
739
+ }
740
+
741
+ function normalizeRejectedLeadLocalPart(local) {
742
+ let value = String(local || '').trim().toLowerCase();
743
+ for (const noise of ['mailto', 'u003e', 'email']) {
744
+ if (value.startsWith(noise)) value = value.slice(noise.length);
745
+ }
746
+ return value.replace(/^[>._+-]+/, '');
747
+ }
748
+
476
749
  export function issue(severity, path, code, message) {
477
750
  return { severity, path, code, message };
478
751
  }
@@ -507,7 +780,7 @@ function normalizeVerificationStatus(value, emailType) {
507
780
 
508
781
  function normalizeWarnings(value) {
509
782
  if (!Array.isArray(value)) return [];
510
- return value.map((item) => stringValue(item)).filter(Boolean);
783
+ return [...new Set(value.map((item) => stringValue(item)).filter(Boolean))];
511
784
  }
512
785
 
513
786
  function normalizeUrlList(value) {
package/modes/_shared.md CHANGED
@@ -28,13 +28,14 @@ Required fields:
28
28
  - Prioritize home, contact, about, team, leadership, people, press, blog, careers, and legal pages.
29
29
  - Respect robots.txt, paywalls, login walls, and obvious anti-scraping notices.
30
30
  - Do not infer email formats. Do not brute-force guessed addresses.
31
+ - Prefer named people over organizational inboxes. Do not emit generic catch-all emails such as `info@`, `hello@`, `contact@`, `support@`, `team@`, or similar aliases. If only a general contact path exists, emit `contact_path` instead.
31
32
  - Keep contact forms as `emailType: "contact_path"` with empty `email`.
32
33
  - Mark operational no-contact inboxes such as `noreply`, `abuse`, `security`, and `legal` as `blocked` with confidence `0`.
33
34
 
34
35
  ## Quality Heuristics
35
36
 
36
37
  - `person`: named person or person-like email with source context.
37
- - `role`: public role inbox such as `sales@`, `partnerships@`, or `press@`.
38
+ - `role`: public role inbox only when it is clearly role-specific and not a generic catch-all alias.
38
39
  - `contact_path`: public form or page where no email is available.
39
40
  - `blocked`: public address that should not be used for outbound.
40
41
 
@@ -45,6 +46,7 @@ Confidence should reflect source quality, person specificity, domain match, and
45
46
  ```bash
46
47
  npx public-leads validate --input data/lead-results.json
47
48
  npx public-leads manifest --input data/lead-results.json
48
- npx public-leads ingest --input data/lead-results.json --dry-run
49
+ PUBLIC_LEADS_API=https://cold-agent-leads.example.com npx public-leads ingest --input data/lead-results.json --dry-run
50
+ npx public-leads pipeline --input data/domains.tsv --ingest
49
51
  npx public-leads verify
50
52
  ```
package/modes/batch.md CHANGED
@@ -10,7 +10,7 @@ Prefer:
10
10
  batch/batch-runner.sh --parallel 2
11
11
  ```
12
12
 
13
- The runner delegates to `scripts/batch-orchestrator.mjs`, which uses `@razroo/iso-orchestrator` for durable workflow records in `.public-leads-runs/`, bounded fan-out, and state updates.
13
+ The runner delegates to `scripts/batch-orchestrator.mjs`, which uses `@agent-pattern-labs/iso-orchestrator` for durable workflow records in `.public-leads-runs/`, bounded fan-out, and state updates.
14
14
 
15
15
  ## Input
16
16
 
package/modes/ingest.md CHANGED
@@ -12,20 +12,23 @@ Use this when the user asks to store validated leads in a configured ingest API.
12
12
  ```bash
13
13
  npx public-leads manifest --input <artifact>
14
14
  ```
15
- 3. Confirm ingest credentials and request settings are available through `config/profile.yml` and the configured environment variable. Do not print token values.
15
+ 3. Confirm upstream ingest settings are available through environment variables, flags, or `config/profile.yml`. Do not print token values.
16
+ - `PUBLIC_LEADS_API`
17
+ - `PUBLIC_LEADS_API_TOKEN`
18
+ - `PUBLIC_LEADS_OPERATOR_EMAIL`
16
19
 
17
20
  ## Dry Run
18
21
 
19
22
  Use dry-run before live ingest unless the user explicitly requested live submission:
20
23
 
21
24
  ```bash
22
- npx public-leads ingest --input <artifact> --dry-run --out data/ingest-response.json
25
+ PUBLIC_LEADS_API=https://cold-agent-leads.example.com npx public-leads ingest --input <artifact> --dry-run --out data/ingest-response.json
23
26
  ```
24
27
 
25
28
  ## Live Ingest
26
29
 
27
30
  ```bash
28
- npx public-leads ingest --input <artifact> --target-project /path/to/cold-agent-leads --out data/ingest-response.json
31
+ npx public-leads ingest --input <artifact> --out data/ingest-response.json
29
32
  ```
30
33
 
31
34
  ## Output
package/modes/pipeline.md CHANGED
@@ -20,7 +20,7 @@ Read domains from the first available source:
20
20
  5. For browser-heavy batch work, use `batch/batch-runner.sh --parallel 2`.
21
21
  6. Validate every artifact with `npx public-leads validate`.
22
22
  7. Merge validated artifacts into a manifest with `npx public-leads manifest --input <artifact>`.
23
- 8. Ingest with `npx public-leads pipeline --ingest --target-project /path/to/cold-agent-leads` only when upload is requested.
23
+ 8. Ingest with `npx public-leads pipeline --ingest` only when upload is requested and upstream API env vars are configured.
24
24
 
25
25
  ## Output
26
26
 
package/modes/setup.md CHANGED
@@ -9,7 +9,7 @@ Use this when the user asks to configure or inspect the lead harness.
9
9
  - `data/domains.tsv`
10
10
  - `data/pipeline.md`
11
11
  - direct domains in the user request
12
- 3. Confirm the ingest API settings:
12
+ 3. Confirm the upstream ingest API settings:
13
13
  - `api.base_url`
14
14
  - `api.ingest_path`
15
15
  - `api.operator_email`
@@ -17,7 +17,8 @@ Use this when the user asks to configure or inspect the lead harness.
17
17
  - `api.auth_header`
18
18
  - `api.auth_scheme`
19
19
  - `api.auth_token_env`
20
- - `api.target_project` when using a local Cold Agent Leads checkout for `.env` credentials
20
+ - or `PUBLIC_LEADS_API`, `PUBLIC_LEADS_API_TOKEN`, and `PUBLIC_LEADS_OPERATOR_EMAIL`
21
+ - `api.target_project` only when intentionally reading credentials from a local Cold Agent Leads checkout
21
22
  4. Run `npx public-leads verify`.
22
23
 
23
24
  ## Output
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@agent-pattern-labs/leads-rig",
3
- "version": "0.1.3",
3
+ "version": "0.1.5",
4
4
  "description": "Agentic public-web lead discovery harness with portable ingest artifacts",
5
5
  "type": "module",
6
6
  "author": "Razroo",
@@ -155,32 +155,32 @@
155
155
  "fast-uri": "3.1.2"
156
156
  },
157
157
  "dependencies": {
158
- "@razroo/iso-cache": "^0.1.0",
159
- "@razroo/iso-canon": "^0.1.0",
160
- "@razroo/iso-capabilities": "^0.1.0",
161
- "@razroo/iso-context": "^0.1.0",
162
- "@razroo/iso-contract": "^0.1.0",
163
- "@razroo/iso-facts": "^0.1.0",
164
- "@razroo/iso-guard": "^0.1.0",
165
- "@razroo/iso-index": "^0.1.0",
166
- "@razroo/iso-ledger": "^0.1.0",
167
- "@razroo/iso-lineage": "^0.1.0",
168
- "@razroo/iso-migrate": "^0.1.0",
169
- "@razroo/iso-orchestrator": "^0.2.0",
170
- "@razroo/iso-postflight": "^0.1.0",
171
- "@razroo/iso-preflight": "^0.1.0",
172
- "@razroo/iso-prioritize": "^0.1.0",
173
- "@razroo/iso-redact": "^0.1.0",
174
- "@razroo/iso-score": "^0.1.0",
175
- "@razroo/iso-timeline": "^0.1.0",
176
- "@razroo/iso-trace": "^0.5.0",
158
+ "@agent-pattern-labs/iso-cache": "^0.1.1",
159
+ "@agent-pattern-labs/iso-canon": "^0.1.1",
160
+ "@agent-pattern-labs/iso-capabilities": "^0.1.1",
161
+ "@agent-pattern-labs/iso-context": "^0.1.1",
162
+ "@agent-pattern-labs/iso-contract": "^0.1.1",
163
+ "@agent-pattern-labs/iso-facts": "^0.1.1",
164
+ "@agent-pattern-labs/iso-guard": "^0.1.1",
165
+ "@agent-pattern-labs/iso-index": "^0.1.1",
166
+ "@agent-pattern-labs/iso-ledger": "^0.1.1",
167
+ "@agent-pattern-labs/iso-lineage": "^0.1.1",
168
+ "@agent-pattern-labs/iso-migrate": "^0.1.1",
169
+ "@agent-pattern-labs/iso-orchestrator": "^0.2.1",
170
+ "@agent-pattern-labs/iso-postflight": "^0.1.1",
171
+ "@agent-pattern-labs/iso-preflight": "^0.1.1",
172
+ "@agent-pattern-labs/iso-prioritize": "^0.1.1",
173
+ "@agent-pattern-labs/iso-redact": "^0.1.1",
174
+ "@agent-pattern-labs/iso-score": "^0.1.1",
175
+ "@agent-pattern-labs/iso-timeline": "^0.1.1",
176
+ "@agent-pattern-labs/iso-trace": "^0.5.1",
177
177
  "playwright": "^1.58.1"
178
178
  },
179
179
  "devDependencies": {
180
- "@razroo/agentmd": "^0.3.0",
181
- "@razroo/iso": "^0.3.1",
182
- "@razroo/iso-eval": "^0.4.0",
183
- "@razroo/iso-harness": "^0.8.0",
184
- "@razroo/iso-route": "^0.5.3"
180
+ "@agent-pattern-labs/agentmd": "^0.3.1",
181
+ "@agent-pattern-labs/iso": "^0.3.2",
182
+ "@agent-pattern-labs/iso-eval": "^0.4.1",
183
+ "@agent-pattern-labs/iso-harness": "^0.8.1",
184
+ "@agent-pattern-labs/iso-route": "^0.6.1"
185
185
  }
186
186
  }