@agent-pattern-labs/leads-rig 0.1.5 → 0.1.7

package/lib/leadharness-leads.mjs

@@ -296,6 +296,9 @@ export function validateLead(lead, path, issues) {
   if (lead.email && isRejectedLeadEmail(lead.email)) {
     issues.push(issue('error', `${path}.email`, 'generic_inbox_rejected', 'generic company inboxes such as info@, hello@, or support@ are not accepted; submit named people instead'));
   }
+  if (lead.email && !isRejectedLeadEmail(lead.email) && ALLOWED_EMAIL_TYPES.has(lead.emailType) && !isGoodLead(lead)) {
+    issues.push(issue('error', `${path}.emailType`, 'non_person_email_rejected', 'email records must identify one named human visible in the evidence; role, unknown, blocked, unsupported, or unnamed person emails are not accepted'));
+  }
   if (lead.emailType && !ALLOWED_EMAIL_TYPES.has(lead.emailType)) {
     issues.push(issue('error', `${path}.emailType`, 'invalid_email_type', `emailType must be one of ${[...ALLOWED_EMAIL_TYPES].join(', ')}`));
   }
@@ -381,11 +384,13 @@ export function summarizePayload(payload) {
   const byType = {};
   let confidenceTotal = 0;
   let withEmail = 0;
+  let goodLeadCount = 0;
   const uniqueEmails = new Set();
   const mxVerifiedEmails = new Set();
   for (const lead of leads) {
     byType[lead.emailType || 'unknown'] = (byType[lead.emailType || 'unknown'] || 0) + 1;
     confidenceTotal += Number.isFinite(lead.confidence) ? lead.confidence : 0;
+    if (isGoodLead(lead)) goodLeadCount++;
     if (lead.email) {
       withEmail++;
       uniqueEmails.add(String(lead.email).toLowerCase());
@@ -399,6 +404,8 @@ export function summarizePayload(payload) {
     domainCount: domains.length,
     leadCount: leads.length,
     resultCount: results.length,
+    goodLeadCount,
+    reviewOnlyLeadCount: Math.max(0, leads.length - goodLeadCount),
     withEmail,
     uniqueEmailCount: uniqueEmails.size,
     mxVerifiedEmailCount: mxVerifiedEmails.size,
@@ -417,6 +424,7 @@ export function createManifestRecord({ inputPath, payload, validation, generated
     generatedAt,
     domains: payload.domains,
     leadCount: payload.leads.length,
+    goodLeadCount: validation.summary?.goodLeadCount || 0,
     resultCount: payload.results.length,
     errorCount: payload.errors.length,
     validation: {
@@ -463,7 +471,18 @@ export function loadProfileConfig(root = PROJECT_DIR) {
     adminTokenEnv: yamlScalar(text, 'admin_token_env'),
     maxPages: yamlScalar(text, 'max_pages'),
     maxDomainsPerBatch: yamlScalar(text, 'max_domains_per_batch'),
+    timeoutMs: yamlScalar(text, 'timeout_ms'),
     concurrency: yamlScalar(text, 'concurrency'),
+    pageConcurrency: yamlScalar(text, 'page_concurrency'),
+    dnsConcurrency: yamlScalar(text, 'dns_concurrency'),
+    cachePath: yamlScalar(text, 'cache_path'),
+    robotsCacheTtlMs: yamlScalar(text, 'robots_cache_ttl_ms'),
+    dnsCacheTtlMs: yamlScalar(text, 'dns_cache_ttl_ms'),
+    pageCache: yamlScalar(text, 'page_cache'),
+    pageCacheTtlMs: yamlScalar(text, 'page_cache_ttl_ms'),
+    stopAfterGoodLeads: yamlScalar(text, 'stop_after_good_leads'),
+    stopAfterContactPath: yamlScalar(text, 'stop_after_contact_path'),
+    batchTimeoutMs: yamlScalar(text, 'batch_timeout_ms'),
     userAgent: yamlScalar(text, 'user_agent'),
     minConfidence: yamlScalar(text, 'min_confidence'),
   };
@@ -648,7 +667,10 @@ function stableLeadKey(lead) {
 }
 
 function leadQualityScore(lead) {
-  return (lead?.confidence || 0) * 1000
+  const goodLead = isGoodLead(lead);
+  return (goodLead ? 50_000 : 0)
+    + (lead?.email && !goodLead ? -20_000 : 0)
+    + (lead?.confidence || 0) * 1000
     + verificationRank(lead?.verificationStatus) * 100
     + pagePriority(lead?.sourceUrl || '') * 10
     + (Array.isArray(lead?.sources) ? lead.sources.length : 0) * 5
@@ -738,6 +760,30 @@ export function isRejectedLeadLocalPart(local) {
   });
 }
 
+export function isGoodLead(lead) {
+  if (!lead || typeof lead !== 'object') return false;
+  if (stringValue(lead.emailType).toLowerCase() !== 'person') return false;
+  if (!lead.email || isRejectedLeadEmail(lead.email)) return false;
+  const contactName = stringValue(lead.contactName);
+  if (!contactName) return false;
+  if (!evidenceIdentifiesContact(lead, contactName)) return false;
+  return true;
+}
+
+function evidenceIdentifiesContact(lead, contactName) {
+  const needle = compactComparable(contactName);
+  if (!needle) return false;
+  const evidence = [
+    lead.evidence,
+    ...(Array.isArray(lead.sources) ? lead.sources.map((source) => source?.evidence) : []),
+  ].map(compactComparable).join(' ');
+  return evidence.includes(needle);
+}
+
+function compactComparable(value) {
+  return stringValue(value).toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim();
+}
+
 function normalizeRejectedLeadLocalPart(local) {
   let value = String(local || '').trim().toLowerCase();
   for (const noise of ['mailto', 'u003e', 'email']) {

package/modes/_shared.md

@@ -28,18 +28,21 @@ Required fields:
 - Prioritize home, contact, about, team, leadership, people, press, blog, careers, and legal pages.
 - Respect robots.txt, paywalls, login walls, and obvious anti-scraping notices.
 - Do not infer email formats. Do not brute-force guessed addresses.
+- Use coding-agent judgment for every email candidate. Only emit an email lead when the source evidence identifies one specific human behind that address.
 - Prefer named people over organizational inboxes. Do not emit generic catch-all emails such as `info@`, `hello@`, `contact@`, `support@`, `team@`, or similar aliases. If only a general contact path exists, emit `contact_path` instead.
+- Reject email records that are role-based, shared, departmental, unknown ownership, blocked operational inboxes, or person-like but missing a visible human name in the source evidence.
 - Keep contact forms as `emailType: "contact_path"` with empty `email`.
 - Mark operational no-contact inboxes such as `noreply`, `abuse`, `security`, and `legal` as `blocked` with confidence `0`.
 
 ## Quality Heuristics
 
-- `person`: named person or person-like email with source context.
-- `role`: public role inbox only when it is clearly role-specific and not a generic catch-all alias.
+- `goodLeadCount`: counts only `person` email records with a non-generic email and named `contactName` visible in the evidence. Generic, role, unknown, blocked, unnamed, unsupported, and `contact_path` records are not good leads.
+- `person`: named human with source context tying that person to the email address.
+- `role`: legacy/review-only classification. Do not emit role email records in new artifacts.
 - `contact_path`: public form or page where no email is available.
 - `blocked`: public address that should not be used for outbound.
 
-Confidence should reflect source quality, person specificity, domain match, and verification. Low-confidence records are allowed if they preserve warnings and evidence.
+Confidence should reflect source quality, person specificity, domain match, and verification. Low-confidence contact paths are allowed if they preserve warnings and evidence; email leads should be high-quality named humans only.
 
 ## Local Commands
 

package/modes/batch.md

@@ -25,9 +25,10 @@ id	domain	company	notes
 
 1. Run `batch/batch-runner.sh --dry-run` first unless the user explicitly asks to start immediately.
 2. Use `--parallel 2` by default. Do not exceed 2 browser-heavy workers per round.
-3. Each worker writes `batch/lead-results-{id}.json`.
-4. Validate every artifact and update `data/lead-manifest.json`.
-5. Run `npx public-leads verify`.
+3. For broad queues, keep the batch deterministic crawler timeout near the default `--timeout-ms 8000` unless target quality is known to be high.
+4. Each worker writes `batch/lead-results-{id}.json`.
+5. Validate every artifact and update `data/lead-manifest.json`.
+6. Run `npx public-leads verify`.
 
 ## Output
 

package/modes/crawl.md

@@ -7,7 +7,7 @@ Use this for one or more assigned domains.
 1. Normalize the domain to lowercase without protocol or `www.`.
 2. Visit only bounded public pages from the official site.
 3. Prefer the deterministic crawler when no browser-only blocker exists:
-   `npx public-leads crawl --input data/domains.tsv --out data/lead-results.json`
+   `npx public-leads crawl --input data/domains.tsv --out data/lead-results.json --concurrency 2 --page-concurrency 1`
 4. Extract public email addresses, person names/titles when visible nearby, contact forms, source URLs, page titles, evidence snippets, and warnings.
 5. Emit a JSON payload matching `templates/lead-schema.json`.
 6. Save the payload to `data/lead-results.json` for single-domain runs or `batch/lead-results-{id}.json` for batch workers.

package/modes/pipeline.md

@@ -15,7 +15,7 @@ Read domains from the first available source:
 1. Build a deduped candidate list.
 2. Drop domains already represented in the manifest unless the user asks to retry.
 3. Prefer the deterministic end-to-end command for normal public pages:
-   `npx public-leads pipeline --input data/domains.tsv --out data/lead-results.json`
+   `npx public-leads pipeline --input data/domains.tsv --out data/lead-results.json --concurrency 2 --page-concurrency 1`
 4. Use browser/MCP workers only when the deterministic crawler cannot reach a public page or the site requires rendering.
 5. For browser-heavy batch work, use `batch/batch-runner.sh --parallel 2`.
 6. Validate every artifact with `npx public-leads validate`.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@agent-pattern-labs/leads-rig",
-  "version": "0.1.5",
+  "version": "0.1.7",
   "description": "Agentic public-web lead discovery harness with portable ingest artifacts",
   "type": "module",
   "author": "Razroo",

package/scripts/batch-orchestrator.mjs

@@ -7,6 +7,7 @@ import { mkdir, readFile, rm, writeFile } from 'fs/promises';
 import { dirname, join, resolve } from 'path';
 import { fileURLToPath } from 'url';
 import { runWorkflow } from '@agent-pattern-labs/iso-orchestrator';
+import { loadProfileConfig } from '../lib/leadharness-leads.mjs';
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const PKG_ROOT = resolve(__dirname, '..');
@@ -37,6 +38,7 @@ Options:
   --retry-failed       Only retry rows marked failed
   --start-from N       Start from numeric id N
   --max-retries N      Max failed retries per domain (default: 2)
+  --timeout-ms N       Deterministic crawler page timeout in worker prompt (default: 8000)
   --workflow-id ID     Durable workflow id (default: public-leads-batch)
   -h, --help           Show help
 
@@ -113,6 +115,7 @@ async function main(opts) {
 }
 
 function parseArgs(argv) {
+  const profile = loadProfileConfig();
   const opts = {
     runner: process.env.PUBLIC_LEADS_BATCH_RUNNER || process.env.LEAD_HARNESS_BATCH_RUNNER || 'opencode',
     parallel: 1,
@@ -121,6 +124,7 @@ function parseArgs(argv) {
     retryFailed: false,
     startFrom: 0,
     maxRetries: 2,
+    timeoutMs: positiveInt(process.env.PUBLIC_LEADS_BATCH_TIMEOUT_MS || process.env.LEAD_HARNESS_BATCH_TIMEOUT_MS || profile.batchTimeoutMs || 8_000, 'batch timeout'),
     workflowId: 'public-leads-batch',
   };
 
@@ -138,6 +142,7 @@ function parseArgs(argv) {
     else if (arg === '--retry-failed') opts.retryFailed = true;
     else if (arg === '--start-from') opts.startFrom = nonNegativeInt(next(), '--start-from');
     else if (arg === '--max-retries') opts.maxRetries = positiveInt(next(), '--max-retries');
+    else if (arg === '--timeout-ms') opts.timeoutMs = positiveInt(next(), '--timeout-ms');
     else if (arg === '--workflow-id') opts.workflowId = sanitizeWorkflowId(next());
     else if (arg === '-h' || arg === '--help') opts.help = true;
     else throw new Error(`unknown option: ${arg}`);
@@ -299,7 +304,7 @@ async function processDomain(workflow, item, opts) {
     retries,
   });
 
-  const prompt = buildWorkerPrompt(item, artifact);
+  const prompt = buildWorkerPrompt(item, artifact, opts);
   const run = await withWorkerLiveness(workflow, item, logFile, () => runWorker(opts, prompt, logFile));
   const statuses = parseStatusLines(run.output);
   const status = statuses.get(item.id);
@@ -356,12 +361,15 @@ async function processDomain(workflow, item, opts) {
   return { id: item.id, status: 'completed', artifact: status.artifact || artifact, leadCount: status.leadCount ?? 0 };
 }
 
-function buildWorkerPrompt(item, artifact) {
+function buildWorkerPrompt(item, artifact, opts) {
   return `Process this assigned public-leads domain.
 
 Assignment:
 ${JSON.stringify({ ...item, artifact }, null, 2)}
 
+Prefer the deterministic crawler first:
+npx public-leads crawl --domain ${item.domain} --out ${artifact} --timeout-ms ${opts.timeoutMs} --stop-after-good-leads 1 --stop-after-contact-path
+
 Write the artifact exactly to ${artifact}. Validate it with:
 npx public-leads validate --input ${artifact}
 

package/scripts/crawl.mjs

@@ -17,7 +17,10 @@ const USAGE = `public-leads crawl -- crawl public company pages into a lead arti
 Usage:
   public-leads crawl [--domain example.com | --domains example.com,example.org]
                      [--input data/domains.tsv] [--out data/lead-results.json]
-                     [--max-pages 10] [--min-confidence 30]
+                     [--max-pages 10] [--timeout-ms 14000] [--min-confidence 30]
+                     [--concurrency 2] [--page-concurrency 1] [--dns-concurrency 8]
+                     [--stop-after-good-leads 1] [--stop-after-contact-path]
+                     [--cache-path .leadharness-cache/crawler-cache.json] [--page-cache] [--page-cache-ttl-ms 3600000] [--no-cache]
                      [--include-blocked] [--allow-empty] [--json]
 
 When no domain or input is supplied, the first existing file is used:
@@ -44,10 +47,21 @@ async function main() {
   const payload = await crawlDomains(domains, {
     maxPages: opts.maxPages || profile.maxPages,
     minConfidence: opts.minConfidence || profile.minConfidence,
+    timeoutMs: opts.timeoutMs || profile.timeoutMs,
+    concurrency: opts.concurrency || profile.concurrency,
+    pageConcurrency: opts.pageConcurrency || profile.pageConcurrency,
+    dnsConcurrency: opts.dnsConcurrency || profile.dnsConcurrency,
     userAgent: opts.userAgent || profile.userAgent,
     includeBlocked: Boolean(opts.includeBlocked),
     delayMs: opts.delayMs,
-    timeoutMs: opts.timeoutMs,
+    cachePath: opts.cachePath || profile.cachePath,
+    noCache: Boolean(opts.noCache),
+    robotsCacheTtlMs: opts.robotsCacheTtlMs || profile.robotsCacheTtlMs,
+    dnsCacheTtlMs: opts.dnsCacheTtlMs || profile.dnsCacheTtlMs,
+    pageCache: opts.pageCache || profile.pageCache,
+    pageCacheTtlMs: opts.pageCacheTtlMs || profile.pageCacheTtlMs,
+    stopAfterGoodLeads: opts.stopAfterGoodLeads || profile.stopAfterGoodLeads,
+    stopAfterContactPath: opts.stopAfterContactPath || profile.stopAfterContactPath,
     jobId: opts.jobId,
   });
   writeJson(resolveProjectPath(out), payload);
@@ -57,7 +71,7 @@ async function main() {
     console.log(JSON.stringify({ output: out, validation, summary: validation.summary }, null, 2));
   } else {
     console.log(`crawl: wrote ${relativeProjectPath(resolveProjectPath(out))}`);
-    console.log(`domains=${validation.summary.domainCount} leads=${validation.summary.leadCount} results=${validation.summary.resultCount} errors=${payload.errors.length}`);
+    console.log(`domains=${validation.summary.domainCount} leads=${validation.summary.leadCount} goodLeads=${validation.summary.goodLeadCount} results=${validation.summary.resultCount} errors=${payload.errors.length}`);
   }
 
   if (!validation.ok) {

package/scripts/manifest.mjs

@@ -50,6 +50,7 @@ try {
     batchCount: manifest.batches.length,
     readyForIngest: manifest.batches.filter((item) => item.readyForIngest).length,
     leadCount: manifest.batches.reduce((sum, item) => sum + item.leadCount, 0),
+    goodLeadCount: manifest.batches.reduce((sum, item) => sum + (item.goodLeadCount || 0), 0),
     errorCount: manifest.batches.reduce((sum, item) => sum + item.validation.errors, 0),
     warningCount: manifest.batches.reduce((sum, item) => sum + item.validation.warnings, 0),
   };
@@ -60,8 +61,8 @@ try {
     console.log(JSON.stringify({ manifest: relativeProjectPath(manifestPath), record, summary: manifest.summary }, null, 2));
   } else {
     console.log(`manifest: ${relativeProjectPath(manifestPath)}`);
-    console.log(`batch: ${record.id} leads=${record.leadCount} ready=${record.readyForIngest}`);
-    console.log(`summary: batches=${manifest.summary.batchCount} ready=${manifest.summary.readyForIngest} leads=${manifest.summary.leadCount}`);
+    console.log(`batch: ${record.id} leads=${record.leadCount} goodLeads=${record.goodLeadCount} ready=${record.readyForIngest}`);
+    console.log(`summary: batches=${manifest.summary.batchCount} ready=${manifest.summary.readyForIngest} leads=${manifest.summary.leadCount} goodLeads=${manifest.summary.goodLeadCount}`);
   }
 
   process.exit(validation.ok ? 0 : 1);

package/scripts/pipeline.mjs

@@ -21,7 +21,10 @@ Usage:
   public-leads pipeline [--domain example.com | --domains example.com,example.org]
                         [--input data/domains.tsv] [--out data/lead-results.json]
                         [--manifest data/lead-manifest.json]
-                        [--max-pages 10] [--min-confidence 30]
+                        [--max-pages 10] [--timeout-ms 14000] [--min-confidence 30]
+                        [--concurrency 2] [--page-concurrency 1] [--dns-concurrency 8]
+                        [--stop-after-good-leads 1] [--stop-after-contact-path]
+                        [--cache-path .leadharness-cache/crawler-cache.json] [--page-cache] [--page-cache-ttl-ms 3600000] [--no-cache]
                         [--ingest | --upload] [--dry-run]
                         [--api https://cold-agent-leads.example.com]
 
@@ -47,10 +50,21 @@ try {
   const payload = await crawlDomains(domains, {
     maxPages: opts.maxPages || profile.maxPages,
     minConfidence: opts.minConfidence || profile.minConfidence,
+    timeoutMs: opts.timeoutMs || profile.timeoutMs,
+    concurrency: opts.concurrency || profile.concurrency,
+    pageConcurrency: opts.pageConcurrency || profile.pageConcurrency,
+    dnsConcurrency: opts.dnsConcurrency || profile.dnsConcurrency,
     userAgent: opts.userAgent || profile.userAgent,
     includeBlocked: Boolean(opts.includeBlocked),
     delayMs: opts.delayMs,
-    timeoutMs: opts.timeoutMs,
+    cachePath: opts.cachePath || profile.cachePath,
+    noCache: Boolean(opts.noCache),
+    robotsCacheTtlMs: opts.robotsCacheTtlMs || profile.robotsCacheTtlMs,
+    dnsCacheTtlMs: opts.dnsCacheTtlMs || profile.dnsCacheTtlMs,
+    pageCache: opts.pageCache || profile.pageCache,
+    pageCacheTtlMs: opts.pageCacheTtlMs || profile.pageCacheTtlMs,
+    stopAfterGoodLeads: opts.stopAfterGoodLeads || profile.stopAfterGoodLeads,
+    stopAfterContactPath: opts.stopAfterContactPath || profile.stopAfterContactPath,
     jobId: opts.jobId,
   });
   writeJson(resolveProjectPath(out), payload);
@@ -66,7 +80,7 @@ try {
   const manifest = upsertManifest(manifestPath, out, payload, validation);
   console.log(`pipeline: wrote ${relativeProjectPath(resolveProjectPath(out))}`);
   console.log(`manifest: ${relativeProjectPath(resolveProjectPath(manifestPath))}`);
-  console.log(`domains=${validation.summary.domainCount} leads=${validation.summary.leadCount} results=${validation.summary.resultCount} errors=${payload.errors.length}`);
+  console.log(`domains=${validation.summary.domainCount} leads=${validation.summary.leadCount} goodLeads=${validation.summary.goodLeadCount} results=${validation.summary.resultCount} errors=${payload.errors.length}`);
 
   if (opts.ingest || opts.upload) {
     const ingestResult = await ingestPayload(payload, {
@@ -110,6 +124,7 @@ function upsertManifest(manifestPath, inputPath, payload, validation) {
     batchCount: manifest.batches.length,
     readyForIngest: manifest.batches.filter((item) => item.readyForIngest).length,
     leadCount: manifest.batches.reduce((sum, item) => sum + item.leadCount, 0),
+    goodLeadCount: manifest.batches.reduce((sum, item) => sum + (item.goodLeadCount || 0), 0),
     errorCount: manifest.batches.reduce((sum, item) => sum + item.validation.errors, 0),
     warningCount: manifest.batches.reduce((sum, item) => sum + item.validation.warnings, 0),
   };

package/scripts/validate-leads.mjs

@@ -58,7 +58,7 @@ try {
 function printReport(report) {
   const rel = relativeProjectPath(report.input);
   console.log(`${report.ok ? 'OK' : 'ERROR'} ${rel}`);
-  console.log(`domains=${report.summary.domainCount} leads=${report.summary.leadCount} results=${report.summary.resultCount} avgConfidence=${report.summary.averageConfidence}`);
+  console.log(`domains=${report.summary.domainCount} leads=${report.summary.leadCount} goodLeads=${report.summary.goodLeadCount} results=${report.summary.resultCount} avgConfidence=${report.summary.averageConfidence}`);
   if (Object.keys(report.summary.byType).length > 0) {
     console.log(`types=${Object.entries(report.summary.byType).map(([type, count]) => `${type}:${count}`).join(', ')}`);
   }