@agent-pattern-labs/leads-rig 0.1.3

package/scripts/ingest.mjs

@@ -0,0 +1,48 @@
+#!/usr/bin/env node
+
+import { ingestArtifact } from '../lib/leadharness-ingest.mjs';
+import { parseArgs, relativeProjectPath } from '../lib/leadharness-leads.mjs';
+
+const USAGE = `public-leads ingest -- submit validated leads to an ingest API
+
+Usage:
+  public-leads ingest --input <file> [--api <base-url>] [--ingest-path </path>]
+                      [--operator-email <email>] [--operator-email-header <header>]
+                      [--auth-header <header>] [--auth-scheme <scheme>]
+                      [--token <token>] [--token-env ADMIN_API_TOKEN]
+                      [--target-project /path/to/cold-agent-leads]
+                      [--job-id <id>] [--out data/ingest-response.json] [--dry-run]
+
+Defaults are read from config/profile.yml when present:
+  api.base_url, api.ingest_path, api.operator_email, api.operator_email_header,
+  api.auth_header, api.auth_scheme, api.auth_token_env, api.target_project
+
+Compatibility fallbacks are still accepted:
+  --admin-email, api.admin_email, api.admin_token_env, $PUBLIC_LEADS_API_TOKEN
+`;
+
+const opts = parseArgs(process.argv.slice(2));
+
+if (opts.help || !opts.input) {
+  console.log(USAGE);
+  process.exit(opts.help ? 0 : 2);
+}
+
+try {
+  const outputPath = opts.out || 'data/ingest-response.json';
+  const output = await ingestArtifact(opts.input, {
+    ...opts,
+    out: outputPath,
+  });
+
+  if (opts.dryRun) {
+    console.log(`dry run: wrote ${relativeProjectPath(outputPath)} (${output.payload.leads.length} leads)`);
+    process.exit(0);
+  }
+
+  const jobId = output.response?.job?.id || opts.jobId || '';
+  console.log(`ingested ${output.leadCount} leads${jobId ? ` as job ${jobId}` : ''}; wrote ${relativeProjectPath(outputPath)}`);
+} catch (error) {
+  console.error(error instanceof Error ? error.message : String(error));
+  process.exit(1);
+}

package/scripts/manifest.mjs

@@ -0,0 +1,71 @@
+#!/usr/bin/env node
+
+import { existsSync } from 'fs';
+import {
+  createManifestRecord,
+  parseArgs,
+  readJson,
+  readLeadArtifact,
+  relativeProjectPath,
+  resolveProjectPath,
+  validatePayload,
+  writeJson,
+} from '../lib/leadharness-leads.mjs';
+
+const USAGE = `public-leads manifest -- build/update the lead artifact manifest
+
+Usage:
+  public-leads manifest --input <file> [--manifest data/lead-manifest.json] [--json]
+`;
+
+const opts = parseArgs(process.argv.slice(2));
+
+if (opts.help || !opts.input) {
+  console.log(USAGE);
+  process.exit(opts.help ? 0 : 2);
+}
+
+try {
+  const manifestPath = resolveProjectPath(opts.manifest || 'data/lead-manifest.json');
+  const payload = readLeadArtifact(opts.input);
+  const validation = validatePayload(payload);
+  const record = createManifestRecord({
+    inputPath: opts.input,
+    payload,
+    validation,
+  });
+  const manifest = existsSync(manifestPath)
+    ? readJson(manifestPath)
+    : { version: 1, generatedAt: new Date().toISOString(), batches: [] };
+
+  manifest.version = 1;
+  manifest.generatedAt = new Date().toISOString();
+  manifest.batches = Array.isArray(manifest.batches) ? manifest.batches : [];
+
+  const existingIndex = manifest.batches.findIndex((item) => item.input === record.input);
+  if (existingIndex === -1) manifest.batches.push(record);
+  else manifest.batches[existingIndex] = record;
+
+  manifest.summary = {
+    batchCount: manifest.batches.length,
+    readyForIngest: manifest.batches.filter((item) => item.readyForIngest).length,
+    leadCount: manifest.batches.reduce((sum, item) => sum + item.leadCount, 0),
+    errorCount: manifest.batches.reduce((sum, item) => sum + item.validation.errors, 0),
+    warningCount: manifest.batches.reduce((sum, item) => sum + item.validation.warnings, 0),
+  };
+
+  writeJson(manifestPath, manifest);
+
+  if (opts.json) {
+    console.log(JSON.stringify({ manifest: relativeProjectPath(manifestPath), record, summary: manifest.summary }, null, 2));
+  } else {
+    console.log(`manifest: ${relativeProjectPath(manifestPath)}`);
+    console.log(`batch: ${record.id} leads=${record.leadCount} ready=${record.readyForIngest}`);
+    console.log(`summary: batches=${manifest.summary.batchCount} ready=${manifest.summary.readyForIngest} leads=${manifest.summary.leadCount}`);
+  }
+
+  process.exit(validation.ok ? 0 : 1);
+} catch (error) {
+  console.error(error instanceof Error ? error.message : String(error));
+  process.exit(1);
+}

package/scripts/pipeline.mjs

@@ -0,0 +1,118 @@
+#!/usr/bin/env node
+
+import { existsSync } from 'fs';
+import { crawlDomains } from '../lib/leadharness-crawler.mjs';
+import { ingestPayload } from '../lib/leadharness-ingest.mjs';
+import {
+  createManifestRecord,
+  loadProfileConfig,
+  parseArgs,
+  readJson,
+  relativeProjectPath,
+  resolveProjectPath,
+  validatePayload,
+  writeJson,
+} from '../lib/leadharness-leads.mjs';
+import { readRequestedDomains } from './crawl.mjs';
+
+const USAGE = `public-leads pipeline -- crawl, validate, manifest, and optionally ingest leads
+
+Usage:
+  public-leads pipeline [--domain example.com | --domains example.com,example.org]
+                        [--input data/domains.tsv] [--out data/lead-results.json]
+                        [--manifest data/lead-manifest.json]
+                        [--max-pages 10] [--min-confidence 30]
+                        [--ingest | --upload] [--dry-run]
+                        [--target-project /path/to/cold-agent-leads]
+
+Examples:
+  public-leads pipeline --domains example.com,example.org
+  public-leads pipeline --input data/domains.tsv --ingest --target-project /path/to/cold-agent-leads
+`;
+
+const opts = parseArgs(process.argv.slice(2));
+
+if (opts.help) {
+  console.log(USAGE);
+  process.exit(0);
+}
+
+try {
+  const profile = loadProfileConfig();
+  const domains = readRequestedDomains(opts);
+  const out = opts.out || 'data/lead-results.json';
+  const manifestPath = opts.manifest || 'data/lead-manifest.json';
+  const ingestOut = opts.ingestOut || opts.ingestResponse || 'data/ingest-response.json';
+
+  const payload = await crawlDomains(domains, {
+    maxPages: opts.maxPages || profile.maxPages,
+    minConfidence: opts.minConfidence || profile.minConfidence,
+    userAgent: opts.userAgent || profile.userAgent,
+    includeBlocked: Boolean(opts.includeBlocked),
+    delayMs: opts.delayMs,
+    timeoutMs: opts.timeoutMs,
+    jobId: opts.jobId,
+  });
+  writeJson(resolveProjectPath(out), payload);
+
+  const validation = validatePayload(payload, { allowEmpty: Boolean(opts.allowEmpty) });
+  if (!validation.ok) {
+    for (const item of validation.issues.filter((issue) => issue.severity === 'error')) {
+      console.error(`error: ${item.path}: ${item.code}: ${item.message}`);
+    }
+    process.exit(1);
+  }
+
+  const manifest = upsertManifest(manifestPath, out, payload, validation);
+  console.log(`pipeline: wrote ${relativeProjectPath(resolveProjectPath(out))}`);
+  console.log(`manifest: ${relativeProjectPath(resolveProjectPath(manifestPath))}`);
+  console.log(`domains=${validation.summary.domainCount} leads=${validation.summary.leadCount} results=${validation.summary.resultCount} errors=${payload.errors.length}`);
+
+  if (opts.ingest || opts.upload) {
+    const ingestResult = await ingestPayload(payload, {
+      ...opts,
+      input: out,
+      out: ingestOut,
+      dryRun: Boolean(opts.dryRun),
+    });
+    if (opts.dryRun) {
+      console.log(`dry-run ingest: wrote ${relativeProjectPath(resolveProjectPath(ingestOut))}`);
+    } else {
+      const jobId = ingestResult.response?.job?.id || payload.jobId || '';
+      console.log(`ingested ${ingestResult.leadCount} leads${jobId ? ` as job ${jobId}` : ''}; wrote ${relativeProjectPath(resolveProjectPath(ingestOut))}`);
+    }
+  } else {
+    console.log('next: public-leads ingest --input ' + out);
+  }
+
+  if (opts.json) {
+    console.log(JSON.stringify({ artifact: out, manifest: manifestPath, summary: manifest.summary }, null, 2));
+  }
+} catch (error) {
+  console.error(error instanceof Error ? error.message : String(error));
+  process.exit(1);
+}
+
+function upsertManifest(manifestPath, inputPath, payload, validation) {
+  const abs = resolveProjectPath(manifestPath);
+  const record = createManifestRecord({ inputPath, payload, validation });
+  const manifest = existsSync(abs)
+    ? readJson(abs)
+    : { version: 1, generatedAt: new Date().toISOString(), batches: [] };
+
+  manifest.version = 1;
+  manifest.generatedAt = new Date().toISOString();
+  manifest.batches = Array.isArray(manifest.batches) ? manifest.batches : [];
+  const index = manifest.batches.findIndex((item) => item.input === record.input);
+  if (index === -1) manifest.batches.push(record);
+  else manifest.batches[index] = record;
+  manifest.summary = {
+    batchCount: manifest.batches.length,
+    readyForIngest: manifest.batches.filter((item) => item.readyForIngest).length,
+    leadCount: manifest.batches.reduce((sum, item) => sum + item.leadCount, 0),
+    errorCount: manifest.batches.reduce((sum, item) => sum + item.validation.errors, 0),
+    warningCount: manifest.batches.reduce((sum, item) => sum + item.validation.warnings, 0),
+  };
+  writeJson(abs, manifest);
+  return manifest;
+}

package/scripts/validate-leads.mjs

@@ -0,0 +1,69 @@
+#!/usr/bin/env node
+
+import {
+  parseArgs,
+  readLeadArtifact,
+  relativeProjectPath,
+  validatePayload,
+  writeJson,
+} from '../lib/leadharness-leads.mjs';
+
+const USAGE = `public-leads validate -- validate lead artifacts
+
+Usage:
+  public-leads validate --input <file> [--json] [--out <file>] [--allow-empty]
+
+Accepted input:
+  - JSON object with { jobId?, domains?, leads, results?, errors? }
+  - JSON array of lead records
+  - JSONL lead records
+`;
+
+const opts = parseArgs(process.argv.slice(2));
+
+if (opts.help || !opts.input) {
+  console.log(USAGE);
+  process.exit(opts.help ? 0 : 2);
+}
+
+try {
+  const payload = readLeadArtifact(opts.input);
+  const result = validatePayload(payload, { allowEmpty: Boolean(opts.allowEmpty) });
+  const report = {
+    input: opts.input,
+    ok: result.ok,
+    summary: result.summary,
+    errors: result.errors,
+    warnings: result.warnings,
+    issues: result.issues,
+    payload,
+  };
+
+  if (opts.out) {
+    writeJson(opts.out, report);
+  }
+
+  if (opts.json) {
+    console.log(JSON.stringify(report, null, 2));
+  } else {
+    printReport(report);
+  }
+
+  process.exit(result.ok ? 0 : 1);
+} catch (error) {
+  console.error(error instanceof Error ? error.message : String(error));
+  process.exit(1);
+}
+
+function printReport(report) {
+  const rel = relativeProjectPath(report.input);
+  console.log(`${report.ok ? 'OK' : 'ERROR'} ${rel}`);
+  console.log(`domains=${report.summary.domainCount} leads=${report.summary.leadCount} results=${report.summary.resultCount} avgConfidence=${report.summary.averageConfidence}`);
+  if (Object.keys(report.summary.byType).length > 0) {
+    console.log(`types=${Object.entries(report.summary.byType).map(([type, count]) => `${type}:${count}`).join(', ')}`);
+  }
+  for (const item of report.issues) {
+    const prefix = item.severity === 'error' ? 'error' : 'warn';
+    console.log(`${prefix}: ${item.path}: ${item.code}: ${item.message}`);
+  }
+}

package/templates/canon.json

@@ -0,0 +1,26 @@
+{
+  "version": 1,
+  "profiles": [
+    {
+      "name": "publicleads",
+      "entities": {
+        "domain": {
+          "case": "lower",
+          "stripPrefixes": ["http://", "https://", "www."],
+          "stripSuffixes": ["/"],
+          "collapseWhitespace": true
+        },
+        "company": {
+          "case": "lower",
+          "collapseWhitespace": true,
+          "stripPunctuation": true,
+          "stripSuffixes": [" inc", " llc", " ltd", " co", " corp", " corporation"]
+        },
+        "lead": {
+          "case": "lower",
+          "join": ["domain", "email", "contactName", "title"]
+        }
+      }
+    }
+  ]
+}

package/templates/capabilities.json

@@ -0,0 +1,23 @@
+{
+  "version": 1,
+  "roles": {
+    "general-free": {
+      "description": "Procedural crawl, extract, validate, and ingest helper.",
+      "tools": ["geometra", "filesystem", "bash"],
+      "filesystem": ["read", "write"],
+      "network": ["public-web", "lead-ingest-api"]
+    },
+    "general-paid": {
+      "description": "Quality-sensitive lead fit and evidence review helper.",
+      "tools": ["geometra", "filesystem", "bash"],
+      "filesystem": ["read", "write"],
+      "network": ["public-web"]
+    },
+    "glm-minimal": {
+      "description": "Narrow JSON transform helper.",
+      "tools": [],
+      "filesystem": [],
+      "network": []
+    }
+  }
+}

package/templates/context.json

@@ -0,0 +1,20 @@
+{
+  "version": 1,
+  "bundles": [
+    {
+      "name": "crawl",
+      "budget": 24000,
+      "files": ["modes/_shared.md", "modes/crawl.md", "templates/lead-schema.json"]
+    },
+    {
+      "name": "pipeline",
+      "budget": 26000,
+      "files": ["modes/_shared.md", "modes/pipeline.md", "templates/lead-schema.json", "data/pipeline.md"]
+    },
+    {
+      "name": "ingest",
+      "budget": 18000,
+      "files": ["modes/_shared.md", "modes/ingest.md", "templates/lead-schema.json", "config/profile.yml"]
+    }
+  ]
+}

package/templates/contracts.json

@@ -0,0 +1,72 @@
+{
+  "contracts": [
+    {
+      "name": "publicleads.lead",
+      "version": "1.0.0",
+      "description": "Source-backed lead record produced by the agentic harness.",
+      "fields": [
+        { "name": "id", "type": "string" },
+        { "name": "company", "type": "string" },
+        { "name": "domain", "type": "string", "required": true },
+        { "name": "websiteUrl", "type": "url" },
+        { "name": "contactName", "type": "string" },
+        { "name": "title", "type": "string" },
+        { "name": "email", "type": "string" },
+        {
+          "name": "emailType",
+          "type": "enum",
+          "required": true,
+          "values": ["person", "role", "blocked", "contact_path", "unknown"]
+        },
+        { "name": "sourceUrl", "type": "url", "required": true },
+        { "name": "sourceLabel", "type": "string" },
+        { "name": "evidence", "type": "string", "required": true },
+        { "name": "extractionMethod", "type": "string", "required": true },
+        {
+          "name": "verificationStatus",
+          "type": "enum",
+          "required": true,
+          "values": ["verified", "mx_verified", "unverified", "not_applicable", "blocked", "unknown"]
+        },
+        { "name": "confidence", "type": "integer", "required": true, "min": 0, "max": 100 },
+        { "name": "warnings", "type": "json" },
+        { "name": "phone", "type": "string" },
+        { "name": "socialUrls", "type": "json" },
+        { "name": "contactUrls", "type": "json" },
+        { "name": "foundAt", "type": "datetime" }
+      ],
+      "formats": {
+        "tsv": {
+          "style": "delimited",
+          "delimiter": "tab",
+          "fields": [
+            "company",
+            "domain",
+            "contactName",
+            "title",
+            "email",
+            "emailType",
+            "verificationStatus",
+            "confidence",
+            "sourceUrl",
+            "evidence"
+          ]
+        }
+      }
+    },
+    {
+      "name": "publicleads.ingest-outcome",
+      "version": "1.0.0",
+      "description": "Structured outcome from submitting an artifact to an ingest API.",
+      "fields": [
+        { "name": "status", "type": "enum", "required": true, "values": ["INGESTED", "INGEST FAILED", "DRY RUN"] },
+        { "name": "input", "type": "string", "required": true },
+        { "name": "jobId", "type": "string" },
+        { "name": "leadCount", "type": "integer", "required": true },
+        { "name": "api", "type": "url" },
+        { "name": "responsePath", "type": "string" },
+        { "name": "reason", "type": "string" }
+      ]
+    }
+  ]
+}

package/templates/facts.json

@@ -0,0 +1,13 @@
+{
+  "version": 1,
+  "facts": [
+    {
+      "name": "lead",
+      "sources": ["data/lead-results*.json", "batch/lead-results*.json"]
+    },
+    {
+      "name": "ingest",
+      "sources": ["data/ingest-response.json", "batch/ingest-response*.json"]
+    }
+  ]
+}

package/templates/index.json

@@ -0,0 +1,13 @@
+{
+  "version": 1,
+  "sources": [
+    {
+      "name": "lead-artifacts",
+      "globs": ["data/lead-results*.json", "batch/lead-results*.json", "batch/ingest-payload*.json"]
+    },
+    {
+      "name": "lead-manifest",
+      "globs": ["data/lead-manifest.json"]
+    }
+  ]
+}

package/templates/lead-schema.json

@@ -0,0 +1,143 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "urn:public-leads-harness:templates:lead-schema",
+  "title": "Lead ingest payload",
+  "description": "Portable payload for lead review or ingest APIs.",
+  "type": "object",
+  "additionalProperties": true,
+  "required": ["leads"],
+  "properties": {
+    "jobId": {
+      "type": "string",
+      "description": "Optional stable crawl job ID. The app generates one when omitted."
+    },
+    "domains": {
+      "type": "array",
+      "items": { "type": "string" },
+      "default": []
+    },
+    "leads": {
+      "type": "array",
+      "items": { "$ref": "#/$defs/lead" }
+    },
+    "results": {
+      "type": "array",
+      "items": { "$ref": "#/$defs/result" },
+      "default": []
+    },
+    "errors": {
+      "type": "array",
+      "items": { "type": "string" },
+      "default": []
+    }
+  },
+  "$defs": {
+    "lead": {
+      "type": "object",
+      "additionalProperties": true,
+      "required": [
+        "domain",
+        "emailType",
+        "sourceUrl",
+        "evidence",
+        "extractionMethod",
+        "verificationStatus",
+        "confidence"
+      ],
+      "properties": {
+        "id": { "type": "string" },
+        "company": { "type": "string" },
+        "domain": { "type": "string" },
+        "websiteUrl": { "type": "string" },
+        "contactName": { "type": "string" },
+        "title": { "type": "string" },
+        "email": { "type": "string" },
+        "emailType": {
+          "type": "string",
+          "enum": ["person", "role", "blocked", "contact_path", "unknown"]
+        },
+        "sourceUrl": { "type": "string" },
+        "sourceLabel": { "type": "string" },
+        "evidence": { "type": "string" },
+        "extractionMethod": { "type": "string" },
+        "verificationStatus": {
+          "type": "string",
+          "enum": ["verified", "mx_verified", "unverified", "not_applicable", "blocked", "unknown"]
+        },
+        "confidence": {
+          "type": "integer",
+          "minimum": 0,
+          "maximum": 100
+        },
+        "warnings": {
+          "type": "array",
+          "items": { "type": "string" },
+          "default": []
+        },
+        "phone": {
+          "type": "string",
+          "description": "Optional normalized phone number found near the public contact evidence."
+        },
+        "socialUrls": {
+          "type": "array",
+          "items": { "type": "string" },
+          "default": [],
+          "description": "Optional public social/profile URLs found on the same source page."
+        },
+        "contactUrls": {
+          "type": "array",
+          "items": { "type": "string" },
+          "default": [],
+          "description": "Optional public scheduling or contact URLs found on the same source page."
+        },
+        "foundAt": {
+          "type": "string",
+          "format": "date-time"
+        }
+      }
+    },
+    "result": {
+      "type": "object",
+      "additionalProperties": true,
+      "required": ["domain"],
+      "properties": {
+        "domain": { "type": "string" },
+        "websiteUrl": { "type": "string" },
+        "leads": {
+          "type": "array",
+          "items": { "$ref": "#/$defs/lead" },
+          "default": []
+        },
+        "pages": {
+          "type": "array",
+          "items": { "$ref": "#/$defs/pageVisit" },
+          "default": []
+        },
+        "warnings": {
+          "type": "array",
+          "items": { "type": "string" },
+          "default": []
+        },
+        "completedAt": {
+          "type": "string",
+          "format": "date-time"
+        }
+      }
+    },
+    "pageVisit": {
+      "type": "object",
+      "additionalProperties": true,
+      "required": ["url"],
+      "properties": {
+        "url": { "type": "string" },
+        "title": { "type": "string" },
+        "statusCode": { "type": "integer" },
+        "emailsFound": { "type": "integer" },
+        "phonesFound": { "type": "integer" },
+        "socialUrlsFound": { "type": "integer" },
+        "contactUrlsFound": { "type": "integer" },
+        "error": { "type": "string" }
+      }
+    }
+  }
+}

package/templates/lineage.json

@@ -0,0 +1,10 @@
+{
+  "version": 1,
+  "artifacts": [
+    {
+      "kind": "ingest-payload",
+      "inputs": ["lead-results"],
+      "outputs": ["ingest-response"]
+    }
+  ]
+}

package/templates/migrations.json

@@ -0,0 +1,4 @@
+{
+  "version": 1,
+  "migrations": []
+}

package/templates/postflight.json

@@ -0,0 +1,11 @@
+{
+  "version": 1,
+  "workflows": [
+    {
+      "name": "publicleads.ingest",
+      "terminalStatuses": ["INGESTED", "INGEST FAILED", "DRY RUN"],
+      "requiredArtifacts": ["input", "responsePath"],
+      "successStatuses": ["INGESTED", "DRY RUN"]
+    }
+  ]
+}

package/templates/preflight.json

@@ -0,0 +1,19 @@
+{
+  "version": 1,
+  "workflows": [
+    {
+      "name": "publicleads.crawl",
+      "maxParallel": 2,
+      "requiredSourceFields": ["domain"],
+      "blockGates": ["duplicate", "robotsBlocked", "privateSource"],
+      "warnGates": ["lowSignal", "noCompany"]
+    },
+    {
+      "name": "publicleads.ingest",
+      "maxParallel": 1,
+      "requiredSourceFields": ["input", "leadCount", "validationPath"],
+      "blockGates": ["validationErrors", "missingToken", "missingOperatorEmail"],
+      "warnGates": ["validationWarnings"]
+    }
+  ]
+}