@agent-pattern-labs/leads-rig 0.1.7 → 0.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/lead-harness.mjs +2 -0
- package/docs/ARCHITECTURE.md +12 -0
- package/docs/PRIMARY_PLATFORM.md +35 -0
- package/lib/leadharness-crawler.mjs +2 -0
- package/lib/leadharness-leads.mjs +124 -0
- package/lib/leadharness-primary-platform.mjs +512 -0
- package/package.json +2 -1
- package/scripts/enrich-platform.mjs +92 -0
- package/templates/contracts.json +16 -0
- package/templates/lead-schema.json +58 -0
package/bin/lead-harness.mjs
CHANGED
|
@@ -17,6 +17,7 @@ const commands = {
|
|
|
17
17
|
validate: 'scripts/validate-leads.mjs',
|
|
18
18
|
manifest: 'scripts/manifest.mjs',
|
|
19
19
|
ingest: 'scripts/ingest.mjs',
|
|
20
|
+
'enrich:platform': 'scripts/enrich-platform.mjs',
|
|
20
21
|
batch: 'scripts/batch-orchestrator.mjs',
|
|
21
22
|
verify: 'verify-pipeline.mjs',
|
|
22
23
|
sync: 'bin/sync.mjs',
|
|
@@ -186,6 +187,7 @@ Core commands:
|
|
|
186
187
|
validate Validate lead artifacts against the local contract
|
|
187
188
|
manifest Build/update data/lead-manifest.json from lead artifacts
|
|
188
189
|
ingest Submit a validated payload to the configured ingest API
|
|
190
|
+
enrich:platform Measure public activity and select a refreshable primary platform
|
|
189
191
|
verify Run the full harness verification gate
|
|
190
192
|
sync Re-run consumer-project symlink sync
|
|
191
193
|
|
package/docs/ARCHITECTURE.md
CHANGED
|
@@ -82,4 +82,16 @@ The package also carries the `@agent-pattern-labs/iso-*` helper ecosystem for tr
|
|
|
82
82
|
- page visits
|
|
83
83
|
- ingest requests
|
|
84
84
|
|
|
85
|
+
Lead records preserve optional enrichment fields such as `linkedinUrl`, `phone`,
|
|
86
|
+
`address`, `streetAddress`, `location`, `city`, `region`, `postalCode`, and
|
|
87
|
+
`country` through normalization, deduplication, and ingest.
|
|
88
|
+
|
|
89
|
+
Primary-platform enrichment adds person-matched `socialProfiles` plus a compact
|
|
90
|
+
consumer-facing selection (`primaryPlatform`, URL, confidence, method,
|
|
91
|
+
evidence, checked timestamp, and next-refresh timestamp). Dated activity is
|
|
92
|
+
measured only through official public GitHub, Bluesky, or YouTube endpoints.
|
|
93
|
+
Networks without a permitted activity endpoint remain presence-only. The
|
|
94
|
+
default refresh interval is seven days, and a fresh checkpoint prevents repeat
|
|
95
|
+
requests.
|
|
96
|
+
|
|
85
97
|
The validator intentionally accepts the same defaults the local runtime normalizes, but it fails missing source evidence, invalid URL fields, invalid `emailType`, invalid `verificationStatus`, invalid confidence, person leads without email, and generic catch-all inboxes such as `info@`, `hello@`, or similar organizational aliases. Any email record must be a named `person` lead with a non-generic email and a human owner visible in the evidence; role inboxes, unknown-owner emails, blocked emails, unsupported emails, and unnamed person-like emails fail validation. Summaries expose `goodLeadCount`, which only counts those high-quality named human email records.
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# Primary Platform Enrichment
|
|
2
|
+
|
|
3
|
+
`public-leads enrich:platform` adds a refreshable channel recommendation to a
|
|
4
|
+
validated lead artifact without paid APIs.
|
|
5
|
+
|
|
6
|
+
```bash
|
|
7
|
+
public-leads enrich:platform \
|
|
8
|
+
--input data/leads.json \
|
|
9
|
+
--out data/leads-primary-platform.json \
|
|
10
|
+
--report data/leads-primary-platform-report.json \
|
|
11
|
+
--checkpoint data/leads-primary-platform.checkpoint.json \
|
|
12
|
+
--activity-window-days 90 \
|
|
13
|
+
--refresh-days 7 \
|
|
14
|
+
--concurrency 4
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
Run the same command weekly with the same checkpoint. Fresh profile
|
|
18
|
+
measurements are reused. `GITHUB_TOKEN` is optional and only raises GitHub's
|
|
19
|
+
public REST rate limit.
|
|
20
|
+
|
|
21
|
+
## Selection Methods
|
|
22
|
+
|
|
23
|
+
- `observed_public_activity` means a supported public endpoint returned dated
|
|
24
|
+
actions inside the configured activity window.
|
|
25
|
+
- `fallback_presence_priority` means no comparable dated signal was available;
|
|
26
|
+
the configured B2B platform order selected the best known public profile.
|
|
27
|
+
|
|
28
|
+
The worker uses GitHub public events, Bluesky's public author feed, and YouTube
|
|
29
|
+
channel Atom feeds. It does not request LinkedIn, X, Facebook, or Instagram
|
|
30
|
+
profile pages. It stores aggregate counts and timestamps, not post content.
|
|
31
|
+
|
|
32
|
+
Page-level social links are accepted only when the profile username matches the
|
|
33
|
+
named contact. Explicit person-profile fields such as `linkedinUrl` are trusted
|
|
34
|
+
inputs. This prevents company footer accounts from being assigned to an
|
|
35
|
+
individual lead.
|
|
@@ -961,6 +961,8 @@ function isPublicProfileUrl(value) {
|
|
|
961
961
|
const host = url.hostname.toLowerCase().replace(/^www\./, '');
|
|
962
962
|
const path = url.pathname.toLowerCase();
|
|
963
963
|
if (host === 'linkedin.com') return /^\/(in|company)\//.test(path);
|
|
964
|
+
if (host === 'github.com') return /^\/[a-z0-9](?:[a-z0-9-]{0,37}[a-z0-9])?\/?$/i.test(path);
|
|
965
|
+
if (host === 'bsky.app') return /^\/profile\/[^/]+\/?$/i.test(path);
|
|
964
966
|
if (host === 'x.com' || host === 'twitter.com') return path.length > 1 && !/^\/(share|intent|search|hashtag)\b/.test(path);
|
|
965
967
|
if (host === 'facebook.com' || host === 'instagram.com' || host === 'youtube.com') return path.length > 1;
|
|
966
968
|
return false;
|
|
@@ -8,6 +8,7 @@ import {
|
|
|
8
8
|
writeFileSync,
|
|
9
9
|
} from 'fs';
|
|
10
10
|
import { basename, dirname, extname, isAbsolute, join, relative, resolve } from 'path';
|
|
11
|
+
import { normalizeSocialProfiles } from './leadharness-primary-platform.mjs';
|
|
11
12
|
|
|
12
13
|
export const PROJECT_DIR = process.env.PUBLIC_LEADS_PROJECT || process.env.LEAD_HARNESS_PROJECT || process.cwd();
|
|
13
14
|
export const HARNESS_ROOT = process.env.PUBLIC_LEADS_ROOT || process.env.LEAD_HARNESS_ROOT || resolve(dirname(new URL(import.meta.url).pathname), '..');
|
|
@@ -162,9 +163,18 @@ export function normalizeLead(input, { now = new Date().toISOString() } = {}) {
|
|
|
162
163
|
const verificationStatus = normalizeVerificationStatus(input.verificationStatus, emailType);
|
|
163
164
|
const confidence = clampInteger(input.confidence, 0, 100, 0);
|
|
164
165
|
const foundAt = normalizeDateTime(input.foundAt, now);
|
|
166
|
+
const linkedinUrl = normalizeLinkedInUrl(input.linkedinUrl);
|
|
165
167
|
const phone = stringValue(input.phone);
|
|
168
|
+
const address = compactWhitespace(input.address);
|
|
169
|
+
const streetAddress = compactWhitespace(input.streetAddress);
|
|
170
|
+
const location = compactWhitespace(input.location);
|
|
171
|
+
const city = compactWhitespace(input.city);
|
|
172
|
+
const region = compactWhitespace(input.region);
|
|
173
|
+
const postalCode = compactWhitespace(input.postalCode);
|
|
174
|
+
const country = compactWhitespace(input.country);
|
|
166
175
|
const socialUrls = normalizeUrlList(input.socialUrls);
|
|
167
176
|
const contactUrls = normalizeUrlList(input.contactUrls);
|
|
177
|
+
const socialProfiles = normalizeSocialProfiles(input.socialProfiles);
|
|
168
178
|
|
|
169
179
|
const lead = {
|
|
170
180
|
id: stringValue(input.id),
|
|
@@ -187,9 +197,25 @@ export function normalizeLead(input, { now = new Date().toISOString() } = {}) {
|
|
|
187
197
|
verificationStatus,
|
|
188
198
|
confidence,
|
|
189
199
|
warnings: normalizeWarnings(input.warnings),
|
|
200
|
+
linkedinUrl,
|
|
190
201
|
phone,
|
|
202
|
+
address,
|
|
203
|
+
streetAddress,
|
|
204
|
+
location,
|
|
205
|
+
city,
|
|
206
|
+
region,
|
|
207
|
+
postalCode,
|
|
208
|
+
country,
|
|
191
209
|
socialUrls,
|
|
192
210
|
contactUrls,
|
|
211
|
+
socialProfiles,
|
|
212
|
+
primaryPlatform: stringValue(input.primaryPlatform).toLowerCase(),
|
|
213
|
+
primaryPlatformUrl: normalizeHttpUrl(input.primaryPlatformUrl),
|
|
214
|
+
primaryPlatformConfidence: clampInteger(input.primaryPlatformConfidence, 0, 100, 0),
|
|
215
|
+
primaryPlatformSelectionMethod: stringValue(input.primaryPlatformSelectionMethod),
|
|
216
|
+
primaryPlatformEvidence: compactWhitespace(input.primaryPlatformEvidence),
|
|
217
|
+
primaryPlatformCheckedAt: normalizeOptionalDateTime(input.primaryPlatformCheckedAt),
|
|
218
|
+
primaryPlatformNextRefreshAt: normalizeOptionalDateTime(input.primaryPlatformNextRefreshAt),
|
|
193
219
|
foundAt,
|
|
194
220
|
};
|
|
195
221
|
|
|
@@ -343,6 +369,20 @@ export function validateLead(lead, path, issues) {
|
|
|
343
369
|
if (!isHttpUrl(url)) issues.push(issue('warning', `${path}.contactUrls[${index}]`, 'invalid_contact_url', 'contactUrls entries should be http(s) URLs'));
|
|
344
370
|
});
|
|
345
371
|
}
|
|
372
|
+
if (!Array.isArray(lead.socialProfiles)) {
|
|
373
|
+
issues.push(issue('error', `${path}.socialProfiles`, 'invalid_social_profiles', 'socialProfiles must be an array'));
|
|
374
|
+
} else {
|
|
375
|
+
lead.socialProfiles.forEach((profile, index) => {
|
|
376
|
+
if (!profile?.platform) issues.push(issue('warning', `${path}.socialProfiles[${index}].platform`, 'missing_platform', 'social profile platform should be present'));
|
|
377
|
+
if (!isHttpUrl(profile?.url)) issues.push(issue('warning', `${path}.socialProfiles[${index}].url`, 'invalid_profile_url', 'social profile URL should be http(s)'));
|
|
378
|
+
});
|
|
379
|
+
}
|
|
380
|
+
if (lead.primaryPlatformUrl && !isHttpUrl(lead.primaryPlatformUrl)) {
|
|
381
|
+
issues.push(issue('warning', `${path}.primaryPlatformUrl`, 'invalid_primary_platform_url', 'primaryPlatformUrl should be an http(s) URL'));
|
|
382
|
+
}
|
|
383
|
+
if (!Number.isInteger(lead.primaryPlatformConfidence) || lead.primaryPlatformConfidence < 0 || lead.primaryPlatformConfidence > 100) {
|
|
384
|
+
issues.push(issue('error', `${path}.primaryPlatformConfidence`, 'invalid_primary_platform_confidence', 'primaryPlatformConfidence must be an integer from 0 to 100'));
|
|
385
|
+
}
|
|
346
386
|
if (!Array.isArray(lead.warnings)) {
|
|
347
387
|
issues.push(issue('error', `${path}.warnings`, 'invalid_warnings', 'warnings must be an array'));
|
|
348
388
|
}
|
|
@@ -547,8 +587,25 @@ function normalizeLeadRecord(lead) {
|
|
|
547
587
|
sources,
|
|
548
588
|
evidence: sources[0]?.evidence || compactWhitespace(lead?.evidence),
|
|
549
589
|
warnings: normalizeWarnings(lead?.warnings),
|
|
590
|
+
linkedinUrl: normalizeLinkedInUrl(lead?.linkedinUrl),
|
|
591
|
+
phone: stringValue(lead?.phone),
|
|
592
|
+
address: compactWhitespace(lead?.address),
|
|
593
|
+
streetAddress: compactWhitespace(lead?.streetAddress),
|
|
594
|
+
location: compactWhitespace(lead?.location),
|
|
595
|
+
city: compactWhitespace(lead?.city),
|
|
596
|
+
region: compactWhitespace(lead?.region),
|
|
597
|
+
postalCode: compactWhitespace(lead?.postalCode),
|
|
598
|
+
country: compactWhitespace(lead?.country),
|
|
550
599
|
socialUrls: normalizeUrlList(lead?.socialUrls),
|
|
551
600
|
contactUrls: normalizeUrlList(lead?.contactUrls),
|
|
601
|
+
socialProfiles: normalizeSocialProfiles(lead?.socialProfiles),
|
|
602
|
+
primaryPlatform: stringValue(lead?.primaryPlatform).toLowerCase(),
|
|
603
|
+
primaryPlatformUrl: normalizeHttpUrl(lead?.primaryPlatformUrl),
|
|
604
|
+
primaryPlatformConfidence: clampInteger(lead?.primaryPlatformConfidence, 0, 100, 0),
|
|
605
|
+
primaryPlatformSelectionMethod: stringValue(lead?.primaryPlatformSelectionMethod),
|
|
606
|
+
primaryPlatformEvidence: compactWhitespace(lead?.primaryPlatformEvidence),
|
|
607
|
+
primaryPlatformCheckedAt: normalizeOptionalDateTime(lead?.primaryPlatformCheckedAt),
|
|
608
|
+
primaryPlatformNextRefreshAt: normalizeOptionalDateTime(lead?.primaryPlatformNextRefreshAt),
|
|
552
609
|
};
|
|
553
610
|
normalized.id = stableLeadID(normalized);
|
|
554
611
|
return normalized;
|
|
@@ -574,9 +631,25 @@ export function mergeLeadRecords(current, candidate) {
|
|
|
574
631
|
? secondary.verificationStatus
|
|
575
632
|
: primary.verificationStatus,
|
|
576
633
|
confidence: Math.max(primary.confidence || 0, secondary.confidence || 0),
|
|
634
|
+
linkedinUrl: primary.linkedinUrl || secondary.linkedinUrl,
|
|
577
635
|
phone: primary.phone || secondary.phone,
|
|
636
|
+
address: longerString(primary.address, secondary.address),
|
|
637
|
+
streetAddress: primary.streetAddress || secondary.streetAddress,
|
|
638
|
+
location: longerString(primary.location, secondary.location),
|
|
639
|
+
city: primary.city || secondary.city,
|
|
640
|
+
region: primary.region || secondary.region,
|
|
641
|
+
postalCode: primary.postalCode || secondary.postalCode,
|
|
642
|
+
country: primary.country || secondary.country,
|
|
578
643
|
socialUrls: normalizeUrlList([...(primary.socialUrls || []), ...(secondary.socialUrls || [])]),
|
|
579
644
|
contactUrls: normalizeUrlList([...(primary.contactUrls || []), ...(secondary.contactUrls || [])]),
|
|
645
|
+
socialProfiles: normalizeSocialProfiles([...(primary.socialProfiles || []), ...(secondary.socialProfiles || [])]),
|
|
646
|
+
primaryPlatform: primary.primaryPlatform || secondary.primaryPlatform,
|
|
647
|
+
primaryPlatformUrl: primary.primaryPlatformUrl || secondary.primaryPlatformUrl,
|
|
648
|
+
primaryPlatformConfidence: Math.max(primary.primaryPlatformConfidence || 0, secondary.primaryPlatformConfidence || 0),
|
|
649
|
+
primaryPlatformSelectionMethod: primary.primaryPlatformSelectionMethod || secondary.primaryPlatformSelectionMethod,
|
|
650
|
+
primaryPlatformEvidence: longerString(primary.primaryPlatformEvidence, secondary.primaryPlatformEvidence),
|
|
651
|
+
primaryPlatformCheckedAt: latestDateTime(primary.primaryPlatformCheckedAt, secondary.primaryPlatformCheckedAt),
|
|
652
|
+
primaryPlatformNextRefreshAt: latestDateTime(primary.primaryPlatformNextRefreshAt, secondary.primaryPlatformNextRefreshAt),
|
|
580
653
|
warnings: normalizeWarnings([...(primary.warnings || []), ...(secondary.warnings || [])]),
|
|
581
654
|
};
|
|
582
655
|
|
|
@@ -842,6 +915,51 @@ function normalizeUrlList(value) {
|
|
|
842
915
|
return urls;
|
|
843
916
|
}
|
|
844
917
|
|
|
918
|
+
function normalizeLinkedInUrl(value) {
|
|
919
|
+
let raw = stringValue(value);
|
|
920
|
+
if (!raw) return '';
|
|
921
|
+
if (!raw.includes('://')) raw = `https://${raw}`;
|
|
922
|
+
|
|
923
|
+
try {
|
|
924
|
+
const url = new URL(raw);
|
|
925
|
+
if (url.protocol !== 'http:' && url.protocol !== 'https:') return '';
|
|
926
|
+
const host = url.hostname.toLowerCase();
|
|
927
|
+
if (host !== 'linkedin.com' && !host.endsWith('.linkedin.com')) return '';
|
|
928
|
+
|
|
929
|
+
const path = url.pathname.replace(/\/+$/, '');
|
|
930
|
+
if (!/^\/(in|pub|company|school|showcase)\//i.test(path)) return '';
|
|
931
|
+
return `https://www.linkedin.com${path}`;
|
|
932
|
+
} catch {
|
|
933
|
+
return '';
|
|
934
|
+
}
|
|
935
|
+
}
|
|
936
|
+
|
|
937
|
+
function normalizeHttpUrl(value) {
|
|
938
|
+
const raw = stringValue(value);
|
|
939
|
+
if (!raw) return '';
|
|
940
|
+
try {
|
|
941
|
+
const url = new URL(raw);
|
|
942
|
+
return ['http:', 'https:'].includes(url.protocol) ? url.toString() : '';
|
|
943
|
+
} catch {
|
|
944
|
+
return '';
|
|
945
|
+
}
|
|
946
|
+
}
|
|
947
|
+
|
|
948
|
+
function normalizeOptionalDateTime(value) {
|
|
949
|
+
const raw = stringValue(value);
|
|
950
|
+
if (!raw) return '';
|
|
951
|
+
const date = new Date(raw);
|
|
952
|
+
return Number.isNaN(date.getTime()) ? '' : date.toISOString();
|
|
953
|
+
}
|
|
954
|
+
|
|
955
|
+
function latestDateTime(...values) {
|
|
956
|
+
const dates = values
|
|
957
|
+
.map(normalizeOptionalDateTime)
|
|
958
|
+
.filter(Boolean)
|
|
959
|
+
.sort();
|
|
960
|
+
return dates.at(-1) || '';
|
|
961
|
+
}
|
|
962
|
+
|
|
845
963
|
function normalizeDateTime(value, fallback) {
|
|
846
964
|
const raw = stringValue(value);
|
|
847
965
|
if (!raw) return fallback;
|
|
@@ -876,6 +994,12 @@ function compactWhitespace(value) {
|
|
|
876
994
|
return stringValue(value).replace(/\s+/g, ' ');
|
|
877
995
|
}
|
|
878
996
|
|
|
997
|
+
function longerString(left, right) {
|
|
998
|
+
const first = stringValue(left);
|
|
999
|
+
const second = stringValue(right);
|
|
1000
|
+
return second.length > first.length ? second : first;
|
|
1001
|
+
}
|
|
1002
|
+
|
|
879
1003
|
function displayCompany(domain) {
|
|
880
1004
|
const clean = normalizeDomain(domain);
|
|
881
1005
|
if (!clean) return '';
|
|
@@ -0,0 +1,512 @@
|
|
|
1
|
+
const DEFAULT_PLATFORM_ORDER = ['linkedin', 'github', 'x', 'bluesky', 'youtube', 'instagram', 'facebook'];
|
|
2
|
+
const ACTIVITY_PLATFORMS = new Set(['github', 'bluesky', 'youtube']);
|
|
3
|
+
const BLOCKED_GITHUB_PATHS = new Set([
|
|
4
|
+
'about', 'apps', 'collections', 'customer-stories', 'enterprise', 'events',
|
|
5
|
+
'features', 'issues', 'marketplace', 'new', 'notifications', 'orgs', 'pricing',
|
|
6
|
+
'pulls', 'search', 'security', 'settings', 'sponsors', 'topics', 'trending',
|
|
7
|
+
]);
|
|
8
|
+
|
|
9
|
+
export async function enrichPrimaryPlatforms(payload, options = {}) {
|
|
10
|
+
const opts = normalizeOptions(options);
|
|
11
|
+
const leads = Array.isArray(payload?.leads) ? payload.leads : [];
|
|
12
|
+
const selected = opts.limit > 0 ? leads.slice(opts.offset, opts.offset + opts.limit) : leads.slice(opts.offset);
|
|
13
|
+
const selectedKeys = new Set(selected.map(leadKey));
|
|
14
|
+
const profilesByLead = new Map();
|
|
15
|
+
const uniqueProfiles = new Map();
|
|
16
|
+
|
|
17
|
+
for (const lead of selected) {
|
|
18
|
+
const profiles = discoverLeadProfiles(lead, opts.now);
|
|
19
|
+
profilesByLead.set(leadKey(lead), profiles);
|
|
20
|
+
for (const profile of profiles) uniqueProfiles.set(profileKey(profile), profile);
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
const measurements = new Map();
|
|
24
|
+
const pending = [...uniqueProfiles.values()].slice(0, opts.maxProfiles);
|
|
25
|
+
await mapLimit(pending, opts.concurrency, async (profile) => {
|
|
26
|
+
const key = profileKey(profile);
|
|
27
|
+
const cached = normalizeActivityProfile(opts.cache?.profiles?.[key]);
|
|
28
|
+
if (!opts.refresh && isFresh(cached, opts.now)) {
|
|
29
|
+
opts.cacheHits += 1;
|
|
30
|
+
measurements.set(key, cached);
|
|
31
|
+
return;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
const measured = await measureProfile(profile, opts);
|
|
35
|
+
if (measured.activityStatus === 'error' && hasUsableMeasurement(cached)) {
|
|
36
|
+
measurements.set(key, {
|
|
37
|
+
...cached,
|
|
38
|
+
stale: true,
|
|
39
|
+
lastError: measured.lastError,
|
|
40
|
+
});
|
|
41
|
+
return;
|
|
42
|
+
}
|
|
43
|
+
measurements.set(key, measured);
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
const enrichedByKey = new Map();
|
|
47
|
+
for (const lead of selected) {
|
|
48
|
+
const profiles = (profilesByLead.get(leadKey(lead)) || []).map((profile) => {
|
|
49
|
+
const measured = measurements.get(profileKey(profile));
|
|
50
|
+
return measured || presenceOnlyProfile(profile, opts);
|
|
51
|
+
});
|
|
52
|
+
enrichedByKey.set(leadKey(lead), applyPrimaryPlatform(lead, profiles, opts));
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
const enrichLead = (lead) => selectedKeys.has(leadKey(lead))
|
|
56
|
+
? enrichedByKey.get(leadKey(lead)) || lead
|
|
57
|
+
: lead;
|
|
58
|
+
const output = {
|
|
59
|
+
...payload,
|
|
60
|
+
leads: leads.map(enrichLead),
|
|
61
|
+
results: Array.isArray(payload?.results)
|
|
62
|
+
? payload.results.map((result) => ({
|
|
63
|
+
...result,
|
|
64
|
+
leads: Array.isArray(result?.leads) ? result.leads.map(enrichLead) : [],
|
|
65
|
+
}))
|
|
66
|
+
: [],
|
|
67
|
+
};
|
|
68
|
+
|
|
69
|
+
const enriched = [...enrichedByKey.values()];
|
|
70
|
+
const report = {
|
|
71
|
+
generatedAt: opts.now,
|
|
72
|
+
inputLeads: leads.length,
|
|
73
|
+
selectedLeads: selected.length,
|
|
74
|
+
leadsWithProfiles: enriched.filter((lead) => lead.socialProfiles.length > 0).length,
|
|
75
|
+
leadsWithPrimaryPlatform: enriched.filter((lead) => lead.primaryPlatform).length,
|
|
76
|
+
observedActivitySelections: enriched.filter((lead) => lead.primaryPlatformSelectionMethod === 'observed_public_activity').length,
|
|
77
|
+
fallbackSelections: enriched.filter((lead) => lead.primaryPlatformSelectionMethod === 'fallback_presence_priority').length,
|
|
78
|
+
uniqueProfiles: uniqueProfiles.size,
|
|
79
|
+
measuredProfiles: [...measurements.values()].filter((profile) => ['observed', 'observed_inactive'].includes(profile.activityStatus)).length,
|
|
80
|
+
publicActivityRequests: opts.publicActivityRequests,
|
|
81
|
+
cacheHits: opts.cacheHits,
|
|
82
|
+
byPrimaryPlatform: countBy(enriched, (lead) => lead.primaryPlatform || 'none'),
|
|
83
|
+
bySelectionMethod: countBy(enriched, (lead) => lead.primaryPlatformSelectionMethod || 'none'),
|
|
84
|
+
directThirdPartyApiCostUsd: 0,
|
|
85
|
+
paidApiRequests: 0,
|
|
86
|
+
refreshDays: opts.refreshDays,
|
|
87
|
+
activityWindowDays: opts.activityWindowDays,
|
|
88
|
+
};
|
|
89
|
+
const cache = {
|
|
90
|
+
version: 1,
|
|
91
|
+
updatedAt: opts.now,
|
|
92
|
+
profiles: Object.fromEntries([...measurements.entries()]),
|
|
93
|
+
};
|
|
94
|
+
return { payload: output, report, cache };
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
export function discoverLeadProfiles(lead, now = new Date().toISOString()) {
|
|
98
|
+
const candidates = [];
|
|
99
|
+
const add = (value, trusted = false) => {
|
|
100
|
+
const profile = normalizeProfileUrl(value);
|
|
101
|
+
if (!profile) return;
|
|
102
|
+
if (!trusted && !profileMatchesContact(profile, lead?.contactName)) return;
|
|
103
|
+
candidates.push({
|
|
104
|
+
...profile,
|
|
105
|
+
activityStatus: 'presence_only',
|
|
106
|
+
checkedAt: '',
|
|
107
|
+
nextRefreshAt: '',
|
|
108
|
+
discoveredAt: validDateTime(lead?.foundAt) || now,
|
|
109
|
+
});
|
|
110
|
+
};
|
|
111
|
+
|
|
112
|
+
add(lead?.linkedinUrl, true);
|
|
113
|
+
add(lead?.githubUrl, true);
|
|
114
|
+
add(lead?.twitterUrl, true);
|
|
115
|
+
add(lead?.youtubeUrl, true);
|
|
116
|
+
add(lead?.blueskyUrl, true);
|
|
117
|
+
for (const profile of Array.isArray(lead?.socialProfiles) ? lead.socialProfiles : []) {
|
|
118
|
+
const normalized = normalizeActivityProfile(profile);
|
|
119
|
+
if (normalized) candidates.push(normalized);
|
|
120
|
+
}
|
|
121
|
+
for (const value of Array.isArray(lead?.socialUrls) ? lead.socialUrls : []) add(value, false);
|
|
122
|
+
|
|
123
|
+
const byKey = new Map();
|
|
124
|
+
for (const profile of candidates) {
|
|
125
|
+
const key = profileKey(profile);
|
|
126
|
+
const current = byKey.get(key);
|
|
127
|
+
if (!current || profileRichness(profile) > profileRichness(current)) byKey.set(key, profile);
|
|
128
|
+
}
|
|
129
|
+
return [...byKey.values()].sort((left, right) => platformRank(left.platform) - platformRank(right.platform));
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
export function normalizeSocialProfiles(value) {
|
|
133
|
+
if (!Array.isArray(value)) return [];
|
|
134
|
+
const byKey = new Map();
|
|
135
|
+
for (const item of value) {
|
|
136
|
+
const normalized = normalizeActivityProfile(item);
|
|
137
|
+
if (!normalized) continue;
|
|
138
|
+
const key = profileKey(normalized);
|
|
139
|
+
const current = byKey.get(key);
|
|
140
|
+
if (!current || profileRichness(normalized) > profileRichness(current)) byKey.set(key, normalized);
|
|
141
|
+
}
|
|
142
|
+
return [...byKey.values()].sort((left, right) => platformRank(left.platform) - platformRank(right.platform));
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
export function normalizeProfileUrl(value) {
|
|
146
|
+
let url;
|
|
147
|
+
try {
|
|
148
|
+
url = new URL(String(value || '').trim().match(/^https?:\/\//i) ? String(value).trim() : `https://${String(value || '').trim()}`);
|
|
149
|
+
} catch {
|
|
150
|
+
return null;
|
|
151
|
+
}
|
|
152
|
+
if (!['http:', 'https:'].includes(url.protocol) || url.username || url.password) return null;
|
|
153
|
+
const host = url.hostname.toLowerCase().replace(/^www\./, '');
|
|
154
|
+
const parts = url.pathname.split('/').filter(Boolean).map(decodeURIComponentSafe);
|
|
155
|
+
url.search = '';
|
|
156
|
+
url.hash = '';
|
|
157
|
+
|
|
158
|
+
if ((host === 'linkedin.com' || host.endsWith('.linkedin.com')) && parts[0]?.toLowerCase() === 'in' && parts[1]) {
|
|
159
|
+
return profile('linkedin', `https://www.linkedin.com/in/${encodeURIComponent(parts[1])}`, parts[1]);
|
|
160
|
+
}
|
|
161
|
+
if (host === 'github.com' && parts.length === 1 && !BLOCKED_GITHUB_PATHS.has(parts[0].toLowerCase())) {
|
|
162
|
+
return profile('github', `https://github.com/${encodeURIComponent(parts[0])}`, parts[0]);
|
|
163
|
+
}
|
|
164
|
+
if ((host === 'x.com' || host === 'twitter.com') && parts.length === 1 && !['home', 'search', 'share', 'intent', 'hashtag'].includes(parts[0].toLowerCase())) {
|
|
165
|
+
return profile('x', `https://x.com/${encodeURIComponent(parts[0])}`, parts[0]);
|
|
166
|
+
}
|
|
167
|
+
if (host === 'bsky.app' && parts[0]?.toLowerCase() === 'profile' && parts[1]) {
|
|
168
|
+
return profile('bluesky', `https://bsky.app/profile/${encodeURIComponent(parts[1])}`, parts[1]);
|
|
169
|
+
}
|
|
170
|
+
if ((host === 'youtube.com' || host === 'm.youtube.com') && parts[0]) {
|
|
171
|
+
if (parts[0].toLowerCase() === 'channel' && parts[1]) {
|
|
172
|
+
return { ...profile('youtube', `https://www.youtube.com/channel/${encodeURIComponent(parts[1])}`, parts[1]), channelId: parts[1] };
|
|
173
|
+
}
|
|
174
|
+
if (parts[0].startsWith('@')) return profile('youtube', `https://www.youtube.com/${encodeURIComponent(parts[0])}`, parts[0].slice(1));
|
|
175
|
+
}
|
|
176
|
+
if (host === 'instagram.com' && parts.length === 1) {
|
|
177
|
+
return profile('instagram', `https://www.instagram.com/${encodeURIComponent(parts[0])}`, parts[0]);
|
|
178
|
+
}
|
|
179
|
+
if (host === 'facebook.com' && parts.length === 1 && !['share', 'sharer', 'groups', 'pages'].includes(parts[0].toLowerCase())) {
|
|
180
|
+
return profile('facebook', `https://www.facebook.com/${encodeURIComponent(parts[0])}`, parts[0]);
|
|
181
|
+
}
|
|
182
|
+
return null;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
export function selectPrimaryPlatform(profiles, options = {}) {
|
|
186
|
+
const order = normalizePlatformOrder(options.platformOrder);
|
|
187
|
+
const active = profiles
|
|
188
|
+
.filter((profile) => profile.activityStatus === 'observed' && profile.publicActivityCount > 0)
|
|
189
|
+
.sort((left, right) => activityScore(right, options.now) - activityScore(left, options.now));
|
|
190
|
+
if (active.length > 0) {
|
|
191
|
+
const winner = active[0];
|
|
192
|
+
const measuredCount = profiles.filter((profile) => ['observed', 'observed_inactive'].includes(profile.activityStatus)).length;
|
|
193
|
+
const gap = active.length > 1 ? activityScore(active[0], options.now) - activityScore(active[1], options.now) : 0;
|
|
194
|
+
const confidence = clamp(60 + (measuredCount > 1 ? 15 : 0) + (gap >= 25 ? 10 : 0), 0, 90);
|
|
195
|
+
return {
|
|
196
|
+
platform: winner.platform,
|
|
197
|
+
url: winner.url,
|
|
198
|
+
confidence,
|
|
199
|
+
selectionMethod: 'observed_public_activity',
|
|
200
|
+
evidence: winner.evidence,
|
|
201
|
+
};
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
const fallback = [...profiles].sort((left, right) => order.indexOf(left.platform) - order.indexOf(right.platform))[0];
|
|
205
|
+
if (!fallback) return { platform: '', url: '', confidence: 0, selectionMethod: '', evidence: '' };
|
|
206
|
+
return {
|
|
207
|
+
platform: fallback.platform,
|
|
208
|
+
url: fallback.url,
|
|
209
|
+
confidence: profiles.length === 1 ? 40 : 30,
|
|
210
|
+
selectionMethod: 'fallback_presence_priority',
|
|
211
|
+
evidence: `No comparable dated public activity was available; selected ${displayPlatform(fallback.platform)} using the configured presence fallback order.`,
|
|
212
|
+
};
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
async function measureProfile(profile, opts) {
|
|
216
|
+
if (!ACTIVITY_PLATFORMS.has(profile.platform)) return presenceOnlyProfile(profile, opts);
|
|
217
|
+
try {
|
|
218
|
+
if (profile.platform === 'github') return await measureGitHub(profile, opts);
|
|
219
|
+
if (profile.platform === 'bluesky') return await measureBluesky(profile, opts);
|
|
220
|
+
if (profile.platform === 'youtube') return await measureYouTube(profile, opts);
|
|
221
|
+
} catch (error) {
|
|
222
|
+
return {
|
|
223
|
+
...presenceOnlyProfile(profile, opts),
|
|
224
|
+
activityStatus: 'error',
|
|
225
|
+
lastError: error instanceof Error ? error.message : String(error),
|
|
226
|
+
};
|
|
227
|
+
}
|
|
228
|
+
return presenceOnlyProfile(profile, opts);
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
async function measureGitHub(profile, opts) {
|
|
232
|
+
const endpoint = `https://api.github.com/users/${encodeURIComponent(profile.username)}/events/public?per_page=100`;
|
|
233
|
+
const headers = {
|
|
234
|
+
accept: 'application/vnd.github+json',
|
|
235
|
+
'user-agent': opts.userAgent,
|
|
236
|
+
'x-github-api-version': '2022-11-28',
|
|
237
|
+
};
|
|
238
|
+
if (opts.githubToken) headers.authorization = `Bearer ${opts.githubToken}`;
|
|
239
|
+
opts.publicActivityRequests += 1;
|
|
240
|
+
const events = await fetchJSON(endpoint, { ...opts, headers });
|
|
241
|
+
if (!Array.isArray(events)) throw new Error('GitHub returned an invalid public-events payload');
|
|
242
|
+
return observedProfile(profile, events.map((event) => event?.created_at), endpoint, opts, 'GitHub public events');
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
async function measureBluesky(profile, opts) {
|
|
246
|
+
const endpoint = new URL('https://public.api.bsky.app/xrpc/app.bsky.feed.getAuthorFeed');
|
|
247
|
+
endpoint.searchParams.set('actor', profile.username);
|
|
248
|
+
endpoint.searchParams.set('filter', 'posts_no_replies');
|
|
249
|
+
endpoint.searchParams.set('limit', '100');
|
|
250
|
+
opts.publicActivityRequests += 1;
|
|
251
|
+
const body = await fetchJSON(endpoint, {
|
|
252
|
+
...opts,
|
|
253
|
+
headers: { accept: 'application/json', 'user-agent': opts.userAgent },
|
|
254
|
+
});
|
|
255
|
+
if (!Array.isArray(body?.feed)) throw new Error('Bluesky returned an invalid author-feed payload');
|
|
256
|
+
const dates = body.feed.map((item) => item?.post?.record?.createdAt || item?.post?.indexedAt);
|
|
257
|
+
return observedProfile(profile, dates, endpoint.toString(), opts, 'Bluesky public author posts');
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
async function measureYouTube(profile, opts) {
|
|
261
|
+
if (!profile.channelId) return presenceOnlyProfile(profile, opts);
|
|
262
|
+
const endpoint = `https://www.youtube.com/feeds/videos.xml?channel_id=${encodeURIComponent(profile.channelId)}`;
|
|
263
|
+
opts.publicActivityRequests += 1;
|
|
264
|
+
const xml = await fetchText(endpoint, {
|
|
265
|
+
...opts,
|
|
266
|
+
headers: { accept: 'application/atom+xml,application/xml,text/xml', 'user-agent': opts.userAgent },
|
|
267
|
+
});
|
|
268
|
+
const dates = [...xml.matchAll(/<published>([^<]+)<\/published>/gi)].map((match) => match[1]);
|
|
269
|
+
return observedProfile(profile, dates, endpoint, opts, 'YouTube public channel uploads');
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
function observedProfile(profile, rawDates, sourceUrl, opts, label) {
|
|
273
|
+
const cutoff = new Date(new Date(opts.now).getTime() - opts.activityWindowDays * 86_400_000);
|
|
274
|
+
const dates = rawDates.map(validDate).filter((date) => date && date >= cutoff).sort((left, right) => right - left);
|
|
275
|
+
const count = dates.length;
|
|
276
|
+
const status = count > 0 ? 'observed' : 'observed_inactive';
|
|
277
|
+
const last = dates[0]?.toISOString() || '';
|
|
278
|
+
return {
|
|
279
|
+
...profile,
|
|
280
|
+
activityStatus: status,
|
|
281
|
+
activityWindowDays: opts.activityWindowDays,
|
|
282
|
+
publicActivityCount: count,
|
|
283
|
+
lastPublicActivityAt: last,
|
|
284
|
+
checkedAt: opts.now,
|
|
285
|
+
nextRefreshAt: addDays(opts.now, opts.refreshDays),
|
|
286
|
+
activitySourceUrl: sourceUrl,
|
|
287
|
+
evidence: count > 0
|
|
288
|
+
? `${label} show ${count} dated public activities in the last ${opts.activityWindowDays} days; the most recent was ${last}.`
|
|
289
|
+
: `${label} showed no dated public activity in the last ${opts.activityWindowDays} days.`,
|
|
290
|
+
stale: false,
|
|
291
|
+
lastError: '',
|
|
292
|
+
};
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
function presenceOnlyProfile(profile, opts) {
|
|
296
|
+
return {
|
|
297
|
+
...profile,
|
|
298
|
+
activityStatus: 'presence_only',
|
|
299
|
+
activityWindowDays: opts.activityWindowDays,
|
|
300
|
+
publicActivityCount: 0,
|
|
301
|
+
lastPublicActivityAt: '',
|
|
302
|
+
checkedAt: opts.now,
|
|
303
|
+
nextRefreshAt: addDays(opts.now, opts.refreshDays),
|
|
304
|
+
activitySourceUrl: '',
|
|
305
|
+
evidence: `${displayPlatform(profile.platform)} profile presence is public, but no permitted comparable activity measurement was available.`,
|
|
306
|
+
stale: false,
|
|
307
|
+
lastError: '',
|
|
308
|
+
};
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
function applyPrimaryPlatform(lead, profiles, opts) {
|
|
312
|
+
const primary = selectPrimaryPlatform(profiles, opts);
|
|
313
|
+
return {
|
|
314
|
+
...lead,
|
|
315
|
+
socialProfiles: profiles,
|
|
316
|
+
primaryPlatform: primary.platform,
|
|
317
|
+
primaryPlatformUrl: primary.url,
|
|
318
|
+
primaryPlatformConfidence: primary.confidence,
|
|
319
|
+
primaryPlatformSelectionMethod: primary.selectionMethod,
|
|
320
|
+
primaryPlatformEvidence: primary.evidence,
|
|
321
|
+
primaryPlatformCheckedAt: primary.platform ? opts.now : '',
|
|
322
|
+
primaryPlatformNextRefreshAt: primary.platform ? addDays(opts.now, opts.refreshDays) : '',
|
|
323
|
+
};
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
function normalizeActivityProfile(value) {
|
|
327
|
+
const base = normalizeProfileUrl(value?.url);
|
|
328
|
+
if (!base) return null;
|
|
329
|
+
return {
|
|
330
|
+
...base,
|
|
331
|
+
...(value?.channelId ? { channelId: String(value.channelId) } : {}),
|
|
332
|
+
activityStatus: String(value?.activityStatus || 'presence_only'),
|
|
333
|
+
activityWindowDays: clampInteger(value?.activityWindowDays, 0, 3650, 0),
|
|
334
|
+
publicActivityCount: clampInteger(value?.publicActivityCount, 0, 1_000_000, 0),
|
|
335
|
+
lastPublicActivityAt: validDateTime(value?.lastPublicActivityAt),
|
|
336
|
+
checkedAt: validDateTime(value?.checkedAt),
|
|
337
|
+
nextRefreshAt: validDateTime(value?.nextRefreshAt),
|
|
338
|
+
activitySourceUrl: httpUrl(value?.activitySourceUrl),
|
|
339
|
+
evidence: compact(value?.evidence),
|
|
340
|
+
discoveredAt: validDateTime(value?.discoveredAt),
|
|
341
|
+
stale: Boolean(value?.stale),
|
|
342
|
+
lastError: compact(value?.lastError),
|
|
343
|
+
};
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
async function fetchJSON(url, opts) {
|
|
347
|
+
const response = await fetchResponse(url, opts);
|
|
348
|
+
return response.json();
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
async function fetchText(url, opts) {
|
|
352
|
+
const response = await fetchResponse(url, opts);
|
|
353
|
+
return response.text();
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
async function fetchResponse(url, opts) {
|
|
357
|
+
const response = await opts.fetchImpl(url, {
|
|
358
|
+
headers: opts.headers,
|
|
359
|
+
signal: AbortSignal.timeout(opts.timeoutMs),
|
|
360
|
+
});
|
|
361
|
+
if (!response.ok) throw new Error(`${new URL(url).hostname} returned HTTP ${response.status}`);
|
|
362
|
+
return response;
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
function normalizeOptions(options) {
|
|
366
|
+
const now = validDateTime(options.now) || new Date().toISOString();
|
|
367
|
+
return {
|
|
368
|
+
now,
|
|
369
|
+
activityWindowDays: clampInteger(options.activityWindowDays, 7, 365, 90),
|
|
370
|
+
refreshDays: clampInteger(options.refreshDays, 1, 30, 7),
|
|
371
|
+
concurrency: clampInteger(options.concurrency, 1, 16, 4),
|
|
372
|
+
timeoutMs: clampInteger(options.timeoutMs, 1_000, 60_000, 8_000),
|
|
373
|
+
limit: clampInteger(options.limit, 0, 1_000_000, 0),
|
|
374
|
+
offset: clampInteger(options.offset, 0, 1_000_000, 0),
|
|
375
|
+
maxProfiles: clampInteger(options.maxProfiles, 1, 1_000_000, 100_000),
|
|
376
|
+
platformOrder: normalizePlatformOrder(options.platformOrder),
|
|
377
|
+
githubToken: String(options.githubToken || process.env.GITHUB_TOKEN || '').trim(),
|
|
378
|
+
userAgent: String(options.userAgent || 'PublicLeadsPrimaryPlatform/0.1 (+https://github.com/Agent-Pattern-Labs/leads-rig)'),
|
|
379
|
+
fetchImpl: options.fetchImpl || globalThis.fetch,
|
|
380
|
+
cache: options.cache && typeof options.cache === 'object' ? options.cache : {},
|
|
381
|
+
refresh: Boolean(options.refresh),
|
|
382
|
+
publicActivityRequests: 0,
|
|
383
|
+
cacheHits: 0,
|
|
384
|
+
};
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
function profile(platform, url, username) {
|
|
388
|
+
return { platform, url, username: String(username || '').replace(/^@/, '') };
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
function profileMatchesContact(profile, contactName) {
|
|
392
|
+
const words = comparable(contactName).split(' ').filter((word) => word.length >= 2);
|
|
393
|
+
if (words.length < 2) return false;
|
|
394
|
+
const first = words[0];
|
|
395
|
+
const last = words.at(-1);
|
|
396
|
+
const username = comparable(profile.username).replaceAll(' ', '');
|
|
397
|
+
return username.includes(`${first}${last}`)
|
|
398
|
+
|| username.includes(`${last}${first}`)
|
|
399
|
+
|| username.includes(`${first[0]}${last}`)
|
|
400
|
+
|| username.includes(`${first}${last[0]}`);
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
function activityScore(profile, now) {
|
|
404
|
+
const countScore = Math.min(50, Math.log2(1 + profile.publicActivityCount) * 10);
|
|
405
|
+
const ageDays = profile.lastPublicActivityAt
|
|
406
|
+
? Math.max(0, (new Date(now).getTime() - new Date(profile.lastPublicActivityAt).getTime()) / 86_400_000)
|
|
407
|
+
: 365;
|
|
408
|
+
return countScore + Math.max(0, 60 - ageDays);
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
function normalizePlatformOrder(value) {
|
|
412
|
+
const values = Array.isArray(value) ? value : String(value || '').split(',');
|
|
413
|
+
const clean = values.map((item) => String(item || '').trim().toLowerCase()).filter(Boolean);
|
|
414
|
+
return [...new Set([...clean, ...DEFAULT_PLATFORM_ORDER])];
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
function platformRank(value) {
|
|
418
|
+
const index = DEFAULT_PLATFORM_ORDER.indexOf(value);
|
|
419
|
+
return index === -1 ? DEFAULT_PLATFORM_ORDER.length : index;
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
function profileKey(profile) {
|
|
423
|
+
return `${profile.platform}|${profile.url}`;
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
function leadKey(lead) {
|
|
427
|
+
return String(lead?.id || `${lead?.domain || ''}|${lead?.email || ''}|${lead?.contactName || ''}`).toLowerCase();
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
function profileRichness(profile) {
|
|
431
|
+
return Number(Boolean(profile.lastPublicActivityAt)) * 10 + Number(profile.publicActivityCount || 0) + Number(Boolean(profile.evidence));
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
function hasUsableMeasurement(profile) {
|
|
435
|
+
return profile && ['observed', 'observed_inactive'].includes(profile.activityStatus);
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
function isFresh(profile, now) {
|
|
439
|
+
if (!profile?.nextRefreshAt) return false;
|
|
440
|
+
return new Date(profile.nextRefreshAt) > new Date(now);
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
function validDate(value) {
|
|
444
|
+
const date = new Date(String(value || ''));
|
|
445
|
+
return Number.isNaN(date.getTime()) ? null : date;
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
function validDateTime(value) {
|
|
449
|
+
return validDate(value)?.toISOString() || '';
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
function addDays(value, days) {
|
|
453
|
+
return new Date(new Date(value).getTime() + days * 86_400_000).toISOString();
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
function httpUrl(value) {
|
|
457
|
+
try {
|
|
458
|
+
const url = new URL(String(value || ''));
|
|
459
|
+
return ['http:', 'https:'].includes(url.protocol) ? url.toString() : '';
|
|
460
|
+
} catch {
|
|
461
|
+
return '';
|
|
462
|
+
}
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
function compact(value) {
|
|
466
|
+
return String(value || '').replace(/\s+/g, ' ').trim();
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
function comparable(value) {
|
|
470
|
+
return compact(value).normalize('NFKD').replace(/[\u0300-\u036f]/g, '').toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim();
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
function displayPlatform(value) {
|
|
474
|
+
return value === 'x' ? 'X' : String(value || '').replace(/^./, (char) => char.toUpperCase());
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
function countBy(values, keyFn) {
|
|
478
|
+
const counts = {};
|
|
479
|
+
for (const value of values) {
|
|
480
|
+
const key = keyFn(value);
|
|
481
|
+
counts[key] = (counts[key] || 0) + 1;
|
|
482
|
+
}
|
|
483
|
+
return Object.fromEntries(Object.entries(counts).sort((left, right) => right[1] - left[1] || left[0].localeCompare(right[0])));
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
function clamp(value, min, max) {
|
|
487
|
+
return Math.max(min, Math.min(max, value));
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
function clampInteger(value, min, max, fallback) {
|
|
491
|
+
const number = Number(value);
|
|
492
|
+
return Number.isFinite(number) ? Math.round(clamp(number, min, max)) : fallback;
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
function decodeURIComponentSafe(value) {
|
|
496
|
+
try {
|
|
497
|
+
return decodeURIComponent(value);
|
|
498
|
+
} catch {
|
|
499
|
+
return value;
|
|
500
|
+
}
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
async function mapLimit(items, limit, worker) {
|
|
504
|
+
let next = 0;
|
|
505
|
+
async function run() {
|
|
506
|
+
while (next < items.length) {
|
|
507
|
+
const index = next++;
|
|
508
|
+
await worker(items[index], index);
|
|
509
|
+
}
|
|
510
|
+
}
|
|
511
|
+
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, run));
|
|
512
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@agent-pattern-labs/leads-rig",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.9",
|
|
4
4
|
"description": "Agentic public-web lead discovery harness with portable ingest artifacts",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"author": "Razroo",
|
|
@@ -30,6 +30,7 @@
|
|
|
30
30
|
"lead:validate": "node bin/lead-harness.mjs validate",
|
|
31
31
|
"lead:manifest": "node bin/lead-harness.mjs manifest",
|
|
32
32
|
"lead:ingest": "node bin/lead-harness.mjs ingest",
|
|
33
|
+
"lead:enrich-platform": "node bin/lead-harness.mjs enrich:platform",
|
|
33
34
|
"batch": "batch/batch-runner.sh",
|
|
34
35
|
"trace:list": "node bin/lead-harness.mjs trace:list",
|
|
35
36
|
"trace:stats": "node bin/lead-harness.mjs trace:stats",
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
import { existsSync, readFileSync } from 'node:fs';
|
|
4
|
+
import {
|
|
5
|
+
parseArgs,
|
|
6
|
+
readLeadArtifact,
|
|
7
|
+
validatePayload,
|
|
8
|
+
writeJson,
|
|
9
|
+
} from '../lib/leadharness-leads.mjs';
|
|
10
|
+
import { enrichPrimaryPlatforms } from '../lib/leadharness-primary-platform.mjs';
|
|
11
|
+
|
|
12
|
+
const USAGE = `public-leads enrich:platform -- measure public platform activity and select a primary profile
|
|
13
|
+
|
|
14
|
+
Usage:
|
|
15
|
+
public-leads enrich:platform --input data/leads.json
|
|
16
|
+
[--out data/leads-primary-platform.json]
|
|
17
|
+
[--report data/leads-primary-platform-report.json]
|
|
18
|
+
[--checkpoint data/leads-primary-platform.checkpoint.json]
|
|
19
|
+
[--activity-window-days 90] [--refresh-days 7]
|
|
20
|
+
[--platform-order linkedin,github,x,bluesky,youtube,instagram,facebook]
|
|
21
|
+
[--limit 0] [--offset 0] [--concurrency 4] [--timeout-ms 8000]
|
|
22
|
+
[--github-token <token>] [--refresh]
|
|
23
|
+
|
|
24
|
+
Only permitted public activity endpoints are requested. LinkedIn, X, Instagram,
|
|
25
|
+
and Facebook are presence-only fallbacks and are never crawled by this command.
|
|
26
|
+
GitHub tokens are optional and only increase the public REST API rate limit.
|
|
27
|
+
`;
|
|
28
|
+
|
|
29
|
+
main().catch((error) => {
|
|
30
|
+
console.error(error instanceof Error ? error.stack || error.message : String(error));
|
|
31
|
+
process.exit(1);
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
async function main() {
|
|
35
|
+
const opts = parseArgs(process.argv.slice(2));
|
|
36
|
+
if (opts.help) {
|
|
37
|
+
console.log(USAGE);
|
|
38
|
+
return;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
const input = String(opts.input || opts._?.[0] || '').trim();
|
|
42
|
+
if (!input) throw new Error('--input is required');
|
|
43
|
+
if (!existsSync(input)) throw new Error(`input not found: ${input}`);
|
|
44
|
+
const stem = input.replace(/\.json$/i, '');
|
|
45
|
+
const out = String(opts.out || `${stem}-primary-platform.json`);
|
|
46
|
+
const reportOut = String(opts.report || `${stem}-primary-platform-report.json`);
|
|
47
|
+
const checkpoint = String(opts.checkpoint || `${stem}-primary-platform.checkpoint.json`);
|
|
48
|
+
const cache = loadCache(checkpoint);
|
|
49
|
+
const payload = readLeadArtifact(input);
|
|
50
|
+
|
|
51
|
+
const enriched = await enrichPrimaryPlatforms(payload, {
|
|
52
|
+
cache,
|
|
53
|
+
activityWindowDays: opts.activityWindowDays,
|
|
54
|
+
refreshDays: opts.refreshDays,
|
|
55
|
+
platformOrder: opts.platformOrder,
|
|
56
|
+
limit: opts.limit,
|
|
57
|
+
offset: opts.offset,
|
|
58
|
+
concurrency: opts.concurrency,
|
|
59
|
+
timeoutMs: opts.timeoutMs,
|
|
60
|
+
maxProfiles: opts.maxProfiles,
|
|
61
|
+
githubToken: opts.githubToken,
|
|
62
|
+
refresh: Boolean(opts.refresh),
|
|
63
|
+
});
|
|
64
|
+
const validation = validatePayload(enriched.payload, { allowEmpty: true });
|
|
65
|
+
enriched.report.validation = validation.summary;
|
|
66
|
+
|
|
67
|
+
writeJson(out, enriched.payload);
|
|
68
|
+
writeJson(reportOut, enriched.report);
|
|
69
|
+
writeJson(checkpoint, enriched.cache);
|
|
70
|
+
if (!validation.ok) {
|
|
71
|
+
const errors = validation.issues.filter((issue) => issue.severity === 'error');
|
|
72
|
+
throw new Error(errors.map((issue) => `${issue.path}: ${issue.message}`).join('\n'));
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
console.log(JSON.stringify({
|
|
76
|
+
status: 'COMPLETED',
|
|
77
|
+
...enriched.report,
|
|
78
|
+
out,
|
|
79
|
+
report: reportOut,
|
|
80
|
+
checkpoint,
|
|
81
|
+
}, null, 2));
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function loadCache(path) {
|
|
85
|
+
if (!existsSync(path)) return {};
|
|
86
|
+
try {
|
|
87
|
+
const value = JSON.parse(readFileSync(path, 'utf8'));
|
|
88
|
+
return value && typeof value === 'object' ? value : {};
|
|
89
|
+
} catch {
|
|
90
|
+
return {};
|
|
91
|
+
}
|
|
92
|
+
}
|
package/templates/contracts.json
CHANGED
|
@@ -30,8 +30,24 @@
|
|
|
30
30
|
},
|
|
31
31
|
{ "name": "confidence", "type": "integer", "required": true, "min": 0, "max": 100 },
|
|
32
32
|
{ "name": "warnings", "type": "json" },
|
|
33
|
+
{ "name": "linkedinUrl", "type": "url" },
|
|
33
34
|
{ "name": "phone", "type": "string" },
|
|
35
|
+
{ "name": "address", "type": "string" },
|
|
36
|
+
{ "name": "streetAddress", "type": "string" },
|
|
37
|
+
{ "name": "location", "type": "string" },
|
|
38
|
+
{ "name": "city", "type": "string" },
|
|
39
|
+
{ "name": "region", "type": "string" },
|
|
40
|
+
{ "name": "postalCode", "type": "string" },
|
|
41
|
+
{ "name": "country", "type": "string" },
|
|
34
42
|
{ "name": "socialUrls", "type": "json" },
|
|
43
|
+
{ "name": "socialProfiles", "type": "json" },
|
|
44
|
+
{ "name": "primaryPlatform", "type": "string" },
|
|
45
|
+
{ "name": "primaryPlatformUrl", "type": "url" },
|
|
46
|
+
{ "name": "primaryPlatformConfidence", "type": "integer", "min": 0, "max": 100 },
|
|
47
|
+
{ "name": "primaryPlatformSelectionMethod", "type": "string" },
|
|
48
|
+
{ "name": "primaryPlatformEvidence", "type": "string" },
|
|
49
|
+
{ "name": "primaryPlatformCheckedAt", "type": "datetime" },
|
|
50
|
+
{ "name": "primaryPlatformNextRefreshAt", "type": "datetime" },
|
|
35
51
|
{ "name": "contactUrls", "type": "json" },
|
|
36
52
|
{ "name": "foundAt", "type": "datetime" }
|
|
37
53
|
],
|
|
@@ -74,16 +74,52 @@
|
|
|
74
74
|
"items": { "type": "string" },
|
|
75
75
|
"default": []
|
|
76
76
|
},
|
|
77
|
+
"linkedinUrl": {
|
|
78
|
+
"type": "string",
|
|
79
|
+
"description": "Optional canonical public LinkedIn profile or organization URL."
|
|
80
|
+
},
|
|
77
81
|
"phone": {
|
|
78
82
|
"type": "string",
|
|
79
83
|
"description": "Optional normalized phone number found near the public contact evidence."
|
|
80
84
|
},
|
|
85
|
+
"address": {
|
|
86
|
+
"type": "string",
|
|
87
|
+
"description": "Optional complete public business address."
|
|
88
|
+
},
|
|
89
|
+
"streetAddress": {
|
|
90
|
+
"type": "string",
|
|
91
|
+
"description": "Optional street-address component."
|
|
92
|
+
},
|
|
93
|
+
"location": {
|
|
94
|
+
"type": "string",
|
|
95
|
+
"description": "Optional display location for the lead."
|
|
96
|
+
},
|
|
97
|
+
"city": { "type": "string" },
|
|
98
|
+
"region": { "type": "string" },
|
|
99
|
+
"postalCode": { "type": "string" },
|
|
100
|
+
"country": { "type": "string" },
|
|
81
101
|
"socialUrls": {
|
|
82
102
|
"type": "array",
|
|
83
103
|
"items": { "type": "string" },
|
|
84
104
|
"default": [],
|
|
85
105
|
"description": "Optional public social/profile URLs found on the same source page."
|
|
86
106
|
},
|
|
107
|
+
"socialProfiles": {
|
|
108
|
+
"type": "array",
|
|
109
|
+
"items": { "$ref": "#/$defs/socialProfile" },
|
|
110
|
+
"default": [],
|
|
111
|
+
"description": "Public person-matched profiles with refreshable activity measurements."
|
|
112
|
+
},
|
|
113
|
+
"primaryPlatform": { "type": "string" },
|
|
114
|
+
"primaryPlatformUrl": { "type": "string" },
|
|
115
|
+
"primaryPlatformConfidence": { "type": "integer", "minimum": 0, "maximum": 100 },
|
|
116
|
+
"primaryPlatformSelectionMethod": {
|
|
117
|
+
"type": "string",
|
|
118
|
+
"enum": ["", "observed_public_activity", "fallback_presence_priority"]
|
|
119
|
+
},
|
|
120
|
+
"primaryPlatformEvidence": { "type": "string" },
|
|
121
|
+
"primaryPlatformCheckedAt": { "type": "string" },
|
|
122
|
+
"primaryPlatformNextRefreshAt": { "type": "string" },
|
|
87
123
|
"contactUrls": {
|
|
88
124
|
"type": "array",
|
|
89
125
|
"items": { "type": "string" },
|
|
@@ -96,6 +132,28 @@
|
|
|
96
132
|
}
|
|
97
133
|
}
|
|
98
134
|
},
|
|
135
|
+
"socialProfile": {
|
|
136
|
+
"type": "object",
|
|
137
|
+
"additionalProperties": true,
|
|
138
|
+
"required": ["platform", "url", "activityStatus"],
|
|
139
|
+
"properties": {
|
|
140
|
+
"platform": { "type": "string" },
|
|
141
|
+
"url": { "type": "string" },
|
|
142
|
+
"username": { "type": "string" },
|
|
143
|
+
"activityStatus": {
|
|
144
|
+
"type": "string",
|
|
145
|
+
"enum": ["presence_only", "observed", "observed_inactive", "unavailable", "error"]
|
|
146
|
+
},
|
|
147
|
+
"activityWindowDays": { "type": "integer", "minimum": 0 },
|
|
148
|
+
"publicActivityCount": { "type": "integer", "minimum": 0 },
|
|
149
|
+
"lastPublicActivityAt": { "type": "string" },
|
|
150
|
+
"checkedAt": { "type": "string" },
|
|
151
|
+
"nextRefreshAt": { "type": "string" },
|
|
152
|
+
"activitySourceUrl": { "type": "string" },
|
|
153
|
+
"evidence": { "type": "string" },
|
|
154
|
+
"stale": { "type": "boolean" }
|
|
155
|
+
}
|
|
156
|
+
},
|
|
99
157
|
"result": {
|
|
100
158
|
"type": "object",
|
|
101
159
|
"additionalProperties": true,
|