@nebulacomponents/citable 1.17.0 → 1.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +22 -0
- package/dist/universal/.agents/skills/citable/SKILL.md +20 -1
- package/dist/universal/.agents/skills/citable/VERSION +1 -1
- package/dist/universal/.agents/skills/citable/manifest.json +6 -5
- package/dist/universal/.agents/skills/citable/schemas/sow.schema.json +92 -0
- package/dist/universal/.claude/agents/citable/manifest.json +2 -2
- package/dist/universal/.claude/skills/citable/SKILL.md +20 -1
- package/dist/universal/.claude/skills/citable/VERSION +1 -1
- package/dist/universal/.claude/skills/citable/manifest.json +6 -5
- package/dist/universal/.claude/skills/citable/schemas/sow.schema.json +92 -0
- package/dist/universal/.cursor/skills/citable/SKILL.md +20 -1
- package/dist/universal/.cursor/skills/citable/VERSION +1 -1
- package/dist/universal/.cursor/skills/citable/manifest.json +6 -5
- package/dist/universal/.cursor/skills/citable/schemas/sow.schema.json +92 -0
- package/dist/universal/.gemini/skills/citable/SKILL.md +20 -1
- package/dist/universal/.gemini/skills/citable/VERSION +1 -1
- package/dist/universal/.gemini/skills/citable/manifest.json +6 -5
- package/dist/universal/.gemini/skills/citable/schemas/sow.schema.json +92 -0
- package/dist/universal/.github/skills/citable/SKILL.md +20 -1
- package/dist/universal/.github/skills/citable/VERSION +1 -1
- package/dist/universal/.github/skills/citable/manifest.json +6 -5
- package/dist/universal/.github/skills/citable/schemas/sow.schema.json +92 -0
- package/dist/universal/.kiro/skills/citable/SKILL.md +20 -1
- package/dist/universal/.kiro/skills/citable/VERSION +1 -1
- package/dist/universal/.kiro/skills/citable/manifest.json +6 -5
- package/dist/universal/.kiro/skills/citable/schemas/sow.schema.json +92 -0
- package/dist/universal/.opencode/skills/citable/SKILL.md +20 -1
- package/dist/universal/.opencode/skills/citable/VERSION +1 -1
- package/dist/universal/.opencode/skills/citable/manifest.json +6 -5
- package/dist/universal/.opencode/skills/citable/schemas/sow.schema.json +92 -0
- package/dist/universal/.pi/agent/skills/citable/SKILL.md +20 -1
- package/dist/universal/.pi/agent/skills/citable/VERSION +1 -1
- package/dist/universal/.pi/agent/skills/citable/manifest.json +6 -5
- package/dist/universal/.pi/agent/skills/citable/schemas/sow.schema.json +92 -0
- package/dist/universal/.qoder/skills/citable/SKILL.md +20 -1
- package/dist/universal/.qoder/skills/citable/VERSION +1 -1
- package/dist/universal/.qoder/skills/citable/manifest.json +6 -5
- package/dist/universal/.qoder/skills/citable/schemas/sow.schema.json +92 -0
- package/dist/universal/.rovodev/skills/citable/SKILL.md +20 -1
- package/dist/universal/.rovodev/skills/citable/VERSION +1 -1
- package/dist/universal/.rovodev/skills/citable/manifest.json +6 -5
- package/dist/universal/.rovodev/skills/citable/schemas/sow.schema.json +92 -0
- package/dist/universal/.trae/skills/citable/SKILL.md +20 -1
- package/dist/universal/.trae/skills/citable/VERSION +1 -1
- package/dist/universal/.trae/skills/citable/manifest.json +6 -5
- package/dist/universal/.trae/skills/citable/schemas/sow.schema.json +92 -0
- package/dist/universal/.trae-cn/skills/citable/SKILL.md +20 -1
- package/dist/universal/.trae-cn/skills/citable/VERSION +1 -1
- package/dist/universal/.trae-cn/skills/citable/manifest.json +6 -5
- package/dist/universal/.trae-cn/skills/citable/schemas/sow.schema.json +92 -0
- package/dist/universal/manifest.json +76 -64
- package/package.json +1 -1
- package/schemas/sow.schema.json +92 -0
- package/skill/SKILL.md +20 -1
- package/src/analysis/behavioral.js +126 -0
- package/src/analysis/croAudit.js +302 -0
- package/src/analysis/croRoadmap.js +121 -0
- package/src/analysis/eeat.js +300 -0
- package/src/analysis/funnelAnalysis.js +171 -0
- package/src/analysis/iceMatrix.js +205 -0
- package/src/analysis/offpage.js +228 -0
- package/src/analysis/readiness.js +253 -0
- package/src/analysis/schemaRecommendations.js +173 -0
- package/src/analysis/strategicRoadmap.js +148 -0
- package/src/cli/index.js +149 -3
- package/src/commands/answerEngineReadiness.js +48 -0
- package/src/commands/auditBacklinks.js +79 -0
- package/src/commands/croBacklog.js +182 -0
- package/src/commands/croSuite.js +156 -0
- package/src/commands/executive.js +6 -0
- package/src/commands/executiveCro.js +30 -0
- package/src/commands/executiveSearch.js +28 -0
- package/src/commands/inspectCro.js +7 -0
- package/src/commands/inspectEeat.js +53 -0
- package/src/commands/prioritize.js +42 -6
- package/src/commands/roadmapCmd.js +69 -0
- package/src/commands/schemaCmd.js +10 -2
- package/src/commands/sowCmd.js +67 -0
- package/src/commands/sweep.js +357 -0
- package/src/reporting/executiveCroReport.js +731 -0
- package/src/reporting/executiveExport.js +62 -0
- package/src/reporting/executiveSearchReport.js +704 -0
- package/src/sow/admissibilityGate.js +194 -0
- package/src/sow/generateSow.js +762 -0
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
import { URL } from 'node:url';
|
|
2
|
+
|
|
3
|
+
const SPAM_TLDS = new Set([
|
|
4
|
+
'top', 'work', 'gdn', 'click', 'monster', 'buzz', 'cfd', 'sbs',
|
|
5
|
+
'rest', 'cam', 'fit', 'surf', 'stream', 'tk', 'ml', 'ga', 'cf',
|
|
6
|
+
]);
|
|
7
|
+
|
|
8
|
+
const COMMERCIAL_SPAM_KEYWORDS = /\b(casino|viagra|cialis|payday\s*loans?|replica|cheap\s*essay|free\s*download|warez|crack|hack|gambling|betting|slots|crypto\s*airdrop)\b/i;
|
|
9
|
+
|
|
10
|
+
function extractHostname(urlString) {
|
|
11
|
+
try {
|
|
12
|
+
return new URL(urlString).hostname.toLowerCase();
|
|
13
|
+
} catch {
|
|
14
|
+
return null;
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
function extractRootDomain(hostname) {
|
|
19
|
+
if (!hostname) return null;
|
|
20
|
+
const parts = hostname.split('.');
|
|
21
|
+
if (parts.length <= 2) return hostname;
|
|
22
|
+
return parts.slice(-2).join('.');
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function extractTld(hostname) {
|
|
26
|
+
if (!hostname) return null;
|
|
27
|
+
const parts = hostname.split('.');
|
|
28
|
+
return parts.at(-1) || null;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Audit an inbound backlink profile for authority distribution and toxic domain risks.
|
|
33
|
+
*/
|
|
34
|
+
export function auditBacklinkProfile(backlinks, { targetDomain = null } = {}) {
|
|
35
|
+
const links = Array.isArray(backlinks) ? backlinks : (backlinks?.backlinks || []);
|
|
36
|
+
const normalizedTarget = targetDomain ? targetDomain.toLowerCase().replace(/^www\./, '') : null;
|
|
37
|
+
|
|
38
|
+
const referringDomains = new Map();
|
|
39
|
+
const cClassClusters = new Map();
|
|
40
|
+
let dofollowCount = 0;
|
|
41
|
+
let nofollowCount = 0;
|
|
42
|
+
let ugcCount = 0;
|
|
43
|
+
let sponsoredCount = 0;
|
|
44
|
+
let homepageLinksCount = 0;
|
|
45
|
+
let deepLinksCount = 0;
|
|
46
|
+
|
|
47
|
+
const anchorTypes = {
|
|
48
|
+
branded: 0,
|
|
49
|
+
naked_url: 0,
|
|
50
|
+
generic: 0,
|
|
51
|
+
exact_match_commercial: 0,
|
|
52
|
+
other: 0,
|
|
53
|
+
};
|
|
54
|
+
|
|
55
|
+
const toxicDomains = [];
|
|
56
|
+
const GENERIC_ANCHORS = /^(?:click here|here|website|link|read more|learn more|source|this site|visit|more info)$/i;
|
|
57
|
+
|
|
58
|
+
// 1. Process each backlink
|
|
59
|
+
for (const link of links) {
|
|
60
|
+
const srcHost = extractHostname(link.source_url);
|
|
61
|
+
if (!srcHost) continue;
|
|
62
|
+
const rootDomain = extractRootDomain(srcHost) || srcHost;
|
|
63
|
+
const tld = extractTld(srcHost);
|
|
64
|
+
const anchor = (link.anchor_text || '').trim();
|
|
65
|
+
const rel = (link.rel || '').toLowerCase();
|
|
66
|
+
const isNofollow = rel.includes('nofollow');
|
|
67
|
+
const isUgc = rel.includes('ugc');
|
|
68
|
+
const isSponsored = rel.includes('sponsored');
|
|
69
|
+
|
|
70
|
+
if (isNofollow) nofollowCount++;
|
|
71
|
+
else if (isUgc) ugcCount++;
|
|
72
|
+
else if (isSponsored) sponsoredCount++;
|
|
73
|
+
else dofollowCount++;
|
|
74
|
+
|
|
75
|
+
// Deep link vs homepage
|
|
76
|
+
const targetPath = link.target_url ? (() => { try { return new URL(link.target_url).pathname; } catch { return '/'; } })() : '/';
|
|
77
|
+
if (targetPath === '/' || targetPath === '') homepageLinksCount++;
|
|
78
|
+
else deepLinksCount++;
|
|
79
|
+
|
|
80
|
+
// Anchor categorization
|
|
81
|
+
if (normalizedTarget && (anchor.toLowerCase().includes(normalizedTarget) || normalizedTarget.includes(anchor.toLowerCase()))) {
|
|
82
|
+
anchorTypes.branded++;
|
|
83
|
+
} else if (/^https?:\/\//i.test(anchor) || anchor.includes('www.') || anchor.includes('.com') || anchor.includes('.org')) {
|
|
84
|
+
anchorTypes.naked_url++;
|
|
85
|
+
} else if (GENERIC_ANCHORS.test(anchor)) {
|
|
86
|
+
anchorTypes.generic++;
|
|
87
|
+
} else if (COMMERCIAL_SPAM_KEYWORDS.test(anchor) || /\b(buy|best|cheap|discount|order)\b/i.test(anchor)) {
|
|
88
|
+
anchorTypes.exact_match_commercial++;
|
|
89
|
+
} else {
|
|
90
|
+
anchorTypes.other++;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// C-class IP clustering
|
|
94
|
+
if (link.source_ip) {
|
|
95
|
+
const parts = link.source_ip.split('.');
|
|
96
|
+
if (parts.length === 4) {
|
|
97
|
+
const cClass = parts.slice(0, 3).join('.');
|
|
98
|
+
const cluster = cClassClusters.get(cClass) || new Set();
|
|
99
|
+
cluster.add(rootDomain);
|
|
100
|
+
cClassClusters.set(cClass, cluster);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
// Domain tracking
|
|
105
|
+
const domainData = referringDomains.get(rootDomain) || {
|
|
106
|
+
domain: rootDomain,
|
|
107
|
+
sample_source_url: link.source_url,
|
|
108
|
+
total_links: 0,
|
|
109
|
+
anchors: [],
|
|
110
|
+
tld,
|
|
111
|
+
ip: link.source_ip || null,
|
|
112
|
+
dofollow: false,
|
|
113
|
+
};
|
|
114
|
+
domainData.total_links++;
|
|
115
|
+
if (!isNofollow && !isSponsored) domainData.dofollow = true;
|
|
116
|
+
if (anchor && !domainData.anchors.includes(anchor)) domainData.anchors.push(anchor);
|
|
117
|
+
referringDomains.set(rootDomain, domainData);
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
// 2. Toxic domain risk analysis per domain
|
|
121
|
+
for (const [domain, data] of referringDomains.entries()) {
|
|
122
|
+
const reasons = [];
|
|
123
|
+
let riskTier = 'clean';
|
|
124
|
+
|
|
125
|
+
// Spam TLD rule
|
|
126
|
+
if (data.tld && SPAM_TLDS.has(data.tld)) {
|
|
127
|
+
reasons.push(`Known high-abuse spam TLD (.${data.tld})`);
|
|
128
|
+
riskTier = 'high';
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
// Spam anchor rule
|
|
132
|
+
const hasSpamAnchor = data.anchors.some((a) => COMMERCIAL_SPAM_KEYWORDS.test(a));
|
|
133
|
+
if (hasSpamAnchor) {
|
|
134
|
+
reasons.push('Anchor text matches known high-risk commercial spam patterns');
|
|
135
|
+
riskTier = 'critical';
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
// PBN IP clustering rule (3+ distinct domains on the exact same /24 C-block)
|
|
139
|
+
if (data.ip) {
|
|
140
|
+
const parts = data.ip.split('.');
|
|
141
|
+
if (parts.length === 4) {
|
|
142
|
+
const cClass = parts.slice(0, 3).join('.');
|
|
143
|
+
const cluster = cClassClusters.get(cClass);
|
|
144
|
+
if (cluster && cluster.size >= 3) {
|
|
145
|
+
reasons.push(`Part of hosting cluster on ${cClass}.0/24 with ${cluster.size} domains (PBN footprint)`);
|
|
146
|
+
riskTier = 'critical';
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
// Excessive sitewide link density without nofollow
|
|
152
|
+
if (data.total_links > 50 && data.dofollow && (data.tld && SPAM_TLDS.has(data.tld))) {
|
|
153
|
+
reasons.push(`Sitewide link farm pattern (${data.total_links} links without rel="nofollow")`);
|
|
154
|
+
riskTier = 'critical';
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
if (riskTier !== 'clean') {
|
|
158
|
+
toxicDomains.push({
|
|
159
|
+
domain,
|
|
160
|
+
risk_tier: riskTier,
|
|
161
|
+
reasons,
|
|
162
|
+
sample_url: data.sample_source_url,
|
|
163
|
+
anchors: data.anchors.slice(0, 5),
|
|
164
|
+
total_links: data.total_links,
|
|
165
|
+
});
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
// Sort toxic domains by severity (critical first)
|
|
170
|
+
toxicDomains.sort((a, b) => (a.risk_tier === 'critical' ? -1 : 1));
|
|
171
|
+
|
|
172
|
+
const totalLinks = links.length;
|
|
173
|
+
const totalDomains = referringDomains.size;
|
|
174
|
+
const commercialAnchorPct = totalLinks > 0 ? Math.round((anchorTypes.exact_match_commercial / totalLinks) * 100) : 0;
|
|
175
|
+
const brandedAnchorPct = totalLinks > 0 ? Math.round((anchorTypes.branded / totalLinks) * 100) : 0;
|
|
176
|
+
const deepLinkPct = totalLinks > 0 ? Math.round((deepLinksCount / totalLinks) * 100) : 0;
|
|
177
|
+
|
|
178
|
+
// Authority health assessment
|
|
179
|
+
let profileHealth = 'natural';
|
|
180
|
+
if (toxicDomains.some((d) => d.risk_tier === 'critical') || commercialAnchorPct > 30) {
|
|
181
|
+
profileHealth = 'high_risk_footprint';
|
|
182
|
+
} else if (toxicDomains.length > 0 || commercialAnchorPct > 15) {
|
|
183
|
+
profileHealth = 'moderate_risk';
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
// Generate GSC-compatible disavow lines
|
|
187
|
+
const disavowLines = [
|
|
188
|
+
`# Citable Google Disavow Candidate Export`,
|
|
189
|
+
`# Generated at: ${new Date().toISOString()}`,
|
|
190
|
+
`# Flagged domains: ${toxicDomains.length} (critical: ${toxicDomains.filter((d) => d.risk_tier === 'critical').length})`,
|
|
191
|
+
``,
|
|
192
|
+
];
|
|
193
|
+
for (const td of toxicDomains) {
|
|
194
|
+
disavowLines.push(`# Risk: ${td.risk_tier.toUpperCase()} — ${td.reasons.join('; ')}`);
|
|
195
|
+
disavowLines.push(`domain:${td.domain}`);
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
return {
|
|
199
|
+
fact_status: 'observable_risk_indicators',
|
|
200
|
+
target_domain: targetDomain,
|
|
201
|
+
profile_health: profileHealth,
|
|
202
|
+
summary: {
|
|
203
|
+
total_backlinks: totalLinks,
|
|
204
|
+
total_referring_domains: totalDomains,
|
|
205
|
+
toxic_domains_count: toxicDomains.length,
|
|
206
|
+
critical_risk_domains: toxicDomains.filter((d) => d.risk_tier === 'critical').length,
|
|
207
|
+
high_risk_domains: toxicDomains.filter((d) => d.risk_tier === 'high').length,
|
|
208
|
+
dofollow_count: dofollowCount,
|
|
209
|
+
nofollow_count: nofollowCount,
|
|
210
|
+
ugc_count: ugcCount,
|
|
211
|
+
sponsored_count: sponsoredCount,
|
|
212
|
+
deep_link_ratio_pct: deepLinkPct,
|
|
213
|
+
},
|
|
214
|
+
anchor_profile: {
|
|
215
|
+
branded_pct: brandedAnchorPct,
|
|
216
|
+
commercial_exact_match_pct: commercialAnchorPct,
|
|
217
|
+
naked_url_count: anchorTypes.naked_url,
|
|
218
|
+
generic_count: anchorTypes.generic,
|
|
219
|
+
over_optimization_risk: commercialAnchorPct > 25 ? 'elevated' : 'normal',
|
|
220
|
+
},
|
|
221
|
+
toxic_domains: toxicDomains,
|
|
222
|
+
disavow_export: disavowLines.join('\n'),
|
|
223
|
+
limitations: [
|
|
224
|
+
'Search engines do not disclose algorithmic penalty thresholds; toxic classifications reflect observable risk patterns.',
|
|
225
|
+
'Manual verification is required before submitting disavow files to Google Search Console.',
|
|
226
|
+
],
|
|
227
|
+
};
|
|
228
|
+
}
|
|
@@ -0,0 +1,253 @@
|
|
|
1
|
+
import { parse as parseHtml } from 'node-html-parser';
|
|
2
|
+
import { registryPageFor, safePath } from '../detectors/framework.js';
|
|
3
|
+
|
|
4
|
+
const QUESTION_RX = /^(?:how|what|why|when|where|who|can|is|does|which|should)\b|\?$/i;
|
|
5
|
+
const COPULA_RX = /\b(?:is|are|refers\s+to|denotes|represents|means|provides|enables)\b/i;
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Evaluates a page's extraction, retrieval, and citation readiness across
|
|
9
|
+
* Perplexity, Bing Copilot, and ChatGPT / SearchGPT.
|
|
10
|
+
*/
|
|
11
|
+
export function evaluateAnswerEngineReadiness(page, ctx = {}) {
|
|
12
|
+
const html = page.html || '';
|
|
13
|
+
const root = parseHtml(html);
|
|
14
|
+
const text = (page.text || root.textContent || '').replace(/\s+/g, ' ').trim();
|
|
15
|
+
const headings = page.headings || [];
|
|
16
|
+
const paragraphs = page.paragraphs || root.querySelectorAll('p').map((p) => p.text.trim()).filter(Boolean);
|
|
17
|
+
const jsonLd = page.jsonLd || [];
|
|
18
|
+
const tablesCount = (page.tables ?? root.querySelectorAll('table').length);
|
|
19
|
+
const listsCount = (page.orderedLists ?? root.querySelectorAll('ol').length) + root.querySelectorAll('ul').length;
|
|
20
|
+
const wordCount = page.rawVisibleWordCount || page.wordCount || text.split(/\s+/).filter(Boolean).length;
|
|
21
|
+
|
|
22
|
+
// Crawler directives check (from robots Directives or headers)
|
|
23
|
+
const robotsDirectives = Array.from(page.robotsDirectives || []);
|
|
24
|
+
const isNoindex = robotsDirectives.some((d) => d.includes('noindex'));
|
|
25
|
+
|
|
26
|
+
// -----------------------------------------------------------------
|
|
27
|
+
// 1. PERPLEXITY READINESS
|
|
28
|
+
// -----------------------------------------------------------------
|
|
29
|
+
const perplexityChecks = [];
|
|
30
|
+
let perplexityScore = 0;
|
|
31
|
+
|
|
32
|
+
// Check 1: Crawler policy (PerplexityBot)
|
|
33
|
+
const perplexityBotBlocked = robotsDirectives.some((d) => d.includes('perplexitybot: noindex') || d.includes('perplexitybot: none'));
|
|
34
|
+
if (!perplexityBotBlocked && !isNoindex) {
|
|
35
|
+
perplexityScore += 25;
|
|
36
|
+
perplexityChecks.push({ name: 'Crawler Policy', passed: true, note: 'PerplexityBot allowed' });
|
|
37
|
+
} else {
|
|
38
|
+
perplexityChecks.push({ name: 'Crawler Policy', passed: false, note: 'Blocked by robots directive' });
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
// Check 2: Direct Definitional Lead (Copula in first paragraph)
|
|
42
|
+
const firstParagraph = paragraphs[0] || '';
|
|
43
|
+
const hasCopularLead = COPULA_RX.test(firstParagraph.slice(0, 160));
|
|
44
|
+
if (hasCopularLead) {
|
|
45
|
+
perplexityScore += 25;
|
|
46
|
+
perplexityChecks.push({ name: 'Definitional Lead', passed: true, note: 'Concise copular definition in opening passage' });
|
|
47
|
+
} else {
|
|
48
|
+
perplexityChecks.push({ name: 'Definitional Lead', passed: false, note: 'Opening passage lacks concise definitional copula ("X is Y")' });
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
// Check 3: Structured question headings
|
|
52
|
+
const questionHeadings = headings.filter((h) => QUESTION_RX.test(h.text.trim()));
|
|
53
|
+
if (questionHeadings.length >= 2) {
|
|
54
|
+
perplexityScore += 25;
|
|
55
|
+
perplexityChecks.push({ name: 'Question Density', passed: true, note: `${questionHeadings.length} interrogative headings found` });
|
|
56
|
+
} else if (questionHeadings.length === 1) {
|
|
57
|
+
perplexityScore += 15;
|
|
58
|
+
perplexityChecks.push({ name: 'Question Density', passed: true, note: '1 interrogative heading found' });
|
|
59
|
+
} else {
|
|
60
|
+
perplexityChecks.push({ name: 'Question Density', passed: false, note: 'No interrogative headings found (Q&A format optimizes extraction)' });
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
// Check 4: Structured data & tables for citation synthesis
|
|
64
|
+
if (tablesCount >= 1 || listsCount >= 2) {
|
|
65
|
+
perplexityScore += 25;
|
|
66
|
+
perplexityChecks.push({ name: 'Tabular / List Data', passed: true, note: `${tablesCount} table(s), ${listsCount} list(s) for direct citation` });
|
|
67
|
+
} else {
|
|
68
|
+
perplexityScore += 10;
|
|
69
|
+
perplexityChecks.push({ name: 'Tabular / List Data', passed: false, note: 'Limited tables or structured lists for bulleted answers' });
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
// -----------------------------------------------------------------
|
|
73
|
+
// 2. BING COPILOT READINESS
|
|
74
|
+
// -----------------------------------------------------------------
|
|
75
|
+
const bingChecks = [];
|
|
76
|
+
let bingScore = 0;
|
|
77
|
+
|
|
78
|
+
// Check 1: Bingbot allowed
|
|
79
|
+
const bingbotBlocked = robotsDirectives.some((d) => d.includes('bingbot: noindex'));
|
|
80
|
+
if (!bingbotBlocked && !isNoindex) {
|
|
81
|
+
bingScore += 25;
|
|
82
|
+
bingChecks.push({ name: 'Crawler Policy', passed: true, note: 'Bingbot allowed' });
|
|
83
|
+
} else {
|
|
84
|
+
bingChecks.push({ name: 'Crawler Policy', passed: false, note: 'Bingbot blocked' });
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
// Check 2: IndexNow protocol support / fast-indexing
|
|
88
|
+
const hasIndexNow = Boolean(
|
|
89
|
+
ctx.registries?.connections?.entries?.some((c) => c.provider === 'indexnow') ||
|
|
90
|
+
root.querySelector('link[rel="indexnow"]') ||
|
|
91
|
+
ctx.config?.indexnow
|
|
92
|
+
);
|
|
93
|
+
if (hasIndexNow) {
|
|
94
|
+
bingScore += 25;
|
|
95
|
+
bingChecks.push({ name: 'IndexNow Protocol', passed: true, note: 'IndexNow instant discovery integration detected' });
|
|
96
|
+
} else {
|
|
97
|
+
bingScore += 10;
|
|
98
|
+
bingChecks.push({ name: 'IndexNow Protocol', passed: false, note: 'IndexNow not configured; submission relies on standard crawl schedule' });
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
// Check 3: Schema.org JSON-LD completeness
|
|
102
|
+
const validSchemaBlocks = jsonLd.flatMap((j) => j.blocks || []).filter((b) => b['@type']);
|
|
103
|
+
const schemaTypes = validSchemaBlocks.map((b) => [].concat(b['@type']).join(',')).join('; ');
|
|
104
|
+
if (validSchemaBlocks.length >= 2) {
|
|
105
|
+
bingScore += 25;
|
|
106
|
+
bingChecks.push({ name: 'Structured Data', passed: true, note: `Rich schema graphs (${schemaTypes})` });
|
|
107
|
+
} else if (validSchemaBlocks.length === 1) {
|
|
108
|
+
bingScore += 15;
|
|
109
|
+
bingChecks.push({ name: 'Structured Data', passed: true, note: `Basic schema present (${schemaTypes})` });
|
|
110
|
+
} else {
|
|
111
|
+
bingChecks.push({ name: 'Structured Data', passed: false, note: 'No valid JSON-LD schema detected; Bing Copilot cards hindered' });
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
// Check 4: Deep page internal linking and canonical clarity
|
|
115
|
+
const canonicals = page.canonicals || [];
|
|
116
|
+
if (canonicals.length === 1) {
|
|
117
|
+
bingScore += 25;
|
|
118
|
+
bingChecks.push({ name: 'Canonical Authority', passed: true, note: `Canonical unambiguously specified (${canonicals[0]})` });
|
|
119
|
+
} else {
|
|
120
|
+
bingChecks.push({ name: 'Canonical Authority', passed: false, note: canonicals.length === 0 ? 'Missing canonical URL' : 'Multiple contradictory canonical tags' });
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
// -----------------------------------------------------------------
|
|
124
|
+
// 3. CHATGPT / SEARCHGPT READINESS
|
|
125
|
+
// -----------------------------------------------------------------
|
|
126
|
+
const chatGptChecks = [];
|
|
127
|
+
let chatGptScore = 0;
|
|
128
|
+
|
|
129
|
+
// Check 1: OAI-SearchBot / ChatGPT-User allowed
|
|
130
|
+
const oaiBlocked = robotsDirectives.some((d) => d.includes('oai-searchbot: noindex') || d.includes('chatgpt-user: noindex'));
|
|
131
|
+
if (!oaiBlocked && !isNoindex) {
|
|
132
|
+
chatGptScore += 25;
|
|
133
|
+
chatGptScore = Math.min(100, chatGptScore);
|
|
134
|
+
chatGptChecks.push({ name: 'SearchBot Policy', passed: true, note: 'OAI-SearchBot and ChatGPT-User allowed for search retrieval' });
|
|
135
|
+
} else {
|
|
136
|
+
chatGptChecks.push({ name: 'SearchBot Policy', passed: false, note: 'OAI-SearchBot / ChatGPT-User blocked by robots directives' });
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
// Check 2: RAG Chunkability (H2/H3 headers under 350 words per chunk)
|
|
140
|
+
const estimatedChunks = Math.max(1, Math.ceil(wordCount / 300));
|
|
141
|
+
const wordsPerHeading = headings.length > 0 ? Math.round(wordCount / headings.length) : wordCount;
|
|
142
|
+
if (headings.length >= 2 && wordsPerHeading >= 80 && wordsPerHeading <= 400) {
|
|
143
|
+
chatGptScore += 25;
|
|
144
|
+
chatGptChecks.push({ name: 'RAG Chunkability', passed: true, note: `Optimal chunk sizes (${wordsPerHeading} words/heading across ${headings.length} headings)` });
|
|
145
|
+
} else if (headings.length >= 1) {
|
|
146
|
+
chatGptScore += 15;
|
|
147
|
+
chatGptChecks.push({ name: 'RAG Chunkability', passed: true, note: `Sub-optimal chunking (${wordsPerHeading} words/heading)` });
|
|
148
|
+
} else {
|
|
149
|
+
chatGptChecks.push({ name: 'RAG Chunkability', passed: false, note: 'Monolithic content without H2/H3 hierarchy risks retrieval severance' });
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
// Check 3: Direct answer lead in first 100 words of section
|
|
153
|
+
const SUMMARY_RX = /\b(?:executive\s+summary|key\s+takeaways?|summary|overview|quick\s+facts|highlights?)\b/i;
|
|
154
|
+
const hasSummary = headings.some((h) => SUMMARY_RX.test(h.text));
|
|
155
|
+
if (hasSummary || hasCopularLead) {
|
|
156
|
+
chatGptScore += 25;
|
|
157
|
+
chatGptChecks.push({ name: 'Direct Answer Synthesis', passed: true, note: hasSummary ? 'Executive summary / key takeaways section present' : 'Immediate answer thesis in opening paragraph' });
|
|
158
|
+
} else {
|
|
159
|
+
chatGptChecks.push({ name: 'Direct Answer Synthesis', passed: false, note: 'Missing executive summary or lead key takeaways section' });
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
// Check 4: Quantitative claim grounding
|
|
163
|
+
const hasQuantitativeData = /(?:\b\d+(?:\.\d+)?%|\b\d+\s*ms\b|\b\d+\s*s\b|\$\d+(?:,\d{3})*)/.test(text);
|
|
164
|
+
if (hasQuantitativeData && wordCount >= 300) {
|
|
165
|
+
chatGptScore += 25;
|
|
166
|
+
chatGptChecks.push({ name: 'Claim Grounding', passed: true, note: 'Specific numerical metrics and quantitative facts present' });
|
|
167
|
+
} else if (wordCount >= 300) {
|
|
168
|
+
chatGptScore += 15;
|
|
169
|
+
chatGptChecks.push({ name: 'Claim Grounding', passed: false, note: 'Qualitative assertions without empirical statistics' });
|
|
170
|
+
} else {
|
|
171
|
+
chatGptChecks.push({ name: 'Claim Grounding', passed: false, note: 'Thin content lacks verifiable data' });
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
// Posture per engine
|
|
175
|
+
const posture = (s) => (s >= 80 ? 'optimal' : s >= 50 ? 'needs_optimization' : 'obstructed');
|
|
176
|
+
const perplexityStatus = posture(perplexityScore);
|
|
177
|
+
const bingStatus = posture(bingScore);
|
|
178
|
+
const chatGptStatus = posture(chatGptScore);
|
|
179
|
+
|
|
180
|
+
const compositeScore = Math.round((perplexityScore + bingScore + chatGptScore) / 3);
|
|
181
|
+
|
|
182
|
+
// Cross-engine prioritized recommendations
|
|
183
|
+
const recommendations = [];
|
|
184
|
+
if (!hasCopularLead) {
|
|
185
|
+
recommendations.push({
|
|
186
|
+
priority: 'high',
|
|
187
|
+
engine: 'all',
|
|
188
|
+
action: 'Add a concise definitional sentence ("X is Y") within the first 140 characters of the page',
|
|
189
|
+
});
|
|
190
|
+
}
|
|
191
|
+
if (!hasSummary) {
|
|
192
|
+
recommendations.push({
|
|
193
|
+
priority: 'medium',
|
|
194
|
+
engine: 'chatgpt',
|
|
195
|
+
action: 'Include a "Key Takeaways" or "Executive Summary" section at the top of long-form content',
|
|
196
|
+
});
|
|
197
|
+
}
|
|
198
|
+
if (questionHeadings.length < 2) {
|
|
199
|
+
recommendations.push({
|
|
200
|
+
priority: 'medium',
|
|
201
|
+
engine: 'perplexity',
|
|
202
|
+
action: 'Rephrase section headings as explicit user questions (e.g. "How does X work?", "What are the benefits of Y?")',
|
|
203
|
+
});
|
|
204
|
+
}
|
|
205
|
+
if (validSchemaBlocks.length < 2) {
|
|
206
|
+
recommendations.push({
|
|
207
|
+
priority: 'medium',
|
|
208
|
+
engine: 'bing_copilot',
|
|
209
|
+
action: 'Deploy structured JSON-LD schema (FAQPage, Article, or Product) to power Copilot card extraction',
|
|
210
|
+
});
|
|
211
|
+
}
|
|
212
|
+
if (!hasIndexNow) {
|
|
213
|
+
recommendations.push({
|
|
214
|
+
priority: 'low',
|
|
215
|
+
engine: 'bing_copilot',
|
|
216
|
+
action: 'Configure IndexNow protocol to notify Bing immediately of content updates',
|
|
217
|
+
});
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
return {
|
|
221
|
+
url: page.url,
|
|
222
|
+
fact_status: 'deterministic_engine_readiness_audit',
|
|
223
|
+
composite_readiness_score: compositeScore,
|
|
224
|
+
engines: {
|
|
225
|
+
perplexity: {
|
|
226
|
+
score: perplexityScore,
|
|
227
|
+
status: perplexityStatus,
|
|
228
|
+
primary_bot: 'PerplexityBot',
|
|
229
|
+
checks: perplexityChecks,
|
|
230
|
+
},
|
|
231
|
+
bing_copilot: {
|
|
232
|
+
score: bingScore,
|
|
233
|
+
status: bingStatus,
|
|
234
|
+
primary_bot: 'Bingbot',
|
|
235
|
+
checks: bingChecks,
|
|
236
|
+
},
|
|
237
|
+
chatgpt: {
|
|
238
|
+
score: chatGptScore,
|
|
239
|
+
status: chatGptStatus,
|
|
240
|
+
primary_bot: 'OAI-SearchBot',
|
|
241
|
+
checks: chatGptChecks,
|
|
242
|
+
},
|
|
243
|
+
},
|
|
244
|
+
comparison_matrix: [
|
|
245
|
+
{ capability: 'Direct Definitional Lead', perplexity: hasCopularLead ? 'SUPPORTED' : 'MISSING', bing_copilot: hasCopularLead ? 'SUPPORTED' : 'OPTIONAL', chatgpt: hasCopularLead ? 'SUPPORTED' : 'MISSING' },
|
|
246
|
+
{ capability: 'Q&A Heading Structure', perplexity: questionHeadings.length >= 2 ? 'HIGH' : 'LOW', bing_copilot: questionHeadings.length >= 2 ? 'SUPPORTED' : 'OPTIONAL', chatgpt: questionHeadings.length >= 2 ? 'OPTIMAL' : 'MODERATE' },
|
|
247
|
+
{ capability: 'Structured Data (JSON-LD)', perplexity: validSchemaBlocks.length ? 'PARSED' : 'FALLBACK', bing_copilot: validSchemaBlocks.length ? 'REQUIRED_CARD' : 'OBSTRUCTED', chatgpt: validSchemaBlocks.length ? 'GROUNDED' : 'PLAIN_TEXT' },
|
|
248
|
+
{ capability: 'RAG Section Chunkability', perplexity: wordsPerHeading <= 400 ? 'OPTIMAL' : 'LONG', bing_copilot: 'STANDARD', chatgpt: wordsPerHeading <= 400 ? 'OPTIMAL' : 'SEVERED' },
|
|
249
|
+
{ capability: 'Crawler Directives', perplexity: perplexityBotBlocked ? 'BLOCKED' : 'ALLOWED', bing_copilot: bingbotBlocked ? 'BLOCKED' : 'ALLOWED', chatgpt: oaiBlocked ? 'BLOCKED' : 'ALLOWED' },
|
|
250
|
+
],
|
|
251
|
+
recommendations,
|
|
252
|
+
};
|
|
253
|
+
}
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
import { parse as parseHtml } from 'node-html-parser';
|
|
2
|
+
|
|
3
|
+
const QUESTION_RX = /^(?:how|what|why|when|where|who|can|is|does|which|should)\b|\?$/i;
|
|
4
|
+
const HOWTO_TITLE_RX = /^(?:how\s+to|how\s+do\s+i|step-by-step|guide\s+to)\b/i;
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Scan audited site pages for FAQPage, Speakable, and HowTo markup opportunities.
|
|
8
|
+
*/
|
|
9
|
+
export function recommendSchemaForPages(pages = []) {
|
|
10
|
+
const recommendations = {
|
|
11
|
+
faq: [],
|
|
12
|
+
speakable: [],
|
|
13
|
+
howto: [],
|
|
14
|
+
};
|
|
15
|
+
|
|
16
|
+
for (const page of pages) {
|
|
17
|
+
if (!page || page.status !== 200) continue;
|
|
18
|
+
const html = page.html || '';
|
|
19
|
+
const root = parseHtml(html);
|
|
20
|
+
const existingTypes = new Set(
|
|
21
|
+
(page.jsonLd || []).flatMap((j) => j.blocks || []).flatMap((b) => [].concat(b['@type'] || []))
|
|
22
|
+
);
|
|
23
|
+
|
|
24
|
+
// -------------------------------------------------------------
|
|
25
|
+
// 1. FAQPage Recommendation
|
|
26
|
+
// -------------------------------------------------------------
|
|
27
|
+
if (!existingTypes.has('FAQPage')) {
|
|
28
|
+
const qas = [];
|
|
29
|
+
|
|
30
|
+
// Pattern A: <details><summary>Q</summary><p>A</p></details>
|
|
31
|
+
for (const details of root.querySelectorAll('details')) {
|
|
32
|
+
const summary = details.querySelector('summary');
|
|
33
|
+
if (summary) {
|
|
34
|
+
const questionText = summary.text.trim();
|
|
35
|
+
const clone = parseHtml(details.innerHTML);
|
|
36
|
+
const s = clone.querySelector('summary');
|
|
37
|
+
if (s) s.remove();
|
|
38
|
+
const answerText = clone.text.replace(/\s+/g, ' ').trim();
|
|
39
|
+
if (questionText && answerText) {
|
|
40
|
+
qas.push({ question: questionText, answer: answerText });
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
// Pattern B: Question headings followed by paragraphs
|
|
46
|
+
if (qas.length < 2) {
|
|
47
|
+
const headings = root.querySelectorAll('h2, h3');
|
|
48
|
+
for (const h of headings) {
|
|
49
|
+
const hText = h.text.trim();
|
|
50
|
+
if (QUESTION_RX.test(hText)) {
|
|
51
|
+
// Find next sibling paragraph
|
|
52
|
+
let next = h.nextElementSibling;
|
|
53
|
+
let answerText = '';
|
|
54
|
+
while (next && !['H1', 'H2', 'H3', 'H4'].includes(next.tagName)) {
|
|
55
|
+
if (['P', 'UL', 'OL', 'DIV'].includes(next.tagName)) {
|
|
56
|
+
answerText += (answerText ? ' ' : '') + next.text.replace(/\s+/g, ' ').trim();
|
|
57
|
+
if (answerText.length > 60) break;
|
|
58
|
+
}
|
|
59
|
+
next = next.nextElementSibling;
|
|
60
|
+
}
|
|
61
|
+
if (hText && answerText) {
|
|
62
|
+
qas.push({ question: hText, answer: answerText });
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
if (qas.length >= 2) {
|
|
69
|
+
recommendations.faq.push({
|
|
70
|
+
target_url: page.url,
|
|
71
|
+
source_file: page.sourceFile || null,
|
|
72
|
+
schema_type: 'FAQPage',
|
|
73
|
+
rationale: `Detected ${qas.length} visible question-and-answer pairs without FAQPage schema.`,
|
|
74
|
+
jsonld: {
|
|
75
|
+
'@context': 'https://schema.org',
|
|
76
|
+
'@type': 'FAQPage',
|
|
77
|
+
mainEntity: qas.slice(0, 10).map((qa) => ({
|
|
78
|
+
'@type': 'Question',
|
|
79
|
+
name: qa.question,
|
|
80
|
+
acceptedAnswer: {
|
|
81
|
+
'@type': 'Answer',
|
|
82
|
+
text: qa.answer,
|
|
83
|
+
},
|
|
84
|
+
})),
|
|
85
|
+
},
|
|
86
|
+
});
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
// -------------------------------------------------------------
|
|
91
|
+
// 2. Speakable Recommendation (SpeakableSpecification)
|
|
92
|
+
// -------------------------------------------------------------
|
|
93
|
+
const hasExistingSpeakable = (page.jsonLd || []).some((j) =>
|
|
94
|
+
(j.blocks || []).some((b) => b.speakable)
|
|
95
|
+
);
|
|
96
|
+
|
|
97
|
+
if (!hasExistingSpeakable) {
|
|
98
|
+
const speakableSelectors = [];
|
|
99
|
+
if (root.querySelector('h1')) speakableSelectors.push('h1');
|
|
100
|
+
if (root.querySelector('#summary, .executive-summary, .summary, .key-takeaways')) {
|
|
101
|
+
speakableSelectors.push(root.querySelector('#summary') ? '#summary' : '.executive-summary');
|
|
102
|
+
} else if (root.querySelector('article > p:first-of-type, main > p:first-of-type, p:first-of-type')) {
|
|
103
|
+
speakableSelectors.push('article > p:first-of-type');
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
if (speakableSelectors.length >= 2) {
|
|
107
|
+
recommendations.speakable.push({
|
|
108
|
+
target_url: page.url,
|
|
109
|
+
source_file: page.sourceFile || null,
|
|
110
|
+
schema_type: 'SpeakableSpecification',
|
|
111
|
+
rationale: 'Page has clear title and executive lead passage suitable for smart speaker / assistant audio playback.',
|
|
112
|
+
jsonld: {
|
|
113
|
+
'@context': 'https://schema.org',
|
|
114
|
+
'@type': 'WebPage',
|
|
115
|
+
'@id': `${page.url}#webpage`,
|
|
116
|
+
url: page.url,
|
|
117
|
+
name: page.title || 'Page',
|
|
118
|
+
speakable: {
|
|
119
|
+
'@type': 'SpeakableSpecification',
|
|
120
|
+
cssSelector: speakableSelectors,
|
|
121
|
+
},
|
|
122
|
+
},
|
|
123
|
+
});
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
// -------------------------------------------------------------
|
|
128
|
+
// 3. HowTo Recommendation
|
|
129
|
+
// -------------------------------------------------------------
|
|
130
|
+
if (!existingTypes.has('HowTo')) {
|
|
131
|
+
const pageTitle = page.title || '';
|
|
132
|
+
const h1s = (page.headings || []).filter((h) => h.level === 1).map((h) => h.text);
|
|
133
|
+
const isHowToPage = HOWTO_TITLE_RX.test(pageTitle) || h1s.some((h) => HOWTO_TITLE_RX.test(h));
|
|
134
|
+
|
|
135
|
+
if (isHowToPage) {
|
|
136
|
+
const steps = [];
|
|
137
|
+
// Look for ordered list items or step subheadings
|
|
138
|
+
const olItems = root.querySelectorAll('ol > li');
|
|
139
|
+
if (olItems.length >= 2) {
|
|
140
|
+
olItems.forEach((li, idx) => {
|
|
141
|
+
const stepText = li.text.replace(/\s+/g, ' ').trim();
|
|
142
|
+
if (stepText) {
|
|
143
|
+
steps.push({
|
|
144
|
+
'@type': 'HowToStep',
|
|
145
|
+
position: idx + 1,
|
|
146
|
+
name: `Step ${idx + 1}`,
|
|
147
|
+
text: stepText,
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
});
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
if (steps.length >= 2) {
|
|
154
|
+
recommendations.howto.push({
|
|
155
|
+
target_url: page.url,
|
|
156
|
+
source_file: page.sourceFile || null,
|
|
157
|
+
schema_type: 'HowTo',
|
|
158
|
+
rationale: `Procedural instructional guide detected with ${steps.length} ordered steps.`,
|
|
159
|
+
jsonld: {
|
|
160
|
+
'@context': 'https://schema.org',
|
|
161
|
+
'@type': 'HowTo',
|
|
162
|
+
name: h1s[0] || pageTitle,
|
|
163
|
+
description: page.paragraphs?.[0] || 'Procedural guide and instructions.',
|
|
164
|
+
step: steps,
|
|
165
|
+
},
|
|
166
|
+
});
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
return recommendations;
|
|
173
|
+
}
|