@nebulacomponents/citable 1.17.0 → 1.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. package/CHANGELOG.md +22 -0
  2. package/dist/universal/.agents/skills/citable/SKILL.md +20 -1
  3. package/dist/universal/.agents/skills/citable/VERSION +1 -1
  4. package/dist/universal/.agents/skills/citable/manifest.json +6 -5
  5. package/dist/universal/.agents/skills/citable/schemas/sow.schema.json +92 -0
  6. package/dist/universal/.claude/agents/citable/manifest.json +2 -2
  7. package/dist/universal/.claude/skills/citable/SKILL.md +20 -1
  8. package/dist/universal/.claude/skills/citable/VERSION +1 -1
  9. package/dist/universal/.claude/skills/citable/manifest.json +6 -5
  10. package/dist/universal/.claude/skills/citable/schemas/sow.schema.json +92 -0
  11. package/dist/universal/.cursor/skills/citable/SKILL.md +20 -1
  12. package/dist/universal/.cursor/skills/citable/VERSION +1 -1
  13. package/dist/universal/.cursor/skills/citable/manifest.json +6 -5
  14. package/dist/universal/.cursor/skills/citable/schemas/sow.schema.json +92 -0
  15. package/dist/universal/.gemini/skills/citable/SKILL.md +20 -1
  16. package/dist/universal/.gemini/skills/citable/VERSION +1 -1
  17. package/dist/universal/.gemini/skills/citable/manifest.json +6 -5
  18. package/dist/universal/.gemini/skills/citable/schemas/sow.schema.json +92 -0
  19. package/dist/universal/.github/skills/citable/SKILL.md +20 -1
  20. package/dist/universal/.github/skills/citable/VERSION +1 -1
  21. package/dist/universal/.github/skills/citable/manifest.json +6 -5
  22. package/dist/universal/.github/skills/citable/schemas/sow.schema.json +92 -0
  23. package/dist/universal/.kiro/skills/citable/SKILL.md +20 -1
  24. package/dist/universal/.kiro/skills/citable/VERSION +1 -1
  25. package/dist/universal/.kiro/skills/citable/manifest.json +6 -5
  26. package/dist/universal/.kiro/skills/citable/schemas/sow.schema.json +92 -0
  27. package/dist/universal/.opencode/skills/citable/SKILL.md +20 -1
  28. package/dist/universal/.opencode/skills/citable/VERSION +1 -1
  29. package/dist/universal/.opencode/skills/citable/manifest.json +6 -5
  30. package/dist/universal/.opencode/skills/citable/schemas/sow.schema.json +92 -0
  31. package/dist/universal/.pi/agent/skills/citable/SKILL.md +20 -1
  32. package/dist/universal/.pi/agent/skills/citable/VERSION +1 -1
  33. package/dist/universal/.pi/agent/skills/citable/manifest.json +6 -5
  34. package/dist/universal/.pi/agent/skills/citable/schemas/sow.schema.json +92 -0
  35. package/dist/universal/.qoder/skills/citable/SKILL.md +20 -1
  36. package/dist/universal/.qoder/skills/citable/VERSION +1 -1
  37. package/dist/universal/.qoder/skills/citable/manifest.json +6 -5
  38. package/dist/universal/.qoder/skills/citable/schemas/sow.schema.json +92 -0
  39. package/dist/universal/.rovodev/skills/citable/SKILL.md +20 -1
  40. package/dist/universal/.rovodev/skills/citable/VERSION +1 -1
  41. package/dist/universal/.rovodev/skills/citable/manifest.json +6 -5
  42. package/dist/universal/.rovodev/skills/citable/schemas/sow.schema.json +92 -0
  43. package/dist/universal/.trae/skills/citable/SKILL.md +20 -1
  44. package/dist/universal/.trae/skills/citable/VERSION +1 -1
  45. package/dist/universal/.trae/skills/citable/manifest.json +6 -5
  46. package/dist/universal/.trae/skills/citable/schemas/sow.schema.json +92 -0
  47. package/dist/universal/.trae-cn/skills/citable/SKILL.md +20 -1
  48. package/dist/universal/.trae-cn/skills/citable/VERSION +1 -1
  49. package/dist/universal/.trae-cn/skills/citable/manifest.json +6 -5
  50. package/dist/universal/.trae-cn/skills/citable/schemas/sow.schema.json +92 -0
  51. package/dist/universal/manifest.json +76 -64
  52. package/package.json +1 -1
  53. package/schemas/sow.schema.json +92 -0
  54. package/skill/SKILL.md +20 -1
  55. package/src/analysis/behavioral.js +126 -0
  56. package/src/analysis/croAudit.js +302 -0
  57. package/src/analysis/croRoadmap.js +121 -0
  58. package/src/analysis/eeat.js +300 -0
  59. package/src/analysis/funnelAnalysis.js +171 -0
  60. package/src/analysis/iceMatrix.js +205 -0
  61. package/src/analysis/offpage.js +228 -0
  62. package/src/analysis/readiness.js +253 -0
  63. package/src/analysis/schemaRecommendations.js +173 -0
  64. package/src/analysis/strategicRoadmap.js +148 -0
  65. package/src/cli/index.js +149 -3
  66. package/src/commands/answerEngineReadiness.js +48 -0
  67. package/src/commands/auditBacklinks.js +79 -0
  68. package/src/commands/croBacklog.js +182 -0
  69. package/src/commands/croSuite.js +156 -0
  70. package/src/commands/executive.js +6 -0
  71. package/src/commands/executiveCro.js +30 -0
  72. package/src/commands/executiveSearch.js +28 -0
  73. package/src/commands/inspectCro.js +7 -0
  74. package/src/commands/inspectEeat.js +53 -0
  75. package/src/commands/prioritize.js +42 -6
  76. package/src/commands/roadmapCmd.js +69 -0
  77. package/src/commands/schemaCmd.js +10 -2
  78. package/src/commands/sowCmd.js +67 -0
  79. package/src/commands/sweep.js +357 -0
  80. package/src/reporting/executiveCroReport.js +731 -0
  81. package/src/reporting/executiveExport.js +62 -0
  82. package/src/reporting/executiveSearchReport.js +704 -0
  83. package/src/sow/admissibilityGate.js +194 -0
  84. package/src/sow/generateSow.js +762 -0
@@ -0,0 +1,228 @@
1
+ import { URL } from 'node:url';
2
+
3
+ const SPAM_TLDS = new Set([
4
+ 'top', 'work', 'gdn', 'click', 'monster', 'buzz', 'cfd', 'sbs',
5
+ 'rest', 'cam', 'fit', 'surf', 'stream', 'tk', 'ml', 'ga', 'cf',
6
+ ]);
7
+
8
+ const COMMERCIAL_SPAM_KEYWORDS = /\b(casino|viagra|cialis|payday\s*loans?|replica|cheap\s*essay|free\s*download|warez|crack|hack|gambling|betting|slots|crypto\s*airdrop)\b/i;
9
+
10
+ function extractHostname(urlString) {
11
+ try {
12
+ return new URL(urlString).hostname.toLowerCase();
13
+ } catch {
14
+ return null;
15
+ }
16
+ }
17
+
18
+ function extractRootDomain(hostname) {
19
+ if (!hostname) return null;
20
+ const parts = hostname.split('.');
21
+ if (parts.length <= 2) return hostname;
22
+ return parts.slice(-2).join('.');
23
+ }
24
+
25
+ function extractTld(hostname) {
26
+ if (!hostname) return null;
27
+ const parts = hostname.split('.');
28
+ return parts.at(-1) || null;
29
+ }
30
+
31
+ /**
32
+ * Audit an inbound backlink profile for authority distribution and toxic domain risks.
33
+ */
34
+ export function auditBacklinkProfile(backlinks, { targetDomain = null } = {}) {
35
+ const links = Array.isArray(backlinks) ? backlinks : (backlinks?.backlinks || []);
36
+ const normalizedTarget = targetDomain ? targetDomain.toLowerCase().replace(/^www\./, '') : null;
37
+
38
+ const referringDomains = new Map();
39
+ const cClassClusters = new Map();
40
+ let dofollowCount = 0;
41
+ let nofollowCount = 0;
42
+ let ugcCount = 0;
43
+ let sponsoredCount = 0;
44
+ let homepageLinksCount = 0;
45
+ let deepLinksCount = 0;
46
+
47
+ const anchorTypes = {
48
+ branded: 0,
49
+ naked_url: 0,
50
+ generic: 0,
51
+ exact_match_commercial: 0,
52
+ other: 0,
53
+ };
54
+
55
+ const toxicDomains = [];
56
+ const GENERIC_ANCHORS = /^(?:click here|here|website|link|read more|learn more|source|this site|visit|more info)$/i;
57
+
58
+ // 1. Process each backlink
59
+ for (const link of links) {
60
+ const srcHost = extractHostname(link.source_url);
61
+ if (!srcHost) continue;
62
+ const rootDomain = extractRootDomain(srcHost) || srcHost;
63
+ const tld = extractTld(srcHost);
64
+ const anchor = (link.anchor_text || '').trim();
65
+ const rel = (link.rel || '').toLowerCase();
66
+ const isNofollow = rel.includes('nofollow');
67
+ const isUgc = rel.includes('ugc');
68
+ const isSponsored = rel.includes('sponsored');
69
+
70
+ if (isNofollow) nofollowCount++;
71
+ else if (isUgc) ugcCount++;
72
+ else if (isSponsored) sponsoredCount++;
73
+ else dofollowCount++;
74
+
75
+ // Deep link vs homepage
76
+ const targetPath = link.target_url ? (() => { try { return new URL(link.target_url).pathname; } catch { return '/'; } })() : '/';
77
+ if (targetPath === '/' || targetPath === '') homepageLinksCount++;
78
+ else deepLinksCount++;
79
+
80
+ // Anchor categorization
81
+ if (normalizedTarget && (anchor.toLowerCase().includes(normalizedTarget) || normalizedTarget.includes(anchor.toLowerCase()))) {
82
+ anchorTypes.branded++;
83
+ } else if (/^https?:\/\//i.test(anchor) || anchor.includes('www.') || anchor.includes('.com') || anchor.includes('.org')) {
84
+ anchorTypes.naked_url++;
85
+ } else if (GENERIC_ANCHORS.test(anchor)) {
86
+ anchorTypes.generic++;
87
+ } else if (COMMERCIAL_SPAM_KEYWORDS.test(anchor) || /\b(buy|best|cheap|discount|order)\b/i.test(anchor)) {
88
+ anchorTypes.exact_match_commercial++;
89
+ } else {
90
+ anchorTypes.other++;
91
+ }
92
+
93
+ // C-class IP clustering
94
+ if (link.source_ip) {
95
+ const parts = link.source_ip.split('.');
96
+ if (parts.length === 4) {
97
+ const cClass = parts.slice(0, 3).join('.');
98
+ const cluster = cClassClusters.get(cClass) || new Set();
99
+ cluster.add(rootDomain);
100
+ cClassClusters.set(cClass, cluster);
101
+ }
102
+ }
103
+
104
+ // Domain tracking
105
+ const domainData = referringDomains.get(rootDomain) || {
106
+ domain: rootDomain,
107
+ sample_source_url: link.source_url,
108
+ total_links: 0,
109
+ anchors: [],
110
+ tld,
111
+ ip: link.source_ip || null,
112
+ dofollow: false,
113
+ };
114
+ domainData.total_links++;
115
+ if (!isNofollow && !isSponsored) domainData.dofollow = true;
116
+ if (anchor && !domainData.anchors.includes(anchor)) domainData.anchors.push(anchor);
117
+ referringDomains.set(rootDomain, domainData);
118
+ }
119
+
120
+ // 2. Toxic domain risk analysis per domain
121
+ for (const [domain, data] of referringDomains.entries()) {
122
+ const reasons = [];
123
+ let riskTier = 'clean';
124
+
125
+ // Spam TLD rule
126
+ if (data.tld && SPAM_TLDS.has(data.tld)) {
127
+ reasons.push(`Known high-abuse spam TLD (.${data.tld})`);
128
+ riskTier = 'high';
129
+ }
130
+
131
+ // Spam anchor rule
132
+ const hasSpamAnchor = data.anchors.some((a) => COMMERCIAL_SPAM_KEYWORDS.test(a));
133
+ if (hasSpamAnchor) {
134
+ reasons.push('Anchor text matches known high-risk commercial spam patterns');
135
+ riskTier = 'critical';
136
+ }
137
+
138
+ // PBN IP clustering rule (3+ distinct domains on the exact same /24 C-block)
139
+ if (data.ip) {
140
+ const parts = data.ip.split('.');
141
+ if (parts.length === 4) {
142
+ const cClass = parts.slice(0, 3).join('.');
143
+ const cluster = cClassClusters.get(cClass);
144
+ if (cluster && cluster.size >= 3) {
145
+ reasons.push(`Part of hosting cluster on ${cClass}.0/24 with ${cluster.size} domains (PBN footprint)`);
146
+ riskTier = 'critical';
147
+ }
148
+ }
149
+ }
150
+
151
+ // Excessive sitewide link density without nofollow
152
+ if (data.total_links > 50 && data.dofollow && (data.tld && SPAM_TLDS.has(data.tld))) {
153
+ reasons.push(`Sitewide link farm pattern (${data.total_links} links without rel="nofollow")`);
154
+ riskTier = 'critical';
155
+ }
156
+
157
+ if (riskTier !== 'clean') {
158
+ toxicDomains.push({
159
+ domain,
160
+ risk_tier: riskTier,
161
+ reasons,
162
+ sample_url: data.sample_source_url,
163
+ anchors: data.anchors.slice(0, 5),
164
+ total_links: data.total_links,
165
+ });
166
+ }
167
+ }
168
+
169
+ // Sort toxic domains by severity (critical first)
170
+ toxicDomains.sort((a, b) => (a.risk_tier === 'critical' ? -1 : 1));
171
+
172
+ const totalLinks = links.length;
173
+ const totalDomains = referringDomains.size;
174
+ const commercialAnchorPct = totalLinks > 0 ? Math.round((anchorTypes.exact_match_commercial / totalLinks) * 100) : 0;
175
+ const brandedAnchorPct = totalLinks > 0 ? Math.round((anchorTypes.branded / totalLinks) * 100) : 0;
176
+ const deepLinkPct = totalLinks > 0 ? Math.round((deepLinksCount / totalLinks) * 100) : 0;
177
+
178
+ // Authority health assessment
179
+ let profileHealth = 'natural';
180
+ if (toxicDomains.some((d) => d.risk_tier === 'critical') || commercialAnchorPct > 30) {
181
+ profileHealth = 'high_risk_footprint';
182
+ } else if (toxicDomains.length > 0 || commercialAnchorPct > 15) {
183
+ profileHealth = 'moderate_risk';
184
+ }
185
+
186
+ // Generate GSC-compatible disavow lines
187
+ const disavowLines = [
188
+ `# Citable Google Disavow Candidate Export`,
189
+ `# Generated at: ${new Date().toISOString()}`,
190
+ `# Flagged domains: ${toxicDomains.length} (critical: ${toxicDomains.filter((d) => d.risk_tier === 'critical').length})`,
191
+ ``,
192
+ ];
193
+ for (const td of toxicDomains) {
194
+ disavowLines.push(`# Risk: ${td.risk_tier.toUpperCase()} — ${td.reasons.join('; ')}`);
195
+ disavowLines.push(`domain:${td.domain}`);
196
+ }
197
+
198
+ return {
199
+ fact_status: 'observable_risk_indicators',
200
+ target_domain: targetDomain,
201
+ profile_health: profileHealth,
202
+ summary: {
203
+ total_backlinks: totalLinks,
204
+ total_referring_domains: totalDomains,
205
+ toxic_domains_count: toxicDomains.length,
206
+ critical_risk_domains: toxicDomains.filter((d) => d.risk_tier === 'critical').length,
207
+ high_risk_domains: toxicDomains.filter((d) => d.risk_tier === 'high').length,
208
+ dofollow_count: dofollowCount,
209
+ nofollow_count: nofollowCount,
210
+ ugc_count: ugcCount,
211
+ sponsored_count: sponsoredCount,
212
+ deep_link_ratio_pct: deepLinkPct,
213
+ },
214
+ anchor_profile: {
215
+ branded_pct: brandedAnchorPct,
216
+ commercial_exact_match_pct: commercialAnchorPct,
217
+ naked_url_count: anchorTypes.naked_url,
218
+ generic_count: anchorTypes.generic,
219
+ over_optimization_risk: commercialAnchorPct > 25 ? 'elevated' : 'normal',
220
+ },
221
+ toxic_domains: toxicDomains,
222
+ disavow_export: disavowLines.join('\n'),
223
+ limitations: [
224
+ 'Search engines do not disclose algorithmic penalty thresholds; toxic classifications reflect observable risk patterns.',
225
+ 'Manual verification is required before submitting disavow files to Google Search Console.',
226
+ ],
227
+ };
228
+ }
@@ -0,0 +1,253 @@
1
+ import { parse as parseHtml } from 'node-html-parser';
2
+ import { registryPageFor, safePath } from '../detectors/framework.js';
3
+
4
+ const QUESTION_RX = /^(?:how|what|why|when|where|who|can|is|does|which|should)\b|\?$/i;
5
+ const COPULA_RX = /\b(?:is|are|refers\s+to|denotes|represents|means|provides|enables)\b/i;
6
+
7
+ /**
8
+ * Evaluates a page's extraction, retrieval, and citation readiness across
9
+ * Perplexity, Bing Copilot, and ChatGPT / SearchGPT.
10
+ */
11
+ export function evaluateAnswerEngineReadiness(page, ctx = {}) {
12
+ const html = page.html || '';
13
+ const root = parseHtml(html);
14
+ const text = (page.text || root.textContent || '').replace(/\s+/g, ' ').trim();
15
+ const headings = page.headings || [];
16
+ const paragraphs = page.paragraphs || root.querySelectorAll('p').map((p) => p.text.trim()).filter(Boolean);
17
+ const jsonLd = page.jsonLd || [];
18
+ const tablesCount = (page.tables ?? root.querySelectorAll('table').length);
19
+ const listsCount = (page.orderedLists ?? root.querySelectorAll('ol').length) + root.querySelectorAll('ul').length;
20
+ const wordCount = page.rawVisibleWordCount || page.wordCount || text.split(/\s+/).filter(Boolean).length;
21
+
22
+ // Crawler directives check (from robots Directives or headers)
23
+ const robotsDirectives = Array.from(page.robotsDirectives || []);
24
+ const isNoindex = robotsDirectives.some((d) => d.includes('noindex'));
25
+
26
+ // -----------------------------------------------------------------
27
+ // 1. PERPLEXITY READINESS
28
+ // -----------------------------------------------------------------
29
+ const perplexityChecks = [];
30
+ let perplexityScore = 0;
31
+
32
+ // Check 1: Crawler policy (PerplexityBot)
33
+ const perplexityBotBlocked = robotsDirectives.some((d) => d.includes('perplexitybot: noindex') || d.includes('perplexitybot: none'));
34
+ if (!perplexityBotBlocked && !isNoindex) {
35
+ perplexityScore += 25;
36
+ perplexityChecks.push({ name: 'Crawler Policy', passed: true, note: 'PerplexityBot allowed' });
37
+ } else {
38
+ perplexityChecks.push({ name: 'Crawler Policy', passed: false, note: 'Blocked by robots directive' });
39
+ }
40
+
41
+ // Check 2: Direct Definitional Lead (Copula in first paragraph)
42
+ const firstParagraph = paragraphs[0] || '';
43
+ const hasCopularLead = COPULA_RX.test(firstParagraph.slice(0, 160));
44
+ if (hasCopularLead) {
45
+ perplexityScore += 25;
46
+ perplexityChecks.push({ name: 'Definitional Lead', passed: true, note: 'Concise copular definition in opening passage' });
47
+ } else {
48
+ perplexityChecks.push({ name: 'Definitional Lead', passed: false, note: 'Opening passage lacks concise definitional copula ("X is Y")' });
49
+ }
50
+
51
+ // Check 3: Structured question headings
52
+ const questionHeadings = headings.filter((h) => QUESTION_RX.test(h.text.trim()));
53
+ if (questionHeadings.length >= 2) {
54
+ perplexityScore += 25;
55
+ perplexityChecks.push({ name: 'Question Density', passed: true, note: `${questionHeadings.length} interrogative headings found` });
56
+ } else if (questionHeadings.length === 1) {
57
+ perplexityScore += 15;
58
+ perplexityChecks.push({ name: 'Question Density', passed: true, note: '1 interrogative heading found' });
59
+ } else {
60
+ perplexityChecks.push({ name: 'Question Density', passed: false, note: 'No interrogative headings found (Q&A format optimizes extraction)' });
61
+ }
62
+
63
+ // Check 4: Structured data & tables for citation synthesis
64
+ if (tablesCount >= 1 || listsCount >= 2) {
65
+ perplexityScore += 25;
66
+ perplexityChecks.push({ name: 'Tabular / List Data', passed: true, note: `${tablesCount} table(s), ${listsCount} list(s) for direct citation` });
67
+ } else {
68
+ perplexityScore += 10;
69
+ perplexityChecks.push({ name: 'Tabular / List Data', passed: false, note: 'Limited tables or structured lists for bulleted answers' });
70
+ }
71
+
72
+ // -----------------------------------------------------------------
73
+ // 2. BING COPILOT READINESS
74
+ // -----------------------------------------------------------------
75
+ const bingChecks = [];
76
+ let bingScore = 0;
77
+
78
+ // Check 1: Bingbot allowed
79
+ const bingbotBlocked = robotsDirectives.some((d) => d.includes('bingbot: noindex'));
80
+ if (!bingbotBlocked && !isNoindex) {
81
+ bingScore += 25;
82
+ bingChecks.push({ name: 'Crawler Policy', passed: true, note: 'Bingbot allowed' });
83
+ } else {
84
+ bingChecks.push({ name: 'Crawler Policy', passed: false, note: 'Bingbot blocked' });
85
+ }
86
+
87
+ // Check 2: IndexNow protocol support / fast-indexing
88
+ const hasIndexNow = Boolean(
89
+ ctx.registries?.connections?.entries?.some((c) => c.provider === 'indexnow') ||
90
+ root.querySelector('link[rel="indexnow"]') ||
91
+ ctx.config?.indexnow
92
+ );
93
+ if (hasIndexNow) {
94
+ bingScore += 25;
95
+ bingChecks.push({ name: 'IndexNow Protocol', passed: true, note: 'IndexNow instant discovery integration detected' });
96
+ } else {
97
+ bingScore += 10;
98
+ bingChecks.push({ name: 'IndexNow Protocol', passed: false, note: 'IndexNow not configured; submission relies on standard crawl schedule' });
99
+ }
100
+
101
+ // Check 3: Schema.org JSON-LD completeness
102
+ const validSchemaBlocks = jsonLd.flatMap((j) => j.blocks || []).filter((b) => b['@type']);
103
+ const schemaTypes = validSchemaBlocks.map((b) => [].concat(b['@type']).join(',')).join('; ');
104
+ if (validSchemaBlocks.length >= 2) {
105
+ bingScore += 25;
106
+ bingChecks.push({ name: 'Structured Data', passed: true, note: `Rich schema graphs (${schemaTypes})` });
107
+ } else if (validSchemaBlocks.length === 1) {
108
+ bingScore += 15;
109
+ bingChecks.push({ name: 'Structured Data', passed: true, note: `Basic schema present (${schemaTypes})` });
110
+ } else {
111
+ bingChecks.push({ name: 'Structured Data', passed: false, note: 'No valid JSON-LD schema detected; Bing Copilot cards hindered' });
112
+ }
113
+
114
+ // Check 4: Deep page internal linking and canonical clarity
115
+ const canonicals = page.canonicals || [];
116
+ if (canonicals.length === 1) {
117
+ bingScore += 25;
118
+ bingChecks.push({ name: 'Canonical Authority', passed: true, note: `Canonical unambiguously specified (${canonicals[0]})` });
119
+ } else {
120
+ bingChecks.push({ name: 'Canonical Authority', passed: false, note: canonicals.length === 0 ? 'Missing canonical URL' : 'Multiple contradictory canonical tags' });
121
+ }
122
+
123
+ // -----------------------------------------------------------------
124
+ // 3. CHATGPT / SEARCHGPT READINESS
125
+ // -----------------------------------------------------------------
126
+ const chatGptChecks = [];
127
+ let chatGptScore = 0;
128
+
129
+ // Check 1: OAI-SearchBot / ChatGPT-User allowed
130
+ const oaiBlocked = robotsDirectives.some((d) => d.includes('oai-searchbot: noindex') || d.includes('chatgpt-user: noindex'));
131
+ if (!oaiBlocked && !isNoindex) {
132
+ chatGptScore += 25;
133
+ chatGptScore = Math.min(100, chatGptScore);
134
+ chatGptChecks.push({ name: 'SearchBot Policy', passed: true, note: 'OAI-SearchBot and ChatGPT-User allowed for search retrieval' });
135
+ } else {
136
+ chatGptChecks.push({ name: 'SearchBot Policy', passed: false, note: 'OAI-SearchBot / ChatGPT-User blocked by robots directives' });
137
+ }
138
+
139
+ // Check 2: RAG Chunkability (H2/H3 headers under 350 words per chunk)
140
+ const estimatedChunks = Math.max(1, Math.ceil(wordCount / 300));
141
+ const wordsPerHeading = headings.length > 0 ? Math.round(wordCount / headings.length) : wordCount;
142
+ if (headings.length >= 2 && wordsPerHeading >= 80 && wordsPerHeading <= 400) {
143
+ chatGptScore += 25;
144
+ chatGptChecks.push({ name: 'RAG Chunkability', passed: true, note: `Optimal chunk sizes (${wordsPerHeading} words/heading across ${headings.length} headings)` });
145
+ } else if (headings.length >= 1) {
146
+ chatGptScore += 15;
147
+ chatGptChecks.push({ name: 'RAG Chunkability', passed: true, note: `Sub-optimal chunking (${wordsPerHeading} words/heading)` });
148
+ } else {
149
+ chatGptChecks.push({ name: 'RAG Chunkability', passed: false, note: 'Monolithic content without H2/H3 hierarchy risks retrieval severance' });
150
+ }
151
+
152
+ // Check 3: Direct answer lead in first 100 words of section
153
+ const SUMMARY_RX = /\b(?:executive\s+summary|key\s+takeaways?|summary|overview|quick\s+facts|highlights?)\b/i;
154
+ const hasSummary = headings.some((h) => SUMMARY_RX.test(h.text));
155
+ if (hasSummary || hasCopularLead) {
156
+ chatGptScore += 25;
157
+ chatGptChecks.push({ name: 'Direct Answer Synthesis', passed: true, note: hasSummary ? 'Executive summary / key takeaways section present' : 'Immediate answer thesis in opening paragraph' });
158
+ } else {
159
+ chatGptChecks.push({ name: 'Direct Answer Synthesis', passed: false, note: 'Missing executive summary or lead key takeaways section' });
160
+ }
161
+
162
+ // Check 4: Quantitative claim grounding
163
+ const hasQuantitativeData = /(?:\b\d+(?:\.\d+)?%|\b\d+\s*ms\b|\b\d+\s*s\b|\$\d+(?:,\d{3})*)/.test(text);
164
+ if (hasQuantitativeData && wordCount >= 300) {
165
+ chatGptScore += 25;
166
+ chatGptChecks.push({ name: 'Claim Grounding', passed: true, note: 'Specific numerical metrics and quantitative facts present' });
167
+ } else if (wordCount >= 300) {
168
+ chatGptScore += 15;
169
+ chatGptChecks.push({ name: 'Claim Grounding', passed: false, note: 'Qualitative assertions without empirical statistics' });
170
+ } else {
171
+ chatGptChecks.push({ name: 'Claim Grounding', passed: false, note: 'Thin content lacks verifiable data' });
172
+ }
173
+
174
+ // Posture per engine
175
+ const posture = (s) => (s >= 80 ? 'optimal' : s >= 50 ? 'needs_optimization' : 'obstructed');
176
+ const perplexityStatus = posture(perplexityScore);
177
+ const bingStatus = posture(bingScore);
178
+ const chatGptStatus = posture(chatGptScore);
179
+
180
+ const compositeScore = Math.round((perplexityScore + bingScore + chatGptScore) / 3);
181
+
182
+ // Cross-engine prioritized recommendations
183
+ const recommendations = [];
184
+ if (!hasCopularLead) {
185
+ recommendations.push({
186
+ priority: 'high',
187
+ engine: 'all',
188
+ action: 'Add a concise definitional sentence ("X is Y") within the first 140 characters of the page',
189
+ });
190
+ }
191
+ if (!hasSummary) {
192
+ recommendations.push({
193
+ priority: 'medium',
194
+ engine: 'chatgpt',
195
+ action: 'Include a "Key Takeaways" or "Executive Summary" section at the top of long-form content',
196
+ });
197
+ }
198
+ if (questionHeadings.length < 2) {
199
+ recommendations.push({
200
+ priority: 'medium',
201
+ engine: 'perplexity',
202
+ action: 'Rephrase section headings as explicit user questions (e.g. "How does X work?", "What are the benefits of Y?")',
203
+ });
204
+ }
205
+ if (validSchemaBlocks.length < 2) {
206
+ recommendations.push({
207
+ priority: 'medium',
208
+ engine: 'bing_copilot',
209
+ action: 'Deploy structured JSON-LD schema (FAQPage, Article, or Product) to power Copilot card extraction',
210
+ });
211
+ }
212
+ if (!hasIndexNow) {
213
+ recommendations.push({
214
+ priority: 'low',
215
+ engine: 'bing_copilot',
216
+ action: 'Configure IndexNow protocol to notify Bing immediately of content updates',
217
+ });
218
+ }
219
+
220
+ return {
221
+ url: page.url,
222
+ fact_status: 'deterministic_engine_readiness_audit',
223
+ composite_readiness_score: compositeScore,
224
+ engines: {
225
+ perplexity: {
226
+ score: perplexityScore,
227
+ status: perplexityStatus,
228
+ primary_bot: 'PerplexityBot',
229
+ checks: perplexityChecks,
230
+ },
231
+ bing_copilot: {
232
+ score: bingScore,
233
+ status: bingStatus,
234
+ primary_bot: 'Bingbot',
235
+ checks: bingChecks,
236
+ },
237
+ chatgpt: {
238
+ score: chatGptScore,
239
+ status: chatGptStatus,
240
+ primary_bot: 'OAI-SearchBot',
241
+ checks: chatGptChecks,
242
+ },
243
+ },
244
+ comparison_matrix: [
245
+ { capability: 'Direct Definitional Lead', perplexity: hasCopularLead ? 'SUPPORTED' : 'MISSING', bing_copilot: hasCopularLead ? 'SUPPORTED' : 'OPTIONAL', chatgpt: hasCopularLead ? 'SUPPORTED' : 'MISSING' },
246
+ { capability: 'Q&A Heading Structure', perplexity: questionHeadings.length >= 2 ? 'HIGH' : 'LOW', bing_copilot: questionHeadings.length >= 2 ? 'SUPPORTED' : 'OPTIONAL', chatgpt: questionHeadings.length >= 2 ? 'OPTIMAL' : 'MODERATE' },
247
+ { capability: 'Structured Data (JSON-LD)', perplexity: validSchemaBlocks.length ? 'PARSED' : 'FALLBACK', bing_copilot: validSchemaBlocks.length ? 'REQUIRED_CARD' : 'OBSTRUCTED', chatgpt: validSchemaBlocks.length ? 'GROUNDED' : 'PLAIN_TEXT' },
248
+ { capability: 'RAG Section Chunkability', perplexity: wordsPerHeading <= 400 ? 'OPTIMAL' : 'LONG', bing_copilot: 'STANDARD', chatgpt: wordsPerHeading <= 400 ? 'OPTIMAL' : 'SEVERED' },
249
+ { capability: 'Crawler Directives', perplexity: perplexityBotBlocked ? 'BLOCKED' : 'ALLOWED', bing_copilot: bingbotBlocked ? 'BLOCKED' : 'ALLOWED', chatgpt: oaiBlocked ? 'BLOCKED' : 'ALLOWED' },
250
+ ],
251
+ recommendations,
252
+ };
253
+ }
@@ -0,0 +1,173 @@
1
+ import { parse as parseHtml } from 'node-html-parser';
2
+
3
+ const QUESTION_RX = /^(?:how|what|why|when|where|who|can|is|does|which|should)\b|\?$/i;
4
+ const HOWTO_TITLE_RX = /^(?:how\s+to|how\s+do\s+i|step-by-step|guide\s+to)\b/i;
5
+
6
+ /**
7
+ * Scan audited site pages for FAQPage, Speakable, and HowTo markup opportunities.
8
+ */
9
+ export function recommendSchemaForPages(pages = []) {
10
+ const recommendations = {
11
+ faq: [],
12
+ speakable: [],
13
+ howto: [],
14
+ };
15
+
16
+ for (const page of pages) {
17
+ if (!page || page.status !== 200) continue;
18
+ const html = page.html || '';
19
+ const root = parseHtml(html);
20
+ const existingTypes = new Set(
21
+ (page.jsonLd || []).flatMap((j) => j.blocks || []).flatMap((b) => [].concat(b['@type'] || []))
22
+ );
23
+
24
+ // -------------------------------------------------------------
25
+ // 1. FAQPage Recommendation
26
+ // -------------------------------------------------------------
27
+ if (!existingTypes.has('FAQPage')) {
28
+ const qas = [];
29
+
30
+ // Pattern A: <details><summary>Q</summary><p>A</p></details>
31
+ for (const details of root.querySelectorAll('details')) {
32
+ const summary = details.querySelector('summary');
33
+ if (summary) {
34
+ const questionText = summary.text.trim();
35
+ const clone = parseHtml(details.innerHTML);
36
+ const s = clone.querySelector('summary');
37
+ if (s) s.remove();
38
+ const answerText = clone.text.replace(/\s+/g, ' ').trim();
39
+ if (questionText && answerText) {
40
+ qas.push({ question: questionText, answer: answerText });
41
+ }
42
+ }
43
+ }
44
+
45
+ // Pattern B: Question headings followed by paragraphs
46
+ if (qas.length < 2) {
47
+ const headings = root.querySelectorAll('h2, h3');
48
+ for (const h of headings) {
49
+ const hText = h.text.trim();
50
+ if (QUESTION_RX.test(hText)) {
51
+ // Find next sibling paragraph
52
+ let next = h.nextElementSibling;
53
+ let answerText = '';
54
+ while (next && !['H1', 'H2', 'H3', 'H4'].includes(next.tagName)) {
55
+ if (['P', 'UL', 'OL', 'DIV'].includes(next.tagName)) {
56
+ answerText += (answerText ? ' ' : '') + next.text.replace(/\s+/g, ' ').trim();
57
+ if (answerText.length > 60) break;
58
+ }
59
+ next = next.nextElementSibling;
60
+ }
61
+ if (hText && answerText) {
62
+ qas.push({ question: hText, answer: answerText });
63
+ }
64
+ }
65
+ }
66
+ }
67
+
68
+ if (qas.length >= 2) {
69
+ recommendations.faq.push({
70
+ target_url: page.url,
71
+ source_file: page.sourceFile || null,
72
+ schema_type: 'FAQPage',
73
+ rationale: `Detected ${qas.length} visible question-and-answer pairs without FAQPage schema.`,
74
+ jsonld: {
75
+ '@context': 'https://schema.org',
76
+ '@type': 'FAQPage',
77
+ mainEntity: qas.slice(0, 10).map((qa) => ({
78
+ '@type': 'Question',
79
+ name: qa.question,
80
+ acceptedAnswer: {
81
+ '@type': 'Answer',
82
+ text: qa.answer,
83
+ },
84
+ })),
85
+ },
86
+ });
87
+ }
88
+ }
89
+
90
+ // -------------------------------------------------------------
91
+ // 2. Speakable Recommendation (SpeakableSpecification)
92
+ // -------------------------------------------------------------
93
+ const hasExistingSpeakable = (page.jsonLd || []).some((j) =>
94
+ (j.blocks || []).some((b) => b.speakable)
95
+ );
96
+
97
+ if (!hasExistingSpeakable) {
98
+ const speakableSelectors = [];
99
+ if (root.querySelector('h1')) speakableSelectors.push('h1');
100
+ if (root.querySelector('#summary, .executive-summary, .summary, .key-takeaways')) {
101
+ speakableSelectors.push(root.querySelector('#summary') ? '#summary' : '.executive-summary');
102
+ } else if (root.querySelector('article > p:first-of-type, main > p:first-of-type, p:first-of-type')) {
103
+ speakableSelectors.push('article > p:first-of-type');
104
+ }
105
+
106
+ if (speakableSelectors.length >= 2) {
107
+ recommendations.speakable.push({
108
+ target_url: page.url,
109
+ source_file: page.sourceFile || null,
110
+ schema_type: 'SpeakableSpecification',
111
+ rationale: 'Page has clear title and executive lead passage suitable for smart speaker / assistant audio playback.',
112
+ jsonld: {
113
+ '@context': 'https://schema.org',
114
+ '@type': 'WebPage',
115
+ '@id': `${page.url}#webpage`,
116
+ url: page.url,
117
+ name: page.title || 'Page',
118
+ speakable: {
119
+ '@type': 'SpeakableSpecification',
120
+ cssSelector: speakableSelectors,
121
+ },
122
+ },
123
+ });
124
+ }
125
+ }
126
+
127
+ // -------------------------------------------------------------
128
+ // 3. HowTo Recommendation
129
+ // -------------------------------------------------------------
130
+ if (!existingTypes.has('HowTo')) {
131
+ const pageTitle = page.title || '';
132
+ const h1s = (page.headings || []).filter((h) => h.level === 1).map((h) => h.text);
133
+ const isHowToPage = HOWTO_TITLE_RX.test(pageTitle) || h1s.some((h) => HOWTO_TITLE_RX.test(h));
134
+
135
+ if (isHowToPage) {
136
+ const steps = [];
137
+ // Look for ordered list items or step subheadings
138
+ const olItems = root.querySelectorAll('ol > li');
139
+ if (olItems.length >= 2) {
140
+ olItems.forEach((li, idx) => {
141
+ const stepText = li.text.replace(/\s+/g, ' ').trim();
142
+ if (stepText) {
143
+ steps.push({
144
+ '@type': 'HowToStep',
145
+ position: idx + 1,
146
+ name: `Step ${idx + 1}`,
147
+ text: stepText,
148
+ });
149
+ }
150
+ });
151
+ }
152
+
153
+ if (steps.length >= 2) {
154
+ recommendations.howto.push({
155
+ target_url: page.url,
156
+ source_file: page.sourceFile || null,
157
+ schema_type: 'HowTo',
158
+ rationale: `Procedural instructional guide detected with ${steps.length} ordered steps.`,
159
+ jsonld: {
160
+ '@context': 'https://schema.org',
161
+ '@type': 'HowTo',
162
+ name: h1s[0] || pageTitle,
163
+ description: page.paragraphs?.[0] || 'Procedural guide and instructions.',
164
+ step: steps,
165
+ },
166
+ });
167
+ }
168
+ }
169
+ }
170
+ }
171
+
172
+ return recommendations;
173
+ }