@fullstackdatasolutions/articles 1.2.3 → 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +46 -0
- package/README.md +313 -1
- package/dist/index.cjs +308 -79
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +267 -16
- package/dist/index.d.ts +267 -16
- package/dist/index.js +300 -79
- package/dist/index.js.map +1 -1
- package/dist/nextjs.cjs +325 -31
- package/dist/nextjs.cjs.map +1 -1
- package/dist/nextjs.d.cts +179 -2
- package/dist/nextjs.d.ts +179 -2
- package/dist/nextjs.js +325 -31
- package/dist/nextjs.js.map +1 -1
- package/dist/server.cjs +660 -50
- package/dist/server.cjs.map +1 -1
- package/dist/server.d.cts +333 -12
- package/dist/server.d.ts +333 -12
- package/dist/server.js +645 -50
- package/dist/server.js.map +1 -1
- package/package.json +1 -1
- package/src/ArticleAnswer.tsx +35 -0
- package/src/ArticleSchemas.tsx +263 -23
- package/src/AuthorArticlesPage.tsx +38 -8
- package/src/__tests__/ArticleAnswer.test.tsx +25 -0
- package/src/__tests__/ArticleSchemas.test.tsx +516 -0
- package/src/__tests__/AuthorArticlesPage.test.tsx +76 -0
- package/src/__tests__/authorUtils.test.ts +50 -0
- package/src/__tests__/markdown.test.ts +77 -1
- package/src/__tests__/nextjs.test.ts +31 -15
- package/src/__tests__/seoUtils.test.ts +279 -0
- package/src/__tests__/server-articles.test.ts +434 -1
- package/src/__tests__/validateArticles.test.ts +167 -6
- package/src/articleTypes.ts +57 -0
- package/src/articlesConfig.ts +176 -1
- package/src/authorUtils.ts +19 -1
- package/src/errorReporting.ts +1 -0
- package/src/index.ts +17 -1
- package/src/markdown.ts +100 -1
- package/src/nextjs.ts +7 -4
- package/src/seoUtils.ts +247 -26
- package/src/server-articles.ts +385 -25
- package/src/server.ts +35 -4
- package/src/validateArticles.ts +157 -12
|
@@ -1,7 +1,13 @@
|
|
|
1
1
|
import fs from 'node:fs'
|
|
2
2
|
import path from 'node:path'
|
|
3
3
|
import {
|
|
4
|
+
buildMarkdownTwinHeader,
|
|
4
5
|
categoryToSlug,
|
|
6
|
+
getAuthorMarkdown,
|
|
7
|
+
getCategoryMarkdown,
|
|
8
|
+
getMarkdownTwinResponse,
|
|
9
|
+
getSeriesMarkdown,
|
|
10
|
+
matchAiCrawler,
|
|
5
11
|
getAdjacentArticles,
|
|
6
12
|
getAiRobotsTxtRules,
|
|
7
13
|
getAllArticles,
|
|
@@ -26,6 +32,7 @@ import {
|
|
|
26
32
|
searchArticles,
|
|
27
33
|
} from '../server-articles'
|
|
28
34
|
import type { ArticlesConfig } from '../articlesConfig'
|
|
35
|
+
import type { Article } from '../articleTypes'
|
|
29
36
|
import { setArticlesErrorHandler } from '../errorReporting'
|
|
30
37
|
import { markdownToHtml } from '../markdown'
|
|
31
38
|
|
|
@@ -34,6 +41,13 @@ jest.mock('node:fs')
|
|
|
34
41
|
jest.mock('../markdown', () => ({
|
|
35
42
|
markdownToHtml: jest.fn(async () => '<p>content</p>'),
|
|
36
43
|
extractToc: jest.fn(async () => []),
|
|
44
|
+
// Real derivation logic is covered in markdown.test.ts; markdown.ts's
|
|
45
|
+
// unified deps are ESM-only, so this suite only asserts the wiring.
|
|
46
|
+
deriveFaqFromHeadings: jest.fn((markdown: string) =>
|
|
47
|
+
markdown.includes('## What is initiative?')
|
|
48
|
+
? [{ question: 'What is initiative?', answer: 'Turn order for combat.' }]
|
|
49
|
+
: []
|
|
50
|
+
),
|
|
37
51
|
}))
|
|
38
52
|
jest.mock('reading-time', () => () => ({ text: '2 min read', words: 42 }))
|
|
39
53
|
|
|
@@ -171,6 +185,63 @@ describe('getArticleMetadata - frontmatter parsing', () => {
|
|
|
171
185
|
expect(article!.faq![0]).toEqual({ question: 'What is this?', answer: 'A test.' })
|
|
172
186
|
})
|
|
173
187
|
|
|
188
|
+
it('parses answer, about, and citation from frontmatter', async () => {
|
|
189
|
+
setupArticleMock(
|
|
190
|
+
[
|
|
191
|
+
'answer: Roll a d20 and add your modifier.\n',
|
|
192
|
+
'about:\n - name: Pathfinder\n sameAs: https://www.wikidata.org/wiki/Q1194077\n - Combat\n',
|
|
193
|
+
'citation:\n - name: Core Rulebook\n url: https://example.com/crb\n - Errata\n',
|
|
194
|
+
].join('')
|
|
195
|
+
)
|
|
196
|
+
const article = await getArticleMetadata('test-slug')
|
|
197
|
+
expect(article!.answer).toBe('Roll a d20 and add your modifier.')
|
|
198
|
+
expect(article!.about).toEqual([
|
|
199
|
+
{ name: 'Pathfinder', sameAs: 'https://www.wikidata.org/wiki/Q1194077' },
|
|
200
|
+
{ name: 'Combat' },
|
|
201
|
+
])
|
|
202
|
+
expect(article!.citation).toEqual([
|
|
203
|
+
{ name: 'Core Rulebook', url: 'https://example.com/crb' },
|
|
204
|
+
{ name: 'Errata' },
|
|
205
|
+
])
|
|
206
|
+
})
|
|
207
|
+
|
|
208
|
+
it('drops malformed and blank about/citation entries', async () => {
|
|
209
|
+
setupArticleMock('about:\n - ""\n - 42\n - name: ""\ncitation: not-a-list\n')
|
|
210
|
+
const article = await getArticleMetadata('test-slug')
|
|
211
|
+
expect(article!.about).toBeUndefined()
|
|
212
|
+
expect(article!.citation).toBeUndefined()
|
|
213
|
+
})
|
|
214
|
+
|
|
215
|
+
it('derives faq from question headings only when deriveFaqFromHeadings is enabled', async () => {
|
|
216
|
+
const body = ['## What is initiative?', '', 'Turn order for combat.'].join('\n')
|
|
217
|
+
const config: ArticlesConfig = {
|
|
218
|
+
siteUrl: 'https://example.com',
|
|
219
|
+
siteName: 'Example',
|
|
220
|
+
deriveFaqFromHeadings: true,
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
setupArticleMock('', body)
|
|
224
|
+
expect((await getArticleMetadata('test-slug'))!.faq).toBeUndefined()
|
|
225
|
+
|
|
226
|
+
setupArticleMock('', body)
|
|
227
|
+
expect((await getArticleMetadata('test-slug', config))!.faq).toEqual([
|
|
228
|
+
{ question: 'What is initiative?', answer: 'Turn order for combat.' },
|
|
229
|
+
])
|
|
230
|
+
})
|
|
231
|
+
|
|
232
|
+
it('prefers explicit faq frontmatter over derived headings', async () => {
|
|
233
|
+
setupArticleMock(
|
|
234
|
+
'faq:\n - question: "Declared?"\n answer: "Yes."\n',
|
|
235
|
+
['## What is initiative?', '', 'Turn order for combat.'].join('\n')
|
|
236
|
+
)
|
|
237
|
+
const article = await getArticleMetadata('test-slug', {
|
|
238
|
+
siteUrl: 'https://example.com',
|
|
239
|
+
siteName: 'Example',
|
|
240
|
+
deriveFaqFromHeadings: true,
|
|
241
|
+
})
|
|
242
|
+
expect(article!.faq).toEqual([{ question: 'Declared?', answer: 'Yes.' }])
|
|
243
|
+
})
|
|
244
|
+
|
|
174
245
|
it('parses howTo from frontmatter', async () => {
|
|
175
246
|
setupArticleMock(
|
|
176
247
|
'howTo:\n - name: "Step 1"\n text: "Do this."\n - name: "Step 2"\n text: "Then that."\n'
|
|
@@ -924,7 +995,20 @@ describe('AI markdown helpers', () => {
|
|
|
924
995
|
})
|
|
925
996
|
expect(allowed.status).toBe(200)
|
|
926
997
|
expect(allowed.headers.get('Content-Type')).toBe('text/markdown; charset=utf-8')
|
|
927
|
-
await expect(allowed.text()).resolves.toBe(
|
|
998
|
+
await expect(allowed.text()).resolves.toBe(
|
|
999
|
+
[
|
|
1000
|
+
'> Excerpt for allowed',
|
|
1001
|
+
'',
|
|
1002
|
+
'Source: https://example.com/articles/allowed',
|
|
1003
|
+
'Published: 2025-01-01',
|
|
1004
|
+
'Author: Test Author',
|
|
1005
|
+
'Site: Example',
|
|
1006
|
+
'',
|
|
1007
|
+
'---',
|
|
1008
|
+
'',
|
|
1009
|
+
'# Allowed',
|
|
1010
|
+
].join('\n')
|
|
1011
|
+
)
|
|
928
1012
|
|
|
929
1013
|
const blocked = await getArticleMarkdownResponse('blocked', {
|
|
930
1014
|
siteUrl: 'https://example.com',
|
|
@@ -933,6 +1017,27 @@ describe('AI markdown helpers', () => {
|
|
|
933
1017
|
expect(blocked.status).toBe(404)
|
|
934
1018
|
})
|
|
935
1019
|
|
|
1020
|
+
it('links the markdown twin back to its HTML article, not to itself', async () => {
|
|
1021
|
+
setupArticleTreeMock([
|
|
1022
|
+
{
|
|
1023
|
+
slug: 'allowed',
|
|
1024
|
+
frontmatter: 'date: 2025-01-01\naiCrawl: true\n',
|
|
1025
|
+
body: '# Allowed',
|
|
1026
|
+
},
|
|
1027
|
+
])
|
|
1028
|
+
|
|
1029
|
+
const response = await getArticleMarkdownResponse('allowed', {
|
|
1030
|
+
siteUrl: 'https://example.com/',
|
|
1031
|
+
siteName: 'Example',
|
|
1032
|
+
})
|
|
1033
|
+
|
|
1034
|
+
expect(response.headers.get('Link')).toBe(
|
|
1035
|
+
'<https://example.com/articles/allowed>; rel="canonical"'
|
|
1036
|
+
)
|
|
1037
|
+
expect(response.headers.get('Link')).not.toContain('.md')
|
|
1038
|
+
expect(response.headers.get('Link')).not.toContain('alternate')
|
|
1039
|
+
})
|
|
1040
|
+
|
|
936
1041
|
it('generates robots rules for AI crawlers and blocked articles only', async () => {
|
|
937
1042
|
setupArticleTreeMock([
|
|
938
1043
|
{ slug: 'allowed', frontmatter: 'date: 2025-01-01\naiCrawl: true\n' },
|
|
@@ -944,6 +1049,82 @@ describe('AI markdown helpers', () => {
|
|
|
944
1049
|
expect(rules).toContain('Disallow: /articles/blocked')
|
|
945
1050
|
expect(rules).not.toContain('Disallow: /articles/allowed')
|
|
946
1051
|
})
|
|
1052
|
+
|
|
1053
|
+
it('opts every article in when aiCrawlDefault is true, without overriding explicit false', async () => {
|
|
1054
|
+
setupArticleTreeMock([
|
|
1055
|
+
{ slug: 'inherits', frontmatter: 'date: 2025-01-01\n' },
|
|
1056
|
+
{ slug: 'opted-out', frontmatter: 'date: 2025-01-01\naiCrawl: false\n' },
|
|
1057
|
+
])
|
|
1058
|
+
const config: ArticlesConfig = {
|
|
1059
|
+
siteUrl: 'https://example.com',
|
|
1060
|
+
siteName: 'Example',
|
|
1061
|
+
aiCrawlDefault: true,
|
|
1062
|
+
}
|
|
1063
|
+
|
|
1064
|
+
expect((await getArticleMetadata('inherits', config))!.aiCrawl).toBe(true)
|
|
1065
|
+
expect((await getArticleMetadata('opted-out', config))!.aiCrawl).toBe(false)
|
|
1066
|
+
await expect(getArticleMarkdown('inherits', config)).resolves.toContain('Article body content.')
|
|
1067
|
+
await expect(getArticleMarkdown('opted-out', config)).resolves.toBeNull()
|
|
1068
|
+
|
|
1069
|
+
const rules = await getAiRobotsTxtRules(config)
|
|
1070
|
+
expect(rules).toContain('Disallow: /articles/opted-out')
|
|
1071
|
+
expect(rules).not.toContain('Disallow: /articles/inherits')
|
|
1072
|
+
})
|
|
1073
|
+
|
|
1074
|
+
it('keeps a distinct title as its own H1 and includes lastmod when present', () => {
|
|
1075
|
+
const header = buildMarkdownTwinHeader(
|
|
1076
|
+
{
|
|
1077
|
+
slug: 'guides/first-session',
|
|
1078
|
+
title: 'Running Your First Session',
|
|
1079
|
+
excerpt: 'A short summary.',
|
|
1080
|
+
date: '2025-01-01',
|
|
1081
|
+
lastmod: '2025-06-01',
|
|
1082
|
+
author: 'Test Author',
|
|
1083
|
+
} as Article,
|
|
1084
|
+
{ siteUrl: 'https://example.com/', siteName: 'Example' },
|
|
1085
|
+
'Body text that does not repeat the title.'
|
|
1086
|
+
)
|
|
1087
|
+
|
|
1088
|
+
expect(header).toBe(
|
|
1089
|
+
[
|
|
1090
|
+
'# Running Your First Session',
|
|
1091
|
+
'',
|
|
1092
|
+
'> A short summary.',
|
|
1093
|
+
'',
|
|
1094
|
+
'Source: https://example.com/articles/guides/first-session',
|
|
1095
|
+
'Published: 2025-01-01',
|
|
1096
|
+
'Updated: 2025-06-01',
|
|
1097
|
+
'Author: Test Author',
|
|
1098
|
+
'Site: Example',
|
|
1099
|
+
'',
|
|
1100
|
+
'---',
|
|
1101
|
+
'',
|
|
1102
|
+
'',
|
|
1103
|
+
].join('\n')
|
|
1104
|
+
)
|
|
1105
|
+
})
|
|
1106
|
+
|
|
1107
|
+
it('omits the author line when showAuthor is false', () => {
|
|
1108
|
+
const header = buildMarkdownTwinHeader(
|
|
1109
|
+
{ slug: 'a', title: 'A', excerpt: '', author: 'Test Author' } as Article,
|
|
1110
|
+
{ siteUrl: 'https://example.com', siteName: 'Example', showAuthor: false },
|
|
1111
|
+
'Body.'
|
|
1112
|
+
)
|
|
1113
|
+
expect(header).not.toContain('Author:')
|
|
1114
|
+
})
|
|
1115
|
+
|
|
1116
|
+
it('serves the bare body when markdownTwinHeader is false', async () => {
|
|
1117
|
+
setupArticleTreeMock([
|
|
1118
|
+
{ slug: 'allowed', frontmatter: 'date: 2025-01-01\naiCrawl: true\n', body: '# Allowed' },
|
|
1119
|
+
])
|
|
1120
|
+
|
|
1121
|
+
const response = await getArticleMarkdownResponse('allowed', {
|
|
1122
|
+
siteUrl: 'https://example.com',
|
|
1123
|
+
siteName: 'Example',
|
|
1124
|
+
markdownTwinHeader: false,
|
|
1125
|
+
})
|
|
1126
|
+
await expect(response.text()).resolves.toBe('\n# Allowed')
|
|
1127
|
+
})
|
|
947
1128
|
})
|
|
948
1129
|
|
|
949
1130
|
describe('article path helpers', () => {
|
|
@@ -1161,3 +1342,255 @@ describe('getRelatedContent', () => {
|
|
|
1161
1342
|
expect(result.articles.map((a) => a.slug)).toEqual(['two'])
|
|
1162
1343
|
})
|
|
1163
1344
|
})
|
|
1345
|
+
|
|
1346
|
+
describe('markdown twins for listing surfaces', () => {
|
|
1347
|
+
const config: ArticlesConfig = {
|
|
1348
|
+
siteUrl: 'https://example.com/',
|
|
1349
|
+
siteName: 'Example',
|
|
1350
|
+
aiCrawlDefault: true,
|
|
1351
|
+
categoryDescriptions: { campaigns: { short: 'Short.', long: 'The long description.' } },
|
|
1352
|
+
authors: {
|
|
1353
|
+
'jane-doe': {
|
|
1354
|
+
name: 'Jane Doe',
|
|
1355
|
+
slug: 'jane-doe',
|
|
1356
|
+
bio: 'Writes about campaigns.',
|
|
1357
|
+
promise: 'Helping organizers run better campaigns.',
|
|
1358
|
+
servesWho: ['Organizers'],
|
|
1359
|
+
knowsAbout: ['Campaigns'],
|
|
1360
|
+
credentials: ['10 years in the field'],
|
|
1361
|
+
proof: [{ claim: '500 doors knocked', url: 'https://example.com/proof' }],
|
|
1362
|
+
originStory: [{ heading: 'How I started', paragraphs: ['A paragraph.'] }],
|
|
1363
|
+
},
|
|
1364
|
+
},
|
|
1365
|
+
}
|
|
1366
|
+
|
|
1367
|
+
beforeEach(() => {
|
|
1368
|
+
jest.clearAllMocks()
|
|
1369
|
+
})
|
|
1370
|
+
|
|
1371
|
+
it('builds a category twin with description, source lines, and .md links', async () => {
|
|
1372
|
+
setupArticleTreeMock([{ slug: 'one', frontmatter: 'date: 2025-01-01\ntags: [campaigns]\n' }])
|
|
1373
|
+
const markdown = await getCategoryMarkdown('campaigns', config)
|
|
1374
|
+
expect(markdown).toContain('# campaigns')
|
|
1375
|
+
expect(markdown).toContain('The long description.')
|
|
1376
|
+
expect(markdown).toContain('Source: https://example.com')
|
|
1377
|
+
expect(markdown).toContain('Site: Example')
|
|
1378
|
+
expect(markdown).toContain('- [one](https://example.com/articles/one.md): Excerpt for one')
|
|
1379
|
+
})
|
|
1380
|
+
|
|
1381
|
+
it('returns null for a category with no articles', async () => {
|
|
1382
|
+
setupArticleTreeMock([{ slug: 'one', frontmatter: 'date: 2025-01-01\ntags: [other]\n' }])
|
|
1383
|
+
await expect(getCategoryMarkdown('campaigns', config)).resolves.toBeNull()
|
|
1384
|
+
})
|
|
1385
|
+
|
|
1386
|
+
it('omits articles that are not aiCrawl-eligible', async () => {
|
|
1387
|
+
setupArticleTreeMock([
|
|
1388
|
+
{ slug: 'shown', frontmatter: 'date: 2025-01-01\ntags: [campaigns]\n' },
|
|
1389
|
+
{ slug: 'hidden', frontmatter: 'date: 2025-01-01\ntags: [campaigns]\naiCrawl: false\n' },
|
|
1390
|
+
])
|
|
1391
|
+
const markdown = await getCategoryMarkdown('campaigns', config)
|
|
1392
|
+
expect(markdown).toContain('shown')
|
|
1393
|
+
expect(markdown).not.toContain('hidden')
|
|
1394
|
+
})
|
|
1395
|
+
|
|
1396
|
+
it('builds an author twin carrying bio, promise, credentials, proof, and background', async () => {
|
|
1397
|
+
setupArticleTreeMock([
|
|
1398
|
+
{ slug: 'one', frontmatter: 'date: 2025-01-01\nauthors:\n - jane-doe\n' },
|
|
1399
|
+
])
|
|
1400
|
+
const markdown = await getAuthorMarkdown('jane-doe', config)
|
|
1401
|
+
expect(markdown).toContain('# Jane Doe')
|
|
1402
|
+
expect(markdown).toContain('Helping organizers run better campaigns.')
|
|
1403
|
+
expect(markdown).toContain('Writes about campaigns.')
|
|
1404
|
+
expect(markdown).toContain('Writes for: Organizers')
|
|
1405
|
+
expect(markdown).toContain('Writes about: Campaigns')
|
|
1406
|
+
expect(markdown).toContain('## Stated experience')
|
|
1407
|
+
expect(markdown).toContain('- 10 years in the field')
|
|
1408
|
+
expect(markdown).toContain('- [500 doors knocked](https://example.com/proof)')
|
|
1409
|
+
expect(markdown).toContain('### How I started')
|
|
1410
|
+
})
|
|
1411
|
+
|
|
1412
|
+
it('returns null for an unknown author', async () => {
|
|
1413
|
+
setupArticleTreeMock([{ slug: 'one', frontmatter: 'date: 2025-01-01\n' }])
|
|
1414
|
+
await expect(getAuthorMarkdown('nobody', config)).resolves.toBeNull()
|
|
1415
|
+
})
|
|
1416
|
+
|
|
1417
|
+
it('builds a series twin and returns null for an empty series', async () => {
|
|
1418
|
+
setupArticleTreeMock([
|
|
1419
|
+
{
|
|
1420
|
+
slug: 'one',
|
|
1421
|
+
frontmatter: 'date: 2025-01-01\nseries: New GM Path\nseriesSlug: new-gm\nseriesOrder: 1\n',
|
|
1422
|
+
},
|
|
1423
|
+
])
|
|
1424
|
+
const markdown = await getSeriesMarkdown('new-gm', config)
|
|
1425
|
+
expect(markdown).toContain('# New GM Path')
|
|
1426
|
+
expect(markdown).toContain('- [one](https://example.com/articles/one.md)')
|
|
1427
|
+
await expect(getSeriesMarkdown('missing', config)).resolves.toBeNull()
|
|
1428
|
+
})
|
|
1429
|
+
|
|
1430
|
+
it('renders a placeholder when every article in a listing is blocked', async () => {
|
|
1431
|
+
setupArticleTreeMock([
|
|
1432
|
+
{ slug: 'one', frontmatter: 'date: 2025-01-01\ntags: [campaigns]\naiCrawl: false\n' },
|
|
1433
|
+
])
|
|
1434
|
+
expect(await getCategoryMarkdown('campaigns', config)).toContain('_No articles available._')
|
|
1435
|
+
})
|
|
1436
|
+
})
|
|
1437
|
+
|
|
1438
|
+
describe('getMarkdownTwinResponse', () => {
|
|
1439
|
+
const config: ArticlesConfig = {
|
|
1440
|
+
siteUrl: 'https://example.com',
|
|
1441
|
+
siteName: 'Example',
|
|
1442
|
+
aiCrawlDefault: true,
|
|
1443
|
+
}
|
|
1444
|
+
|
|
1445
|
+
beforeEach(() => {
|
|
1446
|
+
jest.clearAllMocks()
|
|
1447
|
+
})
|
|
1448
|
+
|
|
1449
|
+
it('dispatches category, author, and series prefixes to the listing twins', async () => {
|
|
1450
|
+
setupArticleTreeMock([{ slug: 'one', frontmatter: 'date: 2025-01-01\ntags: [campaigns]\n' }])
|
|
1451
|
+
const response = await getMarkdownTwinResponse('category/campaigns', config)
|
|
1452
|
+
expect(response.status).toBe(200)
|
|
1453
|
+
expect(response.headers.get('Content-Type')).toBe('text/markdown; charset=utf-8')
|
|
1454
|
+
await expect(response.text()).resolves.toContain('# campaigns')
|
|
1455
|
+
})
|
|
1456
|
+
|
|
1457
|
+
it('404s a listing path that resolves to nothing', async () => {
|
|
1458
|
+
setupArticleTreeMock([{ slug: 'one', frontmatter: 'date: 2025-01-01\n' }])
|
|
1459
|
+
const response = await getMarkdownTwinResponse('category/missing', config)
|
|
1460
|
+
expect(response.status).toBe(404)
|
|
1461
|
+
})
|
|
1462
|
+
|
|
1463
|
+
it('falls through to the article handler for a non-listing slug', async () => {
|
|
1464
|
+
setupArticleTreeMock([{ slug: 'category', frontmatter: 'date: 2025-01-01\n', body: '# Body' }])
|
|
1465
|
+
const response = await getMarkdownTwinResponse('category', config)
|
|
1466
|
+
expect(response.status).toBe(200)
|
|
1467
|
+
await expect(response.text()).resolves.toContain('# Body')
|
|
1468
|
+
})
|
|
1469
|
+
})
|
|
1470
|
+
|
|
1471
|
+
describe('AI crawl telemetry', () => {
|
|
1472
|
+
beforeEach(() => {
|
|
1473
|
+
jest.clearAllMocks()
|
|
1474
|
+
})
|
|
1475
|
+
|
|
1476
|
+
it('matches known crawlers case-insensitively and returns null otherwise', () => {
|
|
1477
|
+
expect(matchAiCrawler('Mozilla/5.0 (compatible; GPTBot/1.0)')).toBe('GPTBot')
|
|
1478
|
+
expect(matchAiCrawler('claudebot/1.0')).toBe('ClaudeBot')
|
|
1479
|
+
expect(matchAiCrawler('Mozilla/5.0 Chrome/120')).toBeNull()
|
|
1480
|
+
expect(matchAiCrawler('')).toBeNull()
|
|
1481
|
+
})
|
|
1482
|
+
|
|
1483
|
+
it('reports the crawler, slug, and raw user agent on a twin fetch', async () => {
|
|
1484
|
+
const onAiCrawl = jest.fn()
|
|
1485
|
+
setupArticleTreeMock([{ slug: 'one', frontmatter: 'date: 2025-01-01\naiCrawl: true\n' }])
|
|
1486
|
+
|
|
1487
|
+
await getMarkdownTwinResponse(
|
|
1488
|
+
'one',
|
|
1489
|
+
{ siteUrl: 'https://example.com', siteName: 'Example', onAiCrawl },
|
|
1490
|
+
{ headers: { get: () => 'Mozilla/5.0 (compatible; PerplexityBot/1.0)' } }
|
|
1491
|
+
)
|
|
1492
|
+
|
|
1493
|
+
expect(onAiCrawl).toHaveBeenCalledWith({
|
|
1494
|
+
slug: 'one',
|
|
1495
|
+
crawler: 'PerplexityBot',
|
|
1496
|
+
userAgent: 'Mozilla/5.0 (compatible; PerplexityBot/1.0)',
|
|
1497
|
+
})
|
|
1498
|
+
})
|
|
1499
|
+
|
|
1500
|
+
it('reports unknown for an unrecognized agent and an absent header', async () => {
|
|
1501
|
+
const onAiCrawl = jest.fn()
|
|
1502
|
+
setupArticleTreeMock([{ slug: 'one', frontmatter: 'date: 2025-01-01\naiCrawl: true\n' }])
|
|
1503
|
+
await getMarkdownTwinResponse('one', {
|
|
1504
|
+
siteUrl: 'https://example.com',
|
|
1505
|
+
siteName: 'Example',
|
|
1506
|
+
onAiCrawl,
|
|
1507
|
+
})
|
|
1508
|
+
expect(onAiCrawl).toHaveBeenCalledWith({ slug: 'one', crawler: 'unknown', userAgent: '' })
|
|
1509
|
+
})
|
|
1510
|
+
|
|
1511
|
+
it('does not let a throwing handler break the response', async () => {
|
|
1512
|
+
const reported: string[] = []
|
|
1513
|
+
setArticlesErrorHandler((report) => reported.push(report.code))
|
|
1514
|
+
setupArticleTreeMock([{ slug: 'one', frontmatter: 'date: 2025-01-01\naiCrawl: true\n' }])
|
|
1515
|
+
|
|
1516
|
+
const response = await getMarkdownTwinResponse('one', {
|
|
1517
|
+
siteUrl: 'https://example.com',
|
|
1518
|
+
siteName: 'Example',
|
|
1519
|
+
onAiCrawl: () => {
|
|
1520
|
+
throw new Error('telemetry down')
|
|
1521
|
+
},
|
|
1522
|
+
})
|
|
1523
|
+
|
|
1524
|
+
expect(response.status).toBe(200)
|
|
1525
|
+
expect(reported).toContain('ai-crawl-handler-failed')
|
|
1526
|
+
setArticlesErrorHandler(() => undefined)
|
|
1527
|
+
})
|
|
1528
|
+
|
|
1529
|
+
it('fires for listing twins too', async () => {
|
|
1530
|
+
const onAiCrawl = jest.fn()
|
|
1531
|
+
setupArticleTreeMock([
|
|
1532
|
+
{ slug: 'one', frontmatter: 'date: 2025-01-01\ntags: [campaigns]\naiCrawl: true\n' },
|
|
1533
|
+
])
|
|
1534
|
+
await getMarkdownTwinResponse(
|
|
1535
|
+
'category/campaigns',
|
|
1536
|
+
{ siteUrl: 'https://example.com', siteName: 'Example', onAiCrawl },
|
|
1537
|
+
{ headers: { get: () => 'CCBot/2.0' } }
|
|
1538
|
+
)
|
|
1539
|
+
expect(onAiCrawl).toHaveBeenCalledWith(
|
|
1540
|
+
expect.objectContaining({ slug: 'category/campaigns', crawler: 'CCBot' })
|
|
1541
|
+
)
|
|
1542
|
+
})
|
|
1543
|
+
})
|
|
1544
|
+
|
|
1545
|
+
describe('lastmod fallback and entity registry', () => {
|
|
1546
|
+
beforeEach(() => {
|
|
1547
|
+
jest.clearAllMocks()
|
|
1548
|
+
})
|
|
1549
|
+
|
|
1550
|
+
it('leaves lastmod unset by default and honors explicit frontmatter', async () => {
|
|
1551
|
+
setupArticleMock('date: 2025-01-01\n')
|
|
1552
|
+
expect((await getArticleMetadata('test-slug'))!.lastmod).toBeUndefined()
|
|
1553
|
+
|
|
1554
|
+
setupArticleMock('date: 2025-01-01\nlastmod: 2025-06-01\n')
|
|
1555
|
+
expect((await getArticleMetadata('test-slug'))!.lastmod).toBe('2025-06-01')
|
|
1556
|
+
})
|
|
1557
|
+
|
|
1558
|
+
it('reads file mtime when lastmodFallback is fileMtime', async () => {
|
|
1559
|
+
setupArticleMock('date: 2025-01-01\n')
|
|
1560
|
+
mockedFs.statSync.mockReturnValue({ mtime: new Date('2026-03-04T00:00:00Z') } as fs.Stats)
|
|
1561
|
+
const article = await getArticleMetadata('test-slug', {
|
|
1562
|
+
siteUrl: 'https://example.com',
|
|
1563
|
+
siteName: 'Example',
|
|
1564
|
+
lastmodFallback: 'fileMtime',
|
|
1565
|
+
})
|
|
1566
|
+
expect(article!.lastmod).toBe('2026-03-04')
|
|
1567
|
+
})
|
|
1568
|
+
|
|
1569
|
+
it('falls back to the publish date when statSync throws', async () => {
|
|
1570
|
+
setupArticleMock('date: 2025-01-01\n')
|
|
1571
|
+
mockedFs.statSync.mockImplementation(() => {
|
|
1572
|
+
throw new Error('no such file')
|
|
1573
|
+
})
|
|
1574
|
+
const article = await getArticleMetadata('test-slug', {
|
|
1575
|
+
siteUrl: 'https://example.com',
|
|
1576
|
+
siteName: 'Example',
|
|
1577
|
+
lastmodFallback: 'fileMtime',
|
|
1578
|
+
})
|
|
1579
|
+
expect(article!.lastmod).toBe('2025-01-01')
|
|
1580
|
+
})
|
|
1581
|
+
|
|
1582
|
+
it('resolves about strings through config.entities', async () => {
|
|
1583
|
+
setupArticleMock('about:\n - pathfinder\n - unregistered\n')
|
|
1584
|
+
const article = await getArticleMetadata('test-slug', {
|
|
1585
|
+
siteUrl: 'https://example.com',
|
|
1586
|
+
siteName: 'Example',
|
|
1587
|
+
entities: {
|
|
1588
|
+
pathfinder: { name: 'Pathfinder', sameAs: 'https://www.wikidata.org/wiki/Q1194077' },
|
|
1589
|
+
},
|
|
1590
|
+
})
|
|
1591
|
+
expect(article!.about).toEqual([
|
|
1592
|
+
{ name: 'Pathfinder', sameAs: 'https://www.wikidata.org/wiki/Q1194077' },
|
|
1593
|
+
{ name: 'unregistered' },
|
|
1594
|
+
])
|
|
1595
|
+
})
|
|
1596
|
+
})
|
|
@@ -279,16 +279,177 @@ describe('validateArticles', () => {
|
|
|
279
279
|
)
|
|
280
280
|
})
|
|
281
281
|
|
|
282
|
-
it('warns
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
282
|
+
it('warns on the rendered title length, including the siteName suffix', () => {
|
|
283
|
+
// 45-char title is fine alone but not once ' | Example' is appended
|
|
284
|
+
// under the default template - the rendered string is what is measured.
|
|
285
|
+
const codes = validateArticles(
|
|
286
|
+
[makeArticle({ title: 'x'.repeat(55) })],
|
|
287
|
+
baseConfig
|
|
288
|
+
).warnings.map((i) => i.code)
|
|
289
|
+
expect(codes).toContain('effective-title-too-long')
|
|
290
|
+
})
|
|
291
|
+
|
|
292
|
+
it('warns even when searchTitle is unset - the common case', () => {
|
|
293
|
+
const codes = validateArticles(
|
|
294
|
+
[makeArticle({ title: 'y'.repeat(80), searchTitle: undefined })],
|
|
295
|
+
baseConfig
|
|
296
|
+
).warnings.map((i) => i.code)
|
|
297
|
+
expect(codes).toContain('effective-title-too-long')
|
|
298
|
+
})
|
|
299
|
+
|
|
300
|
+
it('measures searchTitle when it overrides the title', () => {
|
|
301
|
+
const long = validateArticles(
|
|
302
|
+
[makeArticle({ title: 'Short', searchTitle: 'z'.repeat(70) })],
|
|
303
|
+
baseConfig
|
|
304
|
+
).warnings.map((i) => i.code)
|
|
305
|
+
expect(long).toContain('effective-title-too-long')
|
|
306
|
+
|
|
307
|
+
const short = validateArticles(
|
|
308
|
+
[makeArticle({ title: 'x'.repeat(80), searchTitle: 'Short title' })],
|
|
309
|
+
baseConfig
|
|
310
|
+
).warnings.map((i) => i.code)
|
|
311
|
+
expect(short).not.toContain('effective-title-too-long')
|
|
312
|
+
})
|
|
313
|
+
|
|
314
|
+
it('stops warning once titleTemplate drops the siteName suffix', () => {
|
|
315
|
+
const article = makeArticle({ title: 'x'.repeat(55) })
|
|
316
|
+
expect(validateArticles([article], baseConfig).warnings.map((i) => i.code)).toContain(
|
|
317
|
+
'effective-title-too-long'
|
|
286
318
|
)
|
|
319
|
+
expect(
|
|
320
|
+
validateArticles([article], { ...baseConfig, titleTemplate: '{title}' }).warnings.map(
|
|
321
|
+
(i) => i.code
|
|
322
|
+
)
|
|
323
|
+
).not.toContain('effective-title-too-long')
|
|
324
|
+
})
|
|
325
|
+
|
|
326
|
+
it('warns on the rendered meta description, falling back to excerpt', () => {
|
|
327
|
+
const fromExcerpt = validateArticles(
|
|
328
|
+
[makeArticle({ excerpt: 'e'.repeat(200) })],
|
|
329
|
+
baseConfig
|
|
330
|
+
).warnings.map((i) => i.code)
|
|
331
|
+
expect(fromExcerpt).toContain('effective-description-too-long')
|
|
332
|
+
|
|
333
|
+
const overridden = validateArticles(
|
|
334
|
+
[makeArticle({ excerpt: 'e'.repeat(200), searchDescription: 'Short description.' })],
|
|
335
|
+
baseConfig
|
|
336
|
+
).warnings.map((i) => i.code)
|
|
337
|
+
expect(overridden).not.toContain('effective-description-too-long')
|
|
287
338
|
})
|
|
288
339
|
|
|
289
|
-
it('does not warn when
|
|
340
|
+
it('does not warn when the rendered title is within the limit', () => {
|
|
290
341
|
const result = validateArticles([makeArticle({ searchTitle: 'Short title' })], baseConfig)
|
|
291
|
-
expect(result.warnings).
|
|
342
|
+
expect(result.warnings.map((issue) => issue.code)).not.toContain('effective-title-too-long')
|
|
343
|
+
})
|
|
344
|
+
|
|
345
|
+
it('warns when an article offers nothing quotable', () => {
|
|
346
|
+
const codes = validateArticles([makeArticle({})], baseConfig).warnings.map((i) => i.code)
|
|
347
|
+
expect(codes).toContain('no-answer')
|
|
348
|
+
expect(codes).toContain('missing-about')
|
|
349
|
+
})
|
|
350
|
+
|
|
351
|
+
it('accepts an answer, an faq, or a question-shaped h2 as answerable', () => {
|
|
352
|
+
const withAnswer = validateArticles([makeArticle({ answer: 'Yes.' })], baseConfig)
|
|
353
|
+
const withFaq = validateArticles(
|
|
354
|
+
[makeArticle({ faq: [{ question: 'Why?', answer: 'Because.' }] })],
|
|
355
|
+
baseConfig
|
|
356
|
+
)
|
|
357
|
+
const withHeading = validateArticles(
|
|
358
|
+
[makeArticle({ toc: [{ id: 'q', depth: 2, text: 'What is this?' }] })],
|
|
359
|
+
baseConfig
|
|
360
|
+
)
|
|
361
|
+
for (const result of [withAnswer, withFaq, withHeading]) {
|
|
362
|
+
expect(result.warnings.map((i) => i.code)).not.toContain('no-answer')
|
|
363
|
+
}
|
|
364
|
+
})
|
|
365
|
+
|
|
366
|
+
it('warns on thin content and stays quiet above the threshold', () => {
|
|
367
|
+
expect(
|
|
368
|
+
validateArticles([makeArticle({ wordCount: 120 })], baseConfig).warnings.map((i) => i.code)
|
|
369
|
+
).toContain('thin-content')
|
|
370
|
+
expect(
|
|
371
|
+
validateArticles([makeArticle({ wordCount: 1200 })], baseConfig).warnings.map((i) => i.code)
|
|
372
|
+
).not.toContain('thin-content')
|
|
373
|
+
expect(
|
|
374
|
+
validateArticles([makeArticle({})], baseConfig).warnings.map((i) => i.code)
|
|
375
|
+
).not.toContain('thin-content')
|
|
376
|
+
})
|
|
377
|
+
|
|
378
|
+
it('warns on stale content using the injected clock and prefers lastmod', () => {
|
|
379
|
+
const now = new Date('2026-08-22T00:00:00.000Z')
|
|
380
|
+
const stale = validateArticles([makeArticle({ date: '2024-01-01' })], baseConfig, { now })
|
|
381
|
+
expect(stale.warnings.map((i) => i.code)).toContain('stale-content')
|
|
382
|
+
|
|
383
|
+
const refreshed = validateArticles(
|
|
384
|
+
[makeArticle({ date: '2024-01-01', lastmod: '2026-06-01' })],
|
|
385
|
+
baseConfig,
|
|
386
|
+
{ now }
|
|
387
|
+
)
|
|
388
|
+
expect(refreshed.warnings.map((i) => i.code)).not.toContain('stale-content')
|
|
389
|
+
})
|
|
390
|
+
|
|
391
|
+
it('ignores undated and unparseable timestamps in the stale check', () => {
|
|
392
|
+
const now = new Date('2026-08-22T00:00:00.000Z')
|
|
393
|
+
const codes = validateArticles(
|
|
394
|
+
[
|
|
395
|
+
makeArticle({ slug: 'a', date: undefined }),
|
|
396
|
+
makeArticle({ slug: 'b', date: 'not-a-date' }),
|
|
397
|
+
],
|
|
398
|
+
baseConfig,
|
|
399
|
+
{ now }
|
|
400
|
+
).warnings.map((i) => i.code)
|
|
401
|
+
expect(codes).not.toContain('stale-content')
|
|
402
|
+
})
|
|
403
|
+
|
|
404
|
+
it('warns on orphan articles only when bodies are loaded', () => {
|
|
405
|
+
const linked = validateArticles(
|
|
406
|
+
[
|
|
407
|
+
makeArticle({ slug: 'hub', content: 'See [the guide](/articles/spoke).' }),
|
|
408
|
+
makeArticle({ slug: 'spoke', content: 'Standalone body.' }),
|
|
409
|
+
],
|
|
410
|
+
baseConfig
|
|
411
|
+
).warnings.filter((i) => i.code === 'orphan-article')
|
|
412
|
+
expect(linked.map((i) => i.articleSlug)).toEqual(['hub'])
|
|
413
|
+
|
|
414
|
+
const noBodies = validateArticles(
|
|
415
|
+
[makeArticle({ slug: 'hub' }), makeArticle({ slug: 'spoke' })],
|
|
416
|
+
baseConfig
|
|
417
|
+
).warnings.map((i) => i.code)
|
|
418
|
+
expect(noBodies).not.toContain('orphan-article')
|
|
419
|
+
})
|
|
420
|
+
|
|
421
|
+
it('warns on about entries missing from config.entities', () => {
|
|
422
|
+
const config: ArticlesConfig = {
|
|
423
|
+
...baseConfig,
|
|
424
|
+
entities: { pathfinder: { name: 'Pathfinder' } },
|
|
425
|
+
}
|
|
426
|
+
const codes = validateArticles(
|
|
427
|
+
[makeArticle({ about: [{ name: 'PF2e' }] })],
|
|
428
|
+
config
|
|
429
|
+
).warnings.map((i) => i.code)
|
|
430
|
+
expect(codes).toContain('unknown-entity')
|
|
431
|
+
|
|
432
|
+
const known = validateArticles(
|
|
433
|
+
[makeArticle({ about: [{ name: 'Pathfinder' }] })],
|
|
434
|
+
config
|
|
435
|
+
).warnings.map((i) => i.code)
|
|
436
|
+
expect(known).not.toContain('unknown-entity')
|
|
437
|
+
})
|
|
438
|
+
|
|
439
|
+
it('accepts an unregistered entity that carries its own sameAs', () => {
|
|
440
|
+
const codes = validateArticles(
|
|
441
|
+
[makeArticle({ about: [{ name: 'PF2e', sameAs: 'https://example.com/pf2e' }] })],
|
|
442
|
+
{ ...baseConfig, entities: { pathfinder: { name: 'Pathfinder' } } }
|
|
443
|
+
).warnings.map((i) => i.code)
|
|
444
|
+
expect(codes).not.toContain('unknown-entity')
|
|
445
|
+
})
|
|
446
|
+
|
|
447
|
+
it('skips the entity check when no registry is configured', () => {
|
|
448
|
+
const codes = validateArticles(
|
|
449
|
+
[makeArticle({ about: [{ name: 'Anything' }] })],
|
|
450
|
+
baseConfig
|
|
451
|
+
).warnings.map((i) => i.code)
|
|
452
|
+
expect(codes).not.toContain('unknown-entity')
|
|
292
453
|
})
|
|
293
454
|
|
|
294
455
|
it('warns on category slug collisions', () => {
|