@fullstackdatasolutions/articles 0.7.2 → 0.8.0

package/src/__tests__/server-articles.test.ts

@@ -3,14 +3,20 @@ import path from 'node:path'
 import {
   categoryToSlug,
   getAdjacentArticles,
+  getAiRobotsTxtRules,
   getAllArticles,
   getAllCategories,
+  getArticleAiHeaders,
+  getArticleMarkdown,
+  getArticleMarkdownResponse,
+  getArticleMarkdownUrl,
   getArticleMetadata,
   getArticlesByCategory,
   getAvailableArticleSlugs,
   sanitizeImagePath,
   searchArticles,
 } from '../server-articles'
+import { setArticlesErrorHandler } from '../errorReporting'
 
 jest.mock('react', () => ({ cache: (fn: Function) => fn }))
 jest.mock('node:fs')
@@ -23,6 +29,52 @@ jest.mock('reading-time', () => () => ({ text: '2 min read' }))
 const mockedFs = fs as jest.Mocked<typeof fs>
 const articlesDirectory = path.join(process.cwd(), 'public/articles')
 
+beforeEach(() => {
+  setArticlesErrorHandler(() => undefined)
+})
+
+afterAll(() => {
+  setArticlesErrorHandler()
+})
+
+class TestHeaders {
+  private readonly values: Map<string, string>
+
+  constructor(values?: Record<string, string>) {
+    this.values = new Map(
+      Object.entries(values ?? {}).map(([key, value]) => [key.toLowerCase(), value])
+    )
+  }
+
+  get(name: string): string | null {
+    return this.values.get(name.toLowerCase()) ?? null
+  }
+}
+
+class TestResponse {
+  readonly headers: TestHeaders
+  readonly status: number
+
+  constructor(
+    private readonly body: string,
+    init?: Readonly<{ status?: number; headers?: Record<string, string> }>
+  ) {
+    this.status = init?.status ?? 200
+    this.headers = new TestHeaders(init?.headers)
+  }
+
+  text(): Promise<string> {
+    return Promise.resolve(this.body)
+  }
+}
+
+if (globalThis.Response === undefined) {
+  Object.defineProperty(globalThis, 'Response', {
+    value: TestResponse,
+    configurable: true,
+  })
+}
+
 function mockDirectory(name: string): fs.Dirent {
   return {
     name,
@@ -166,6 +218,21 @@ describe('getArticleMetadata - frontmatter parsing', () => {
     expect(article).not.toBeNull()
     expect(article!.howTo).toBeUndefined()
   })
+
+  const aiCrawlCases = [
+    { frontmatter: 'aiCrawl: true\n', expected: true, description: 'explicit true' },
+    { frontmatter: 'aiCrawl: false\n', expected: false, description: 'explicit false' },
+    { frontmatter: '', expected: false, description: 'omitted value' },
+  ]
+
+  aiCrawlCases.forEach(({ frontmatter, expected, description }) => {
+    it(`parses aiCrawl as ${expected} for ${description}`, async () => {
+      setupArticleMock(frontmatter)
+      const article = await getArticleMetadata('test-slug')
+      expect(article).not.toBeNull()
+      expect(article!.aiCrawl).toBe(expected)
+    })
+  })
 })
 
 describe('getAvailableArticleSlugs', () => {
@@ -223,6 +290,24 @@ describe('getAvailableArticleSlugs', () => {
 
     expect(getAvailableArticleSlugs()).toEqual([])
   })
+
+  it('reports directory read errors through the configured error handler', () => {
+    const reports: Array<{ code: string; context?: Record<string, unknown> }> = []
+    setArticlesErrorHandler((report) => {
+      reports.push({ code: report.code, context: report.context })
+    })
+    mockedFs.existsSync.mockImplementation(() => {
+      throw new Error('read failed')
+    })
+
+    expect(getAvailableArticleSlugs()).toEqual([])
+    expect(reports).toEqual([
+      {
+        code: 'article-directory-read-failed',
+        context: { directory: articlesDirectory },
+      },
+    ])
+  })
 })
 
 describe('article collection utilities', () => {
@@ -337,6 +422,26 @@ describe('article collection utilities', () => {
     ])
   })
 
+  it('skips author matching when showAuthor is false', async () => {
+    setupArticleTreeMock([
+      {
+        slug: 'author-match',
+        frontmatter: 'date: 2025-01-01\n',
+      },
+    ])
+
+    await expect(searchArticles('test author')).resolves.toEqual([
+      expect.objectContaining({ slug: 'author-match' }),
+    ])
+    await expect(
+      searchArticles('test author', {
+        siteUrl: 'https://example.com',
+        siteName: 'Example',
+        showAuthor: false,
+      })
+    ).resolves.toEqual([])
+  })
+
   it('groups categories and filters articles by category slug', async () => {
     setupArticleTreeMock([
       {
@@ -422,6 +527,95 @@ describe('getArticleMetadata - content handling', () => {
   })
 })
 
+describe('AI markdown helpers', () => {
+  beforeEach(() => {
+    jest.clearAllMocks()
+  })
+
+  it('returns markdown only for articles with aiCrawl true', async () => {
+    setupArticleTreeMock([
+      {
+        slug: 'allowed',
+        frontmatter: 'date: 2025-01-01\naiCrawl: true\n',
+        body: '# Allowed',
+      },
+      {
+        slug: 'blocked',
+        frontmatter: 'date: 2025-01-01\naiCrawl: false\n',
+        body: '# Blocked',
+      },
+      {
+        slug: 'default-blocked',
+        frontmatter: 'date: 2025-01-01\n',
+        body: '# Default blocked',
+      },
+    ])
+
+    await expect(getArticleMarkdown('allowed')).resolves.toBe('\n# Allowed')
+    await expect(getArticleMarkdown('blocked')).resolves.toBeNull()
+    await expect(getArticleMarkdown('default-blocked')).resolves.toBeNull()
+  })
+
+  it('builds markdown URLs only for opted-in articles', () => {
+    expect(
+      getArticleMarkdownUrl({ slug: 'allowed', aiCrawl: true }, { siteUrl: 'https://example.com/' })
+    ).toBe('https://example.com/articles/allowed.md')
+    expect(getArticleMarkdownUrl({ slug: 'blocked', aiCrawl: false })).toBeUndefined()
+  })
+
+  it('returns alternate link headers for allowed articles and AI opt-out headers otherwise', () => {
+    expect(
+      getArticleAiHeaders({ slug: 'allowed', aiCrawl: true }, { siteUrl: 'https://example.com' })
+    ).toEqual({
+      Link: '<https://example.com/articles/allowed.md>; rel="alternate"; type="text/markdown"',
+    })
+    expect(getArticleAiHeaders({ slug: 'blocked', aiCrawl: false })).toEqual({
+      'X-Robots-Tag': 'noai, noimageai',
+    })
+  })
+
+  it('returns markdown responses for opted-in articles and 404 for blocked articles', async () => {
+    setupArticleTreeMock([
+      {
+        slug: 'allowed',
+        frontmatter: 'date: 2025-01-01\naiCrawl: true\n',
+        body: '# Allowed',
+      },
+      {
+        slug: 'blocked',
+        frontmatter: 'date: 2025-01-01\n',
+        body: '# Blocked',
+      },
+    ])
+
+    const allowed = await getArticleMarkdownResponse('allowed', {
+      siteUrl: 'https://example.com',
+      siteName: 'Example',
+    })
+    expect(allowed.status).toBe(200)
+    expect(allowed.headers.get('Content-Type')).toBe('text/markdown; charset=utf-8')
+    await expect(allowed.text()).resolves.toBe('\n# Allowed')
+
+    const blocked = await getArticleMarkdownResponse('blocked', {
+      siteUrl: 'https://example.com',
+      siteName: 'Example',
+    })
+    expect(blocked.status).toBe(404)
+  })
+
+  it('generates robots rules for AI crawlers and blocked articles only', async () => {
+    setupArticleTreeMock([
+      { slug: 'allowed', frontmatter: 'date: 2025-01-01\naiCrawl: true\n' },
+      { slug: 'blocked', frontmatter: 'date: 2025-01-01\n' },
+    ])
+
+    const rules = await getAiRobotsTxtRules()
+    expect(rules).toContain('User-agent: GPTBot')
+    expect(rules).toContain('Disallow: /articles/blocked')
+    expect(rules).not.toContain('Disallow: /articles/allowed')
+  })
+})
+
 describe('article path helpers', () => {
   it('sanitizes image paths and rejects unsafe values', () => {
     expect(sanitizeImagePath('image.png', 'article')).toBe('/articles/article/image.png')

package/src/articleTypes.ts

@@ -37,6 +37,7 @@ export interface Article {
   canonicalUrl?: string
   articleType?: string
   series?: string
+  aiCrawl?: boolean
 }
 
 export interface CategoryInfo {

package/src/articlesConfig.ts

@@ -100,6 +100,8 @@ export interface ArticlesConfig {
   showToc?: boolean
   /** Set to false to hide the "Back to Articles" navigation link on article detail pages. Default: true. */
   showBackToArticles?: boolean
+  /** Set to false to hide author names from UI and metadata. Default: true. */
+  showAuthor?: boolean
 }
 
 export const DEFAULT_PAGE_SIZE = 6

package/src/errorReporting.ts

@@ -0,0 +1,34 @@
+export type ArticlesErrorCode =
+  | 'article-directory-read-failed'
+  | 'article-load-failed'
+  | 'article-markdown-load-failed'
+  | 'markdown-conversion-failed'
+  | 'unsafe-image-path'
+
+export type ArticlesErrorContext = Readonly<Record<string, string | number | boolean | undefined>>
+
+export type ArticlesErrorReport = Readonly<{
+  code: ArticlesErrorCode
+  message: string
+  error?: unknown
+  context?: ArticlesErrorContext
+}>
+
+export type ArticlesErrorHandler = (report: ArticlesErrorReport) => void
+
+function defaultArticlesErrorHandler(report: ArticlesErrorReport): void {
+  if (process.env.NODE_ENV === 'production') return
+  const context = report.context ? ` ${JSON.stringify(report.context)}` : ''
+  const detail = report.error instanceof Error ? `: ${report.error.message}` : ''
+  console.warn(`[articles:${report.code}] ${report.message}${context}${detail}`)
+}
+
+let articlesErrorHandler: ArticlesErrorHandler = defaultArticlesErrorHandler
+
+export function setArticlesErrorHandler(handler?: ArticlesErrorHandler): void {
+  articlesErrorHandler = handler ?? defaultArticlesErrorHandler
+}
+
+export function reportArticlesError(report: ArticlesErrorReport): void {
+  articlesErrorHandler(report)
+}

package/src/markdown.ts

@@ -11,6 +11,7 @@ import remarkRehype from 'remark-rehype'
 import { Plugin } from 'unified'
 import { visit } from 'unist-util-visit'
 import type { TocItem } from './articleTypes'
+import { reportArticlesError } from './errorReporting'
 
 // Import the sanitizeImagePath function
 function sanitizeImagePath(rawPath: string, articleSlug: string): string | null {
@@ -29,13 +30,21 @@ function sanitizeImagePath(rawPath: string, articleSlug: string): string | null
 
   // For relative paths, ensure they don't contain dangerous patterns
   if (cleanPath.includes('..') || cleanPath.includes('\\') || cleanPath.startsWith('/')) {
-    console.warn(`Potentially unsafe image path detected: ${cleanPath}`)
+    reportArticlesError({
+      code: 'unsafe-image-path',
+      message: 'Rejected unsafe markdown image path.',
+      context: { articleSlug, path: cleanPath },
+    })
     return null
   }
 
   // Only allow alphanumeric characters, hyphens, underscores, dots, and forward slashes
   if (!/^[a-zA-Z0-9._/-]+$/.test(cleanPath)) {
-    console.warn(`Invalid characters in image path: ${cleanPath}`)
+    reportArticlesError({
+      code: 'unsafe-image-path',
+      message: 'Rejected markdown image path with invalid characters.',
+      context: { articleSlug, path: cleanPath },
+    })
     return null
   }
 
@@ -44,7 +53,11 @@ function sanitizeImagePath(rawPath: string, articleSlug: string): string | null
     // Relative path, ensure it's within the article directory
     const normalizedPath = cleanPath.replaceAll('\\', '/')
     if (normalizedPath.startsWith('..') || normalizedPath.includes('../')) {
-      console.warn(`Path traversal attempt detected: ${cleanPath}`)
+      reportArticlesError({
+        code: 'unsafe-image-path',
+        message: 'Rejected markdown image path traversal attempt.',
+        context: { articleSlug, path: cleanPath },
+      })
       return null
     }
     return `/articles/${articleSlug}/${normalizedPath}`
@@ -252,7 +265,11 @@ const rehypeProcessImages: Plugin<[{ articleSlug?: string }], Root> = (options =
             node.properties.src = sanitizedSrc
           } else {
             // If sanitization fails, use a placeholder
-            console.warn(`Using placeholder for unsafe image path: ${src}`)
+            reportArticlesError({
+              code: 'unsafe-image-path',
+              message: 'Using placeholder for unsafe markdown image path.',
+              context: { articleSlug: options.articleSlug, path: src },
+            })
             node.properties.src = '/placeholder-logo.png'
           }
         }
@@ -290,7 +307,12 @@ export async function markdownToHtml(markdown: string, articleSlug?: string) {
 
     return result.toString()
   } catch (error) {
-    console.error('Markdown conversion error:', error)
+    reportArticlesError({
+      code: 'markdown-conversion-failed',
+      message: 'Unable to convert markdown to HTML.',
+      error,
+      context: { articleSlug },
+    })
     // Return the original markdown as fallback
     return markdown
   }

package/src/seoUtils.ts

@@ -4,6 +4,7 @@ import {
   getAllArticles,
   getAllCategories,
   getArticlesByCategory,
+  getArticleMarkdownUrl,
   getAvailableArticleSlugs,
 } from './server-articles'
 import { ArticlesConfig } from './articlesConfig'
@@ -45,6 +46,8 @@ export async function generateArticleMetadata(
     ? resolveImageUrl(article.featuredImage, siteUrl)
     : `${siteUrl}/placeholder-logo.png`
   const description = article.excerpt ?? `Read ${article.title} on ${config.siteName}.`
+  const showAuthor = config.showAuthor !== false
+  const markdownUrl = getArticleMarkdownUrl(article, config)
 
   return {
     title: `${article.title} | ${config.siteName}`,
@@ -60,7 +63,7 @@ export async function generateArticleMetadata(
       type: 'article',
       ...(article.date && { publishedTime: article.date }),
       ...(article.lastmod && { modifiedTime: new Date(article.lastmod).toISOString() }),
-      authors: [article.author],
+      ...(showAuthor && { authors: [article.author] }),
       tags: article.tags ?? [],
     },
     twitter: {
@@ -71,6 +74,11 @@ export async function generateArticleMetadata(
     },
     alternates: {
       canonical: canonicalUrl,
+      ...(markdownUrl && {
+        types: {
+          'text/markdown': markdownUrl,
+        },
+      }),
     },
     robots: {
       index: true,
@@ -84,7 +92,7 @@ export async function generateArticleMetadata(
       },
     },
     other: {
-      'article:author': article.author,
+      ...(showAuthor && { 'article:author': article.author }),
       ...(article.date && {
         'article:published_time': new Date(article.date).toISOString(),
       }),

package/src/server-articles.ts

@@ -5,8 +5,10 @@ import path from 'node:path'
 import readingTime from 'reading-time'
 import { markdownToHtml, extractToc } from './markdown'
 import type { Article, CategoryInfo, FaqItem, HowToStep } from './articleTypes'
+import type { ArticlesConfig } from './articlesConfig'
+import { reportArticlesError } from './errorReporting'
 
-const articlesDirectory = path.join(process.cwd(), 'public/articles')
+const articlesDirectory = path.join(/* turbopackIgnore: true */ process.cwd(), 'public/articles')
 
 function getReadingTime(content: string): string {
   return readingTime(content).text
@@ -38,17 +40,29 @@ function sanitizeImagePath(rawPath: string, articleSlug: string): string | null
   const cleanPath = rawPath.replaceAll(/[\x00-\x1f\x7f-\x9f]/g, '')
   if (cleanPath.startsWith('http://') || cleanPath.startsWith('https://')) return cleanPath
   if (cleanPath.includes('..') || cleanPath.includes('\\') || cleanPath.startsWith('/')) {
-    console.warn(`Potentially unsafe image path detected: ${cleanPath}`)
+    reportArticlesError({
+      code: 'unsafe-image-path',
+      message: 'Rejected unsafe article image path.',
+      context: { articleSlug, path: cleanPath },
+    })
     return null
   }
   if (!/^[a-zA-Z0-9._/-]+$/.test(cleanPath)) {
-    console.warn(`Invalid characters in image path: ${cleanPath}`)
+    reportArticlesError({
+      code: 'unsafe-image-path',
+      message: 'Rejected article image path with invalid characters.',
+      context: { articleSlug, path: cleanPath },
+    })
     return null
   }
   if (cleanPath.includes('/')) {
     const normalizedPath = path.normalize(cleanPath)
     if (normalizedPath.startsWith('..') || normalizedPath.includes('../')) {
-      console.warn(`Path traversal attempt detected: ${cleanPath}`)
+      reportArticlesError({
+        code: 'unsafe-image-path',
+        message: 'Rejected article image path traversal attempt.',
+        context: { articleSlug, path: cleanPath },
+      })
       return null
     }
     return `/articles/${articleSlug}/${cleanPath}`
@@ -69,7 +83,12 @@ export function getAvailableArticleSlugs(): string[] {
     if (!fs.existsSync(articlesDirectory)) return []
     return walkArticleDir(articlesDirectory, '')
   } catch (error) {
-    console.error('Error reading articles directory:', error)
+    reportArticlesError({
+      code: 'article-directory-read-failed',
+      message: 'Unable to read articles directory.',
+      error,
+      context: { directory: articlesDirectory },
+    })
     return []
   }
 }
@@ -165,9 +184,15 @@ async function getArticleSummary(slug: string): Promise<Article | null> {
       canonicalUrl: typeof data.canonicalUrl === 'string' ? data.canonicalUrl : undefined,
       articleType: typeof data.articleType === 'string' ? data.articleType : undefined,
       series: typeof data.series === 'string' ? data.series : undefined,
+      aiCrawl: data.aiCrawl === true,
     }
   } catch (error) {
-    console.error(`Error loading article ${slug}:`, error)
+    reportArticlesError({
+      code: 'article-load-failed',
+      message: 'Unable to load article summary.',
+      error,
+      context: { slug },
+    })
     return null
   }
 }
@@ -190,7 +215,12 @@ export const getArticleMetadata = cache(async (slug: string): Promise<Article |
     }
     return { ...summary, content: markdownContent, htmlContent, mdxSource, toc }
   } catch (error) {
-    console.error(`Error loading article ${slug}:`, error)
+    reportArticlesError({
+      code: 'article-load-failed',
+      message: 'Unable to load article metadata.',
+      error,
+      context: { slug },
+    })
     return null
   }
 })
@@ -222,14 +252,99 @@ export async function getAdjacentArticles(
   return { previous, next }
 }
 
-export async function searchArticles(query: string): Promise<Article[]> {
+export async function getArticleMarkdown(slug: string): Promise<string | null> {
+  try {
+    const summary = await getArticleSummary(slug)
+    if (!summary?.aiCrawl) return null
+    const found = findArticleFile(slug)
+    if (!found) return null
+    const fileContent = fs.readFileSync(found.filePath, 'utf8')
+    const { content: markdownContent } = matter(fileContent)
+    return markdownContent
+  } catch (error) {
+    reportArticlesError({
+      code: 'article-markdown-load-failed',
+      message: 'Unable to load article markdown.',
+      error,
+      context: { slug },
+    })
+    return null
+  }
+}
+
+export async function getArticleMarkdownResponse(
+  slug: string,
+  config: ArticlesConfig
+): Promise<Response> {
+  const markdown = await getArticleMarkdown(slug)
+  if (markdown === null) return new Response('Not Found', { status: 404 })
+  const article = await getArticleMetadata(slug)
+  return new Response(markdown, {
+    headers: {
+      'Content-Type': 'text/markdown; charset=utf-8',
+      'Cache-Control': 'public, max-age=3600, s-maxage=3600',
+      ...(article ? getArticleAiHeaders(article, config) : {}),
+    },
+  })
+}
+
+export function getArticleMarkdownUrl(
+  article: Pick<Article, 'slug' | 'aiCrawl'>,
+  config?: Pick<ArticlesConfig, 'siteUrl'>
+): string | undefined {
+  if (article.aiCrawl !== true) return undefined
+  const pathname = `/articles/${article.slug}.md`
+  if (!config) return pathname
+  return `${config.siteUrl.replace(/\/$/, '')}${pathname}`
+}
+
+export function getArticleAiHeaders(
+  article: Pick<Article, 'slug' | 'aiCrawl'>,
+  config?: Pick<ArticlesConfig, 'siteUrl'>
+): Record<string, string> {
+  const markdownUrl = getArticleMarkdownUrl(article, config)
+  if (markdownUrl) {
+    return {
+      Link: `<${markdownUrl}>; rel="alternate"; type="text/markdown"`,
+    }
+  }
+  return {
+    'X-Robots-Tag': 'noai, noimageai',
+  }
+}
+
+export async function getAiRobotsTxtRules(): Promise<string> {
+  const articles = await getAllArticles()
+  const blockedArticles = articles.filter((article) => article.aiCrawl !== true)
+  if (blockedArticles.length === 0) return ''
+
+  const aiCrawlers = [
+    'GPTBot',
+    'ChatGPT-User',
+    'CCBot',
+    'ClaudeBot',
+    'Claude-User',
+    'PerplexityBot',
+    'Google-Extended',
+  ]
+  const disallowRules = blockedArticles
+    .map((article) => `Disallow: /articles/${article.slug}`)
+    .join('\n')
+
+  return aiCrawlers
+    .map((crawler) => [`User-agent: ${crawler}`, disallowRules].join('\n'))
+    .join('\n\n')
+}
+
+export async function searchArticles(query: string, config?: ArticlesConfig): Promise<Article[]> {
   if (!query?.trim()) return getAllArticles()
   const articles = await getAllArticles()
   const searchTerm = query.toLowerCase().trim()
+  const includeAuthor = config?.showAuthor !== false
   return articles.filter((article) => {
     const matchesTitle = article.title.toLowerCase().includes(searchTerm)
     const matchesExcerpt = article.excerpt.toLowerCase().includes(searchTerm)
-    const matchesAuthor = article.author.toLowerCase().includes(searchTerm)
+    const matchesAuthor = includeAuthor && article.author.toLowerCase().includes(searchTerm)
     const matchesCategory = article.categories.some((cat) => cat.toLowerCase().includes(searchTerm))
     const matchesTags = article.tags?.some((tag) => tag.toLowerCase().includes(searchTerm))
     return (

package/src/server.ts

@@ -1,6 +1,11 @@
 // Server-only exports — uses fs/path; never import this in a client bundle
 export {
   getAllArticles,
+  getAiRobotsTxtRules,
+  getArticleAiHeaders,
+  getArticleMarkdown,
+  getArticleMarkdownResponse,
+  getArticleMarkdownUrl,
   getArticleMetadata,
   getAvailableArticleSlugs,
   getAdjacentArticles,
@@ -21,8 +26,15 @@ export {
 } from './seoUtils'
 
 export { markdownToHtml, extractToc } from './markdown'
+export { setArticlesErrorHandler } from './errorReporting'
 export { ArticleContent } from './ArticleContent'
 export { ArticleTOC } from './ArticleTOC'
 
 export type { Article, CategoryInfo, TocItem } from './articleTypes'
 export type { ArticlesConfig } from './articlesConfig'
+export type {
+  ArticlesErrorCode,
+  ArticlesErrorContext,
+  ArticlesErrorHandler,
+  ArticlesErrorReport,
+} from './errorReporting'