@fullstackdatasolutions/articles 1.2.3 → 1.3.1

package/dist/nextjs.js

@@ -400,6 +400,64 @@ function extractToc(markdown) {
     return headings;
   });
 }
+var QUESTION_OPENERS = /^(what|how|why|when|where|who|which|can|should|does|do|is|are|will|would|must)\b/i;
+function deriveFaqFromHeadings(markdown) {
+  const items = [];
+  const collector = createFaqCollector(items);
+  let inFence = false;
+  for (const line of markdown.split("\n")) {
+    if (FENCE.test(line)) {
+      inFence = !inFence;
+      continue;
+    }
+    if (inFence) continue;
+    collector.consume(line);
+  }
+  collector.flush();
+  return items;
+}
+var FENCE = /^\s*(```|~~~)/;
+var ANY_HEADING = /^#{1,6}\s/;
+function isSpace(char) {
+  return char === " " || char === "	";
+}
+function trimTrailingHashes(value) {
+  let end = value.length;
+  while (end > 0 && value[end - 1] === "#") end--;
+  return value.slice(0, end).trimEnd();
+}
+function readQuestionHeading(line) {
+  if (!line.startsWith("##") || line.startsWith("###")) return null;
+  if (!isSpace(line[2])) return null;
+  const text = trimTrailingHashes(line.slice(3).trim());
+  if (!text.endsWith("?") || !QUESTION_OPENERS.test(text)) return null;
+  return text;
+}
+function createFaqCollector(items) {
+  let pending = null;
+  let buffer = [];
+  const flush = () => {
+    if (pending && buffer.length > 0) {
+      items.push({ question: pending, answer: buffer.join(" ").trim() });
+    }
+    pending = null;
+    buffer = [];
+  };
+  const consume = (line) => {
+    if (ANY_HEADING.test(line)) {
+      flush();
+      pending = readQuestionHeading(line);
+      return;
+    }
+    if (!pending) return;
+    if (line.trim() === "") {
+      if (buffer.length > 0) flush();
+      return;
+    }
+    buffer.push(line.trim());
+  };
+  return { consume, flush };
+}
 
 // src/server-articles.ts
 var articlesDirectory = path.join(
@@ -532,6 +590,58 @@ function parseHowToSteps(raw) {
 function parseOptionalString(raw) {
   return typeof raw === "string" && raw.trim().length > 0 ? raw.trim() : void 0;
 }
+function deriveFaq(markdownContent, config) {
+  if ((config == null ? void 0 : config.deriveFaqFromHeadings) !== true) return void 0;
+  const derived = deriveFaqFromHeadings(markdownContent);
+  return derived.length ? derived : void 0;
+}
+function resolveLastmod(rawLastmod, rawDate, filePath, config) {
+  const declared = parseDateField(rawLastmod);
+  if (declared) return declared;
+  if ((config == null ? void 0 : config.lastmodFallback) !== "fileMtime") return void 0;
+  try {
+    return parseDateField(fs.statSync(filePath).mtime);
+  } catch (e) {
+    return parseDateField(rawDate);
+  }
+}
+function resolveAiCrawl(raw, config) {
+  if (typeof raw === "boolean") return raw;
+  return (config == null ? void 0 : config.aiCrawlDefault) === true;
+}
+function parseEntityReferences(raw, config) {
+  if (!Array.isArray(raw)) return void 0;
+  const items = raw.map((item) => {
+    var _a, _b;
+    if (typeof item === "string") {
+      const key = item.trim();
+      if (!key) return null;
+      return (_b = (_a = config == null ? void 0 : config.entities) == null ? void 0 : _a[key]) != null ? _b : { name: key };
+    }
+    if (typeof item === "object" && item !== null && typeof item.name === "string") {
+      const entity = item;
+      const name = entity.name.trim();
+      if (!name) return null;
+      return entity.sameAs ? { name, sameAs: entity.sameAs } : { name };
+    }
+    return null;
+  }).filter((item) => item !== null);
+  return items.length ? items : void 0;
+}
+function parseCitations(raw) {
+  if (!Array.isArray(raw)) return void 0;
+  const items = raw.map((item) => {
+    if (typeof item === "string") return item.trim() ? { name: item.trim() } : null;
+    if (typeof item === "object" && item !== null && typeof item.name === "string") {
+      const citation = item;
+      const name = citation.name.trim();
+      if (!name) return null;
+      return citation.url ? { name, url: citation.url } : { name };
+    }
+    return null;
+  }).filter((item) => item !== null);
+  return items.length ? items : void 0;
+}
 function parseSeriesOrder(raw) {
   return typeof raw === "number" && Number.isFinite(raw) ? raw : void 0;
 }
@@ -604,6 +714,7 @@ function getArticleAuthors(article, config) {
 }
 function getArticleSummary(slug, config) {
   return __async(this, null, function* () {
+    var _a;
     try {
       const found = findArticleFile(slug);
       if (!found) return null;
@@ -620,7 +731,7 @@ function getArticleSummary(slug, config) {
         title: data.title || slug.replaceAll("-", " "),
         excerpt: data.excerpt || "",
         date: parseDateField(data.date),
-        lastmod: parseDateField(data.lastmod),
+        lastmod: resolveLastmod(data.lastmod, data.date, found.filePath, config),
         author,
         authors,
         authorSlug: primaryAuthorProfile == null ? void 0 : primaryAuthorProfile.slug,
@@ -633,14 +744,17 @@ function getArticleSummary(slug, config) {
         tags: data.tags || [],
         contentType: found.contentType,
         draft: data.draft === true,
-        faq: parseFaqItems(data.faq),
+        faq: (_a = parseFaqItems(data.faq)) != null ? _a : deriveFaq(markdownContent, config),
         howTo: parseHowToSteps(data.howTo),
+        answer: parseOptionalString(data.answer),
+        about: parseEntityReferences(data.about, config),
+        citation: parseCitations(data.citation),
         canonicalUrl: typeof data.canonicalUrl === "string" ? data.canonicalUrl : void 0,
         articleType: typeof data.articleType === "string" ? data.articleType : void 0,
         series: typeof data.series === "string" ? data.series : void 0,
         seriesSlug: parseOptionalString(data.seriesSlug),
         seriesOrder: parseSeriesOrder(data.seriesOrder),
-        aiCrawl: data.aiCrawl === true,
+        aiCrawl: resolveAiCrawl(data.aiCrawl, config),
         searchTitle: parseOptionalString(data.searchTitle),
         searchDescription: parseOptionalString(data.searchDescription),
         socialTitle: parseOptionalString(data.socialTitle),
@@ -699,10 +813,10 @@ var getAllArticles = cache((config) => __async(void 0, null, function* () {
     return new Date(b.date).getTime() - new Date(a.date).getTime();
   });
 }));
-function getArticleMarkdown(slug) {
+function getArticleMarkdown(slug, config) {
   return __async(this, null, function* () {
     try {
-      const summary = yield getArticleSummary(slug);
+      const summary = yield getArticleSummary(slug, config);
       if (!(summary == null ? void 0 : summary.aiCrawl)) return null;
       const found = findArticleFile(slug);
       if (!found) return null;
@@ -720,49 +834,229 @@ function getArticleMarkdown(slug) {
     }
   });
 }
-function getArticleMarkdownResponse(slug, config) {
+var AI_CRAWLERS = [
+  "GPTBot",
+  "ChatGPT-User",
+  "OAI-SearchBot",
+  "CCBot",
+  "ClaudeBot",
+  "Claude-User",
+  "Claude-SearchBot",
+  "anthropic-ai",
+  "PerplexityBot",
+  "Perplexity-User",
+  "Google-Extended",
+  "Applebot-Extended",
+  "Bytespider",
+  "Amazonbot",
+  "meta-externalagent",
+  "cohere-ai",
+  "DuckAssistBot",
+  "MistralAI-User"
+];
+function matchAiCrawler(userAgent) {
+  var _a;
+  if (!userAgent) return null;
+  const normalized = userAgent.toLowerCase();
+  return (_a = AI_CRAWLERS.find((crawler) => normalized.includes(crawler.toLowerCase()))) != null ? _a : null;
+}
+function buildMarkdownTwinHeader(article, config, body) {
+  var _a, _b;
+  const siteUrl = config.siteUrl.replace(/\/$/, "");
+  const firstLine = (_b = (_a = body.trimStart().split("\n", 1)[0]) == null ? void 0 : _a.trim()) != null ? _b : "";
+  const bodyRepeatsTitle = firstLine.toLowerCase() === `# ${article.title}`.toLowerCase();
+  const facts = [
+    `Source: ${siteUrl}/articles/${article.slug}`,
+    article.date ? `Published: ${article.date}` : "",
+    article.lastmod ? `Updated: ${article.lastmod}` : "",
+    config.showAuthor !== false && article.author ? `Author: ${article.author}` : "",
+    `Site: ${config.siteName}`
+  ].filter(Boolean);
+  const blocks = [
+    bodyRepeatsTitle ? "" : `# ${article.title}`,
+    article.excerpt ? `> ${article.excerpt}` : "",
+    facts.join("\n"),
+    article.answer ? `**Short answer:** ${article.answer}` : "",
+    "---"
+  ].filter((block) => block !== "");
+  return `${blocks.join("\n\n")}
+
+`;
+}
+function reportAiCrawl(slug, config, headers) {
+  var _a, _b;
+  if (!config.onAiCrawl) return;
+  const userAgent = (_a = headers == null ? void 0 : headers.get("user-agent")) != null ? _a : "";
+  try {
+    config.onAiCrawl({ slug, crawler: (_b = matchAiCrawler(userAgent)) != null ? _b : "unknown", userAgent });
+  } catch (error) {
+    reportArticlesError({
+      code: "ai-crawl-handler-failed",
+      message: "onAiCrawl handler threw.",
+      error,
+      context: { slug }
+    });
+  }
+}
+function buildListingMarkdown(heading, intro, articles, config) {
+  const siteUrl = config.siteUrl.replace(/\/$/, "");
+  const crawlable = articles.filter((article) => article.aiCrawl === true);
+  const entries = crawlable.map((article) => {
+    var _a;
+    const summary = (_a = article.answer) != null ? _a : article.excerpt;
+    const line = `- [${article.title}](${siteUrl}/articles/${article.slug}.md)`;
+    return summary ? `${line}: ${summary}` : line;
+  });
+  return [
+    `# ${heading}`,
+    "",
+    ...intro.flatMap((line) => [line, ""]),
+    `Source: ${siteUrl}`,
+    `Site: ${config.siteName}`,
+    "",
+    "---",
+    "",
+    ...entries.length > 0 ? entries : ["_No articles available._"],
+    ""
+  ].join("\n");
+}
+function getCategoryMarkdown(categorySlug, config) {
   return __async(this, null, function* () {
-    const markdown = yield getArticleMarkdown(slug);
+    var _a;
+    const articles = yield getArticlesByCategory(categorySlug, config);
+    if (articles.length === 0) return null;
+    const name = (_a = articles[0].categories.find((c) => categoryToSlug(c) === categorySlug)) != null ? _a : categorySlug;
+    const description = resolveCategoryDescription(categorySlug, config);
+    return buildListingMarkdown(name, description ? [description] : [], articles, config);
+  });
+}
+function resolveCategoryDescription(categorySlug, config) {
+  var _a, _b;
+  const entry = (_a = config.categoryDescriptions) == null ? void 0 : _a[categorySlug];
+  if (!entry) return void 0;
+  return typeof entry === "string" ? entry : (_b = entry.long) != null ? _b : entry.short;
+}
+function getAuthorMarkdown(authorSlug, config) {
+  return __async(this, null, function* () {
+    var _a, _b, _c, _d, _e, _f, _g;
+    const author = getAuthorBySlug(authorSlug, config);
+    if (!author) return null;
+    const articles = yield getArticlesByAuthor(authorSlug, config);
+    const intro = [
+      (_a = author.promise) != null ? _a : "",
+      (_b = author.bio) != null ? _b : "",
+      ...((_c = author.servesWho) == null ? void 0 : _c.length) ? [`Writes for: ${author.servesWho.join(", ")}`] : [],
+      ...((_d = author.knowsAbout) == null ? void 0 : _d.length) ? [`Writes about: ${author.knowsAbout.join(", ")}`] : [],
+      ...((_e = author.credentials) == null ? void 0 : _e.length) ? ["## Stated experience", ...author.credentials.map((item) => `- ${item}`)] : [],
+      ...((_f = author.proof) == null ? void 0 : _f.length) ? [
+        "## Proof points",
+        ...author.proof.map(
+          (item) => item.url ? `- [${item.claim}](${item.url})` : `- ${item.claim}`
+        )
+      ] : [],
+      ...((_g = author.originStory) == null ? void 0 : _g.length) ? [
+        "## Background",
+        ...author.originStory.flatMap((section) => [
+          ...section.heading ? [`### ${section.heading}`] : [],
+          ...section.paragraphs
+        ])
+      ] : []
+    ].filter((line) => line.trim() !== "");
+    return buildListingMarkdown(author.name, intro, articles, config);
+  });
+}
+function getSeriesMarkdown(seriesSlug, config) {
+  return __async(this, null, function* () {
+    var _a;
+    const articles = yield getArticlesBySeries(seriesSlug, config);
+    if (articles.length === 0) return null;
+    const name = (_a = articles[0].series) != null ? _a : seriesSlug;
+    return buildListingMarkdown(name, [], articles, config);
+  });
+}
+function getMarkdownTwinResponse(slug, config, options) {
+  return __async(this, null, function* () {
+    const listing = yield resolveListingMarkdown(slug, config);
+    if (listing !== void 0) {
+      if (listing === null) return new Response("Not Found", { status: 404 });
+      reportAiCrawl(slug, config, options == null ? void 0 : options.headers);
+      return new Response(listing, { headers: LISTING_MARKDOWN_HEADERS });
+    }
+    return getArticleMarkdownResponse(slug, config, options);
+  });
+}
+var LISTING_MARKDOWN_HEADERS = {
+  "Content-Type": "text/markdown; charset=utf-8",
+  "Cache-Control": "public, max-age=3600, s-maxage=3600"
+};
+function resolveListingMarkdown(slug, config) {
+  return __async(this, null, function* () {
+    const [prefix, ...rest] = slug.split("/");
+    const key = rest.join("/");
+    if (!key) return void 0;
+    if (prefix === "category") return getCategoryMarkdown(key, config);
+    if (prefix === "authors") return getAuthorMarkdown(key, config);
+    if (prefix === "series") return getSeriesMarkdown(key, config);
+    return void 0;
+  });
+}
+function getArticleMarkdownResponse(slug, config, options) {
+  return __async(this, null, function* () {
+    const markdown = yield getArticleMarkdown(slug, config);
     if (markdown === null) return new Response("Not Found", { status: 404 });
-    const article = yield getArticleMetadata(slug);
-    return new Response(markdown, {
-      headers: __spreadValues({
+    const article = yield getArticleMetadata(slug, config);
+    reportAiCrawl(slug, config, options == null ? void 0 : options.headers);
+    const body = article && config.markdownTwinHeader !== false ? `${buildMarkdownTwinHeader(article, config, markdown)}${markdown.trimStart()}` : markdown;
+    const canonicalUrl = `${config.siteUrl.replace(/\/$/, "")}/articles/${slug}`;
+    return new Response(body, {
+      headers: {
         "Content-Type": "text/markdown; charset=utf-8",
-        "Cache-Control": "public, max-age=3600, s-maxage=3600"
-      }, article ? getArticleAiHeaders(article, config) : {})
+        "Cache-Control": "public, max-age=3600, s-maxage=3600",
+        Link: `<${canonicalUrl}>; rel="canonical"`
+      }
     });
   });
 }
-function getArticleMarkdownUrl(article, config) {
-  if (article.aiCrawl !== true) return void 0;
-  const pathname = `/articles/${article.slug}.md`;
-  if (!config) return pathname;
-  return `${config.siteUrl.replace(/\/$/, "")}${pathname}`;
-}
-function getArticleAiHeaders(article, config) {
-  const markdownUrl = getArticleMarkdownUrl(article, config);
-  if (markdownUrl) {
-    return {
-      Link: `<${markdownUrl}>; rel="alternate"; type="text/markdown"`
-    };
-  }
-  return {
-    "X-Robots-Tag": "noai, noimageai"
-  };
-}
 function categoryToSlug(category) {
   return category.toLowerCase().replaceAll(/\s+/g, "-").replaceAll(/[^a-z0-9-]/g, "");
 }
+function getArticlesByCategory(categorySlug, config) {
+  return __async(this, null, function* () {
+    const articles = yield getAllArticles(config);
+    return articles.filter(
+      (article) => article.categories.some((cat) => categoryToSlug(cat) === categorySlug)
+    );
+  });
+}
+function getArticlesByAuthor(authorSlug, config) {
+  return __async(this, null, function* () {
+    const articles = yield getAllArticles(config);
+    return articles.filter(
+      (article) => getArticleAuthors(article, config).some((author) => author.slug === authorSlug)
+    );
+  });
+}
+function getArticlesBySeries(seriesSlug, config) {
+  return __async(this, null, function* () {
+    const articles = yield getAllArticles(config);
+    return articles.filter((article) => article.seriesSlug === seriesSlug).sort((a, b) => {
+      var _a, _b;
+      const orderA = (_a = a.seriesOrder) != null ? _a : Number.POSITIVE_INFINITY;
+      const orderB = (_b = b.seriesOrder) != null ? _b : Number.POSITIVE_INFINITY;
+      return orderA - orderB;
+    });
+  });
+}
 
 // src/nextjs.ts
 function createArticleMarkdownHandler(config) {
-  function GET(_request, context) {
+  function GET(request, context) {
     return __async(this, null, function* () {
       const params = yield context.params;
       const slugParts = params["slug"];
       const slug = Array.isArray(slugParts) ? slugParts.join("/") : slugParts != null ? slugParts : "";
       const cleanSlug = slug.endsWith(".md") ? slug.slice(0, -3) : slug;
-      return getArticleMarkdownResponse(cleanSlug, config);
+      return getMarkdownTwinResponse(cleanSlug, config, { headers: request.headers });
     });
   }
   return { GET };