@supersoniks/concorde 4.9.0 → 4.9.2

package/scripts/generate-crawl-docs.mjs

@@ -0,0 +1,569 @@
+import fs from "fs";
+import path from "path";
+import { fileURLToPath } from "url";
+import { marked } from "marked";
+import { gfmHeadingId } from "marked-gfm-heading-id";
+
+const SITE_ORIGIN = "https://concorde.supersoniks.org";
+const SRC_DIR = path.resolve("src");
+const DOCS_OUT = path.resolve("docs");
+const CRAWL_DIR = path.join(DOCS_OUT, "crawl");
+
+const SONIC_CODE_BLOCK_RE = /<sonic-code\b[\s\S]*?<\/sonic-code>/gi;
+const DOC_HASH_LINK_RE = /#([^)\s"']+)\.md\/([^)\s"'#]+)/g;
+
+marked.use(gfmHeadingId());
+
+function listMdFiles(dir, files = []) {
+  for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+    const filePath = path.join(dir, entry.name);
+    if (entry.isDirectory()) {
+      listMdFiles(filePath, files);
+    } else if (entry.name.endsWith(".md") || entry.name.endsWith(".mdx")) {
+      files.push(filePath);
+    }
+  }
+  return files;
+}
+
+function escapeHtml(text) {
+  return text
+    .replace(/&/g, "&amp;")
+    .replace(/</g, "&lt;")
+    .replace(/>/g, "&gt;")
+    .replace(/"/g, "&quot;");
+}
+
+function preprocessSonicCode(markdown) {
+  return markdown.replace(SONIC_CODE_BLOCK_RE, (block) => {
+    const langMatch = block.match(/language="([^"]+)"/);
+    const lang = langMatch ? langMatch[1] : "";
+    const templateMatch = block.match(/<template>([\s\S]*?)<\/template>/);
+    const code = templateMatch ? templateMatch[1].trim() : "";
+    return `\n\n\`\`\`${lang}\n${code}\n\`\`\`\n\n`;
+  });
+}
+
+function mdRelativePath(absPath) {
+  return path.relative(SRC_DIR, absPath).replace(/\\/g, "/");
+}
+
+function crawlHtmlPath(mdRel) {
+  return path.join("crawl", mdRel.replace(/\.mdx?$/, ".html")).replace(/\\/g, "/");
+}
+
+function docId(mdRel) {
+  return mdRel.replace(/\.mdx?$/, "");
+}
+
+function relativeCrawlLink(fromMdRel, toMdPath, anchor) {
+  const fromHtml = crawlHtmlPath(fromMdRel);
+  const toHtml = crawlHtmlPath(`${toMdPath}.md`);
+  const rel = path
+    .relative(path.dirname(fromHtml), toHtml)
+    .replace(/\\/g, "/");
+  const href = rel.startsWith(".") ? rel : `./${rel}`;
+  return `${href}#${anchor}`;
+}
+
+function rewriteDocLinks(text, fromMdRel) {
+  return text.replace(DOC_HASH_LINK_RE, (match, mdPath, anchor) => {
+    return relativeCrawlLink(fromMdRel, mdPath, anchor);
+  });
+}
+
+function extractTitle(markdown) {
+  const match = markdown.match(/^#\s+(.+)$/m);
+  return match ? match[1].trim() : "Concorde";
+}
+
+function markdownToPlainText(markdown) {
+  let text = preprocessSonicCode(markdown);
+  text = text.replace(/\[([^\]]+)\]\([^)]+\)/g, "$1");
+  text = text.replace(/^#+\s+/gm, "");
+  text = text.replace(/```[\s\S]*?```/g, (block) =>
+    block.replace(/```\w*\n?/g, "").replace(/```/g, "\n"),
+  );
+  text = text.replace(/[*_`]/g, "");
+  text = text.replace(/<[^>]+>/g, " ");
+  return text.replace(/[ \t]+\n/g, "\n").replace(/\n{3,}/g, "\n\n").trim();
+}
+
+function cleanDescriptionLine(text) {
+  return text
+    .replace(/\*\*/g, "")
+    .replace(/\[([^\]]+)\]\([^)]+\)/g, "$1")
+    .replace(/`([^`]+)`/g, "$1")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+function extractDescription(markdown, title) {
+  const lines = markdown.split("\n");
+  let pastTitle = false;
+  for (const line of lines) {
+    if (/^#\s+/.test(line)) {
+      pastTitle = true;
+      continue;
+    }
+    if (!pastTitle) continue;
+    const trimmed = line.trim();
+    if (!trimmed || trimmed.startsWith("#") || trimmed.startsWith("|")) continue;
+    if (trimmed.startsWith("<")) continue;
+    const cleaned = cleanDescriptionLine(trimmed);
+    if (cleaned) return cleaned.slice(0, 200);
+  }
+
+  const templateMatch = markdown.match(/<template>([\s\S]*?)<\/template>/);
+  if (templateMatch) {
+    const fromTemplate = cleanDescriptionLine(
+      templateMatch[1].replace(/<[^>]+>/g, " "),
+    );
+    if (fromTemplate.length > 15) {
+      return `${title}: ${fromTemplate}`.slice(0, 200);
+    }
+  }
+
+  const stripped = markdownToPlainText(markdown.replace(/^#\s+.+$/m, ""));
+  if (stripped.length > 15) {
+    return stripped.slice(0, 200);
+  }
+
+  return `Concorde documentation: ${title}`;
+}
+
+function docKind(mdRel) {
+  if (mdRel.includes("/_decorators/")) return "decorator";
+  if (mdRel.includes("/_directives/")) return "directive";
+  if (mdRel.includes("/_getting-started/")) return "guide";
+  if (mdRel.includes("/_core-concept/")) return "concept";
+  if (mdRel.includes("/_misc/")) return "reference";
+  if (mdRel.includes("/components/functional/")) return "functional component";
+  if (mdRel.includes("/components/ui/")) return "UI component";
+  return "documentation";
+}
+
+function extractKeywords(mdRel, title, raw) {
+  const keywords = new Set(["Concorde", "supersoniks"]);
+  const baseName = path.basename(mdRel, path.extname(mdRel));
+  const id = docId(mdRel);
+
+  keywords.add(id);
+  keywords.add(baseName);
+  keywords.add(title);
+
+  if (title.startsWith("@")) {
+    keywords.add("decorator");
+    keywords.add(title.slice(1));
+  }
+
+  const kind = docKind(mdRel);
+  keywords.add(kind);
+
+  if (mdRel.includes("/components/ui/form/")) {
+    keywords.add("form", "FormElement", "formDataProvider");
+  }
+  if (mdRel.includes("/components/")) {
+    keywords.add(`sonic-${baseName}`);
+    keywords.add("web component", "Lit");
+  }
+  if (mdRel.includes("/_decorators/")) {
+    keywords.add("decorator", "DataProvider", "ApiResult", "Endpoint");
+  }
+  if (mdRel.includes("/_core-concept/")) {
+    keywords.add("DataProvider", "data flow");
+  }
+
+  const tickMatches = raw.slice(0, 3000).match(/`([^`]{2,40})`/g);
+  if (tickMatches) {
+    for (const tick of tickMatches.slice(0, 10)) {
+      keywords.add(tick.replace(/`/g, "").replace(/<[^>]+>/g, ""));
+    }
+  }
+
+  return [...keywords].join(", ");
+}
+
+function pageHash(mdRel) {
+  const base = mdRel.replace(/\.mdx?$/, "");
+  const anchor = base.split("/").pop() || "page";
+  return `#${base}.md/${anchor}`;
+}
+
+function crawlPageUrl(crawlRel) {
+  return `${SITE_ORIGIN}/${crawlRel}`;
+}
+
+function buildLeadParagraph({ title, mdRel, description, canonical, keywords }) {
+  const id = docId(mdRel);
+  const kind = docKind(mdRel);
+  return `<p class="doc-lead" id="doc-summary"><strong>Concorde ${escapeHtml(kind)} — ${escapeHtml(title)}</strong>. ${escapeHtml(description)} <span>Doc ID: <code>${escapeHtml(id)}</code>. Keywords: ${escapeHtml(keywords)}. URL: <a href="${escapeHtml(canonical)}">${escapeHtml(canonical)}</a>.</span></p>`;
+}
+
+function renderPage({ mdRel, title, description, keywords, bodyHtml }) {
+  const crawlRel = crawlHtmlPath(mdRel);
+  const canonical = crawlPageUrl(crawlRel);
+  const interactiveDoc = `${SITE_ORIGIN}/${pageHash(mdRel)}`;
+  const depth = path.dirname(crawlRel).split("/").length - 1;
+  const indexHref = `${"../".repeat(depth)}index.html`;
+  const id = docId(mdRel);
+  const leadHtml = buildLeadParagraph({
+    title,
+    mdRel,
+    description,
+    canonical,
+    keywords,
+  });
+
+  return `<!DOCTYPE html>
+<html lang="fr" data-crawl-url="${escapeHtml(canonical)}" data-doc-id="${escapeHtml(id)}">
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <title>${escapeHtml(title)} — Concorde</title>
+  <meta name="description" content="${escapeHtml(description)}">
+  <meta name="keywords" content="${escapeHtml(keywords)}">
+  <meta name="concorde-doc-id" content="${escapeHtml(id)}">
+  <link rel="canonical" href="${escapeHtml(canonical)}">
+</head>
+<body>
+  <header>
+    <p><a href="${indexHref}">Concorde documentation (crawl)</a> · <a href="${escapeHtml(interactiveDoc)}" rel="nofollow">interactive version</a></p>
+  </header>
+  <main>
+    <article>
+      ${leadHtml}
+      ${bodyHtml}
+    </article>
+  </main>
+</body>
+</html>
+`;
+}
+
+function renderIndex(pages) {
+  const items = pages
+    .map(
+      (p) =>
+        `    <li><a href="${escapeHtml(p.crawlHref)}">${escapeHtml(p.title)}</a> <small>(${escapeHtml(p.docId)})</small></li>`,
+    )
+    .join("\n");
+
+  return `<!DOCTYPE html>
+<html lang="fr" data-doc-id="crawl-index">
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <title>Concorde documentation index</title>
+  <meta name="description" content="Static crawl index for Concorde framework documentation. All pages use .html under /crawl/.">
+  <meta name="keywords" content="Concorde, documentation index, web components, Lit, DataProvider">
+  <link rel="canonical" href="${SITE_ORIGIN}/crawl/index.html">
+</head>
+<body>
+  <header>
+    <h1>Concorde documentation</h1>
+    <p>Static pages for crawlers and AI indexing. Interactive docs: <a href="${SITE_ORIGIN}/">${SITE_ORIGIN}/</a></p>
+    <p>Full text mirror: <a href="${SITE_ORIGIN}/llms-full.txt">llms-full.txt</a></p>
+    <ul>
+      <li><a href="hubs/decorators.html">Decorators hub (@get, @post, …)</a></li>
+      <li><a href="hubs/remote-data.html">Remote data hub (@get, List, Queue)</a></li>
+      <li><a href="hubs/form-components.html">Form components hub</a></li>
+    </ul>
+  </header>
+  <main>
+    <ul>
+${items}
+    </ul>
+  </main>
+</body>
+</html>
+`;
+}
+
+function writeFileEnsuringDir(filePath, content) {
+  fs.mkdirSync(path.dirname(filePath), { recursive: true });
+  fs.writeFileSync(filePath, content);
+}
+
+function generateLlmsTxt(pages, hubs = []) {
+  const lines = [
+    "# Concorde",
+    "",
+    "> Web components framework (Lit, DataProvider, decorators).",
+    "",
+    "Documentation for AI indexers lives under `/crawl/` as **HTML** (`.html`), not Markdown.",
+    "Example: https://concorde.supersoniks.org/crawl/docs/_decorators/post.html",
+    "Do not fetch `/crawl/**/*.md` (does not exist) nor `/src/docs/**/*.md` (SPA source only).",
+    "",
+    `Full text: ${SITE_ORIGIN}/llms-full.txt`,
+    "",
+    "## Topic hubs (start here for RAG)",
+    "",
+  ];
+
+  for (const h of hubs) {
+    lines.push(
+      `- [${h.title}](${SITE_ORIGIN}/${h.crawlPath}) (${h.docId}, ${h.sectionCount} pages)`,
+    );
+  }
+
+  lines.push("", "## Docs", "");
+
+  for (const p of pages) {
+    lines.push(
+      `- [${p.title}](${SITE_ORIGIN}/${p.crawlPath}) (${p.docId}): ${p.description}`,
+    );
+  }
+
+  lines.push("", "## Optional", "", `- [Full index](${SITE_ORIGIN}/crawl/index.html)`);
+  return lines.join("\n");
+}
+
+function generateLlmsFullTxt(pages, hubs = []) {
+  const lines = [
+    "# Concorde — full documentation (plain text)",
+    "",
+    "> Mirror for AI indexers. Each page is a section below.",
+    "> HTML URLs use /crawl/**/*.html — never .md under /crawl/.",
+    "",
+    "## Topic hubs",
+    "",
+  ];
+
+  for (const h of hubs) {
+    lines.push(`- ${h.title}: ${SITE_ORIGIN}/${h.crawlPath} (${h.docId})`);
+  }
+
+  lines.push("");
+
+  for (const p of pages) {
+    lines.push(
+      "---",
+      "",
+      `## ${p.title}`,
+      "",
+      `- URL: ${SITE_ORIGIN}/${p.crawlPath}`,
+      `- Doc ID: ${p.docId}`,
+      `- Keywords: ${p.keywords}`,
+      "",
+      p.plainText,
+      "",
+    );
+  }
+
+  return lines.join("\n");
+}
+
+const TOPIC_HUBS = [
+  {
+    slug: "decorators",
+    title: "Concorde decorators reference",
+    docId: "hubs/decorators",
+    intro:
+      "Hub for all Concorde Lit decorators: @get @post @put @patch @subscribe @publish @bind @handle and related. Use this page when asking about any decorator.",
+    keywords:
+      "decorator, @get, @post, @put, @patch, @subscribe, @publish, @bind, @handle, ApiResult, Endpoint, DataProviderKey",
+    filter: (p) => p.docId.startsWith("docs/_decorators/"),
+  },
+  {
+    slug: "remote-data",
+    title: "Fetching remote data in Concorde",
+    docId: "hubs/remote-data",
+    intro:
+      "Hub for loading online data: @get decorator, Endpoint, API configuration, dynamic paths, sonic-list fetch, sonic-queue, legacy sonic-fetch.",
+    keywords:
+      "@get, fetch, remote data, API, Endpoint, ApiResult, serviceURL, sonic-list, sonic-queue, HTTP GET",
+    filter: (p) =>
+      [
+        "docs/_decorators/get",
+        "docs/_decorators/post",
+        "docs/_decorators/put",
+        "docs/_decorators/patch",
+        "docs/_misc/endpoint",
+        "docs/_misc/api-configuration",
+        "docs/_misc/dynamic-path",
+        "docs/_core-concept/dataFlow",
+        "core/components/functional/fetch/fetch",
+        "core/components/functional/list/list",
+        "core/components/functional/queue/queue",
+      ].includes(p.docId),
+  },
+  {
+    slug: "form-components",
+    title: "Concorde form components",
+    docId: "hubs/form-components",
+    intro:
+      "Hub for form UI: sonic-input, sonic-select, sonic-checkbox, sonic-submit, formDataProvider, FormElement.",
+    keywords:
+      "form, FormElement, formDataProvider, sonic-input, sonic-select, sonic-checkbox, sonic-submit",
+    filter: (p) => p.docId.includes("/components/ui/form/"),
+  },
+];
+
+function hubCrawlPath(slug) {
+  return `crawl/hubs/${slug}.html`;
+}
+
+function relativeHubLink(fromHubSlug, targetCrawlPath) {
+  const from = hubCrawlPath(fromHubSlug);
+  const rel = path.relative(path.dirname(from), targetCrawlPath).replace(/\\/g, "/");
+  return rel.startsWith(".") ? rel : `./${rel}`;
+}
+
+function renderHubPage(hub, sections) {
+  const crawlRel = hubCrawlPath(hub.slug);
+  const canonical = crawlPageUrl(crawlRel);
+  const sectionHtml = sections
+    .map((p) => {
+      const excerpt = p.plainText.slice(0, 1200);
+      const href = relativeHubLink(hub.slug, p.crawlPath);
+      return `<section id="${escapeHtml(p.docId.replace(/\//g, "-"))}" data-doc-id="${escapeHtml(p.docId)}">
+  <h2><a href="${escapeHtml(href)}">${escapeHtml(p.title)}</a></h2>
+  <p><strong>Doc ID:</strong> <code>${escapeHtml(p.docId)}</code> · <a href="${escapeHtml(href)}">full page</a></p>
+  <p>${escapeHtml(p.description)}</p>
+  <pre class="doc-excerpt">${escapeHtml(excerpt)}</pre>
+</section>`;
+    })
+    .join("\n");
+
+  const leadHtml = buildLeadParagraph({
+    title: hub.title,
+    mdRel: `${hub.docId}.md`,
+    description: hub.intro,
+    canonical,
+    keywords: hub.keywords,
+  });
+
+  return `<!DOCTYPE html>
+<html lang="fr" data-crawl-url="${escapeHtml(canonical)}" data-doc-id="${escapeHtml(hub.docId)}">
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <title>${escapeHtml(hub.title)} — Concorde</title>
+  <meta name="description" content="${escapeHtml(hub.intro)}">
+  <meta name="keywords" content="${escapeHtml(hub.keywords)}">
+  <meta name="concorde-doc-id" content="${escapeHtml(hub.docId)}">
+  <link rel="canonical" href="${escapeHtml(canonical)}">
+</head>
+<body>
+  <header>
+    <p><a href="../index.html">Concorde documentation (crawl)</a></p>
+  </header>
+  <main>
+    <article>
+      ${leadHtml}
+      ${sectionHtml}
+    </article>
+  </main>
+</body>
+</html>
+`;
+}
+
+function generateTopicHubs(pages) {
+  const hubs = [];
+  for (const hub of TOPIC_HUBS) {
+    const sections = pages.filter(hub.filter);
+    if (sections.length === 0) continue;
+    const crawlRel = hubCrawlPath(hub.slug);
+    writeFileEnsuringDir(
+      path.join(DOCS_OUT, crawlRel),
+      renderHubPage(hub, sections),
+    );
+    hubs.push({
+      ...hub,
+      crawlPath: crawlRel,
+      sectionCount: sections.length,
+    });
+  }
+  return hubs;
+}
+
+function generateSitemap(pages, hubs = []) {
+  const indexUrl = `  <url>\n    <loc>${SITE_ORIGIN}/crawl/index.html</loc>\n  </url>`;
+  const hubUrls = hubs
+    .map((h) => `  <url>\n    <loc>${SITE_ORIGIN}/${h.crawlPath}</loc>\n  </url>`)
+    .join("\n");
+  const urls = pages
+    .map((p) => `  <url>\n    <loc>${SITE_ORIGIN}/${p.crawlPath}</loc>\n  </url>`)
+    .join("\n");
+
+  return `<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+${indexUrl}
+${hubUrls}
+${urls}
+</urlset>
+`;
+}
+
+export function generateCrawlDocs() {
+  if (!fs.existsSync(DOCS_OUT)) {
+    console.warn("generate-crawl-docs: docs/ not found, skipping");
+    return;
+  }
+
+  if (fs.existsSync(CRAWL_DIR)) {
+    fs.rmSync(CRAWL_DIR, { recursive: true, force: true });
+  }
+
+  const mdFiles = listMdFiles(SRC_DIR).sort();
+  const pages = [];
+
+  for (const absPath of mdFiles) {
+    const mdRel = mdRelativePath(absPath);
+    const raw = fs.readFileSync(absPath, "utf8");
+    const title = extractTitle(raw);
+    const description = extractDescription(raw, title);
+    const keywords = extractKeywords(mdRel, title, raw);
+    const plainText = markdownToPlainText(raw);
+    const markdown = rewriteDocLinks(preprocessSonicCode(raw), mdRel);
+    const bodyHtml = marked.parse(markdown);
+    const crawlRel = crawlHtmlPath(mdRel);
+    const outPath = path.join(DOCS_OUT, crawlRel);
+    const id = docId(mdRel);
+
+    writeFileEnsuringDir(
+      outPath,
+      renderPage({ mdRel, title, description, keywords, bodyHtml }),
+    );
+
+    pages.push({
+      mdRel,
+      docId: id,
+      title,
+      description,
+      keywords,
+      plainText,
+      crawlPath: crawlRel,
+      crawlHref: crawlRel.replace(/^crawl\//, ""),
+    });
+  }
+
+  writeFileEnsuringDir(
+    path.join(CRAWL_DIR, "index.html"),
+    renderIndex(
+      pages.map((p) => ({
+        ...p,
+        crawlHref: p.mdRel.replace(/\.mdx?$/, ".html"),
+      })),
+    ),
+  );
+
+  const hubs = generateTopicHubs(pages);
+
+  fs.writeFileSync(path.join(DOCS_OUT, "sitemap.xml"), generateSitemap(pages, hubs));
+  fs.writeFileSync(path.join(DOCS_OUT, "llms.txt"), generateLlmsTxt(pages, hubs));
+  fs.writeFileSync(
+    path.join(DOCS_OUT, "llms-full.txt"),
+    generateLlmsFullTxt(pages, hubs),
+  );
+
+  console.info(
+    `generate-crawl-docs: ${pages.length} pages, ${hubs.length} hubs → docs/crawl/, sitemap.xml, llms.txt, llms-full.txt`,
+  );
+}
+
+if (process.argv[1] === fileURLToPath(import.meta.url)) {
+  generateCrawlDocs();
+}

package/scripts/post-build-docs.js

@@ -1,6 +1,6 @@
 import fs from "fs";
 import path from "path";
-import { exec } from "child_process";
+import { generateCrawlDocs } from "./generate-crawl-docs.mjs";
 
 const src = path.resolve("src");
 const docs = path.resolve("docs");
@@ -46,34 +46,32 @@ function copyDocs() {
   }
 }
 copyDocs();
+generateCrawlDocs();
 
 /**
  * Par rapport au projet, regarder si ../concorde_doc existe et si oui, remplacer le contenu de /concorde_doc par celui de contenu dans docs/ en dehors des fichiers spécifiques à git .git .gitignore .gitattributes)
  */
 
 function copyToConcordeDoc() {
-
-  
   let concordeDoc = path.resolve("../concorde_doc");
-  
-  
-  
-  if (fs.existsSync(concordeDoc)) {
+
+  if (!fs.existsSync(concordeDoc)) {
+    return;
+  }
+
+  try {
     /* first remove all files/foilders in concorde_doc  except .git folder and .gitignore file */
     let files = listFiles(concordeDoc);
     for (const file of files) {
       if (file.indexOf(".git") == -1) {
         fs.unlinkSync(file);
       }
-
     }
 
-
     /*then copy the new files*/
     files = listFiles(docs);
     for (const file of files) {
-
-      let idx = file.indexOf("docs/")+5;
+      let idx = file.indexOf("docs/") + 5;
       let dest = file.substring(idx);
       let split = dest.split("/");
       let dir = path.resolve(concordeDoc);
@@ -86,13 +84,19 @@ function copyToConcordeDoc() {
       dir += "/" + split[split.length - 1];
       fs.copyFileSync(file, dir);
     }
-    //log an Emoji finger showing the user what to do next
 
-  
     console.info("---");
     console.info("NOW run the following command to publish the docs");
-    console.info("👉 cd ../concorde_doc && git add . && git commit -m 'update doc' && git push && cd ../concorde");
+    console.info(
+      "👉 cd ../concorde_doc && git add . && git commit -m 'update doc' && git push && cd ../concorde",
+    );
     console.info("---");
+  } catch (err) {
+    console.warn(
+      "copyToConcordeDoc: skipped (",
+      err.code || err.message,
+      ") — docs/ and docs/crawl/ were still generated.",
+    );
   }
 }