@supersoniks/concorde 5.1.1 → 5.1.3-immutable.1

package/scripts/generate-crawl-docs.test.mjs

@@ -0,0 +1,124 @@
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import os from "node:os";
+import path from "path";
+import test from "node:test";
+import {
+  generateHomepageMarkdown,
+  generateHtaccess,
+  generateNotFoundHtml,
+  generateNotFoundMarkdown,
+  pagePath,
+  writeAgentSiteArtifacts,
+} from "./generate-crawl-docs.mjs";
+
+function stripTags(html) {
+  return html
+    .replace(/<script[\s\S]*?<\/script>/gi, " ")
+    .replace(/<style[\s\S]*?<\/style>/gi, " ")
+    .replace(/<[^>]+>/g, " ")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+test("pagePath builds pretty public pathnames", () => {
+  assert.equal(
+    pagePath("docs/getting-started/start.md"),
+    "/docs/getting-started/start",
+  );
+  assert.equal(
+    pagePath("core/components/ui/button/button.md"),
+    "/core/components/ui/button",
+  );
+  assert.equal(
+    pagePath("core/components/functional/sdui/sdui-extension-points.md"),
+    "/core/components/functional/sdui/sdui-extension-points",
+  );
+});
+
+test("homepage Markdown meets Accept negotiation body requirements", () => {
+  const md = generateHomepageMarkdown();
+  assert.match(md, /^# Concorde by Supersoniks/m);
+  assert.match(md, /@supersoniks\/concorde/);
+  assert.match(md, /llms\.txt/);
+  assert.match(md, /Accept: text\/markdown/);
+  assert.match(md, /docs\/getting-started\/start/);
+  assert.doesNotMatch(md, /\/crawl\//);
+  assert.doesNotMatch(md, /\/docs\/_getting-started\//);
+  assert.ok(md.length >= 500, `expected ≥500 chars, got ${md.length}`);
+});
+
+test("404 Markdown explains the error and links agent entry points", () => {
+  const md = generateNotFoundMarkdown();
+  assert.ok(md.length >= 20);
+  assert.match(md, /not found/i);
+  assert.match(md, /llms\.txt|sitemap\.xml|docs\/index/);
+  assert.doesNotMatch(md, /\/crawl\//);
+  assert.match(md, /https:\/\/concorde\.supersoniks\.org/);
+});
+
+test("404 HTML keeps a human-readable fallback", () => {
+  const html = generateNotFoundHtml();
+  assert.match(html, /<h1>Page not found<\/h1>/);
+  assert.match(html, /llms\.txt/);
+  assert.doesNotMatch(html, /\/crawl\//);
+});
+
+test("htaccess maps components and negotiates Markdown/HTML", () => {
+  const conf = generateHtaccess();
+  assert.match(conf, /RewriteEngine On/);
+  assert.match(conf, /text\/markdown/);
+  assert.match(conf, /index\.md/);
+  assert.match(conf, /ErrorDocument 404 \/404\.md/);
+  assert.match(conf, /\$1\/\$2\/\$2\.md/);
+  assert.match(conf, /Googlebot/);
+  assert.match(conf, /index\.html \[END\]/);
+  assert.match(conf, /DirectorySlash Off/);
+  assert.doesNotMatch(conf, /docs\/_/);
+  assert.doesNotMatch(conf, /THE_REQUEST/);
+  assert.doesNotMatch(conf, /\^crawl\//);
+});
+
+test("writeAgentSiteArtifacts emits negotiation files", () => {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), "concorde-agent-"));
+  try {
+    writeAgentSiteArtifacts(dir);
+    for (const name of ["index.md", "404.md", "404.html", ".htaccess"]) {
+      const filePath = path.join(dir, name);
+      assert.ok(fs.existsSync(filePath), `missing ${name}`);
+      assert.ok(fs.statSync(filePath).size > 20, `${name} too small`);
+    }
+  } finally {
+    fs.rmSync(dir, { recursive: true, force: true });
+  }
+});
+
+test("source index.html ships ≥500 chars of meaningful agent content with H1", () => {
+  const indexPath = path.resolve("index.html");
+  const html = fs.readFileSync(indexPath, "utf8");
+  assert.match(html, /id="docs-agent-content"/);
+  assert.match(html, /<h1>Concorde by Supersoniks<\/h1>/);
+  assert.match(html, /href="\/docs\/getting-started\/start"/);
+  assert.doesNotMatch(html, /\/docs\/_getting-started\//);
+  assert.doesNotMatch(html, /\/crawl\//);
+
+  const section = html.match(
+    /<section id="docs-agent-content"[^>]*>([\s\S]*?)<\/section>/,
+  );
+  assert.ok(section, "docs-agent-content section missing");
+  const text = stripTags(section[1]);
+  assert.ok(
+    text.length >= 500,
+    `agent section text expected ≥500 chars, got ${text.length}`,
+  );
+});
+
+test("public robots.txt allows pathnames and drops /crawl/", () => {
+  const robots = fs.readFileSync(path.resolve("public/robots.txt"), "utf8");
+  assert.match(robots, /Allow: \//);
+  assert.match(robots, /Allow: \/docs\//);
+  assert.match(robots, /Allow: \/core\//);
+  assert.match(robots, /Allow: \/llms\.txt/);
+  assert.doesNotMatch(robots, /Allow: \/crawl\//);
+  assert.doesNotMatch(robots, /Googlebot[\s\S]*Disallow: \//);
+});

package/scripts/generate-sdui-catalog.mjs

@@ -14,7 +14,7 @@ import { fileURLToPath } from "url";
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 const ROOT = path.resolve(__dirname, "..");
 const CEM_DIR = path.join(ROOT, ".tmp-cem");
-const CEM_PATH = path.join(CEM_DIR, "custom-elements.json");
+export const CEM_PATH = path.join(CEM_DIR, "custom-elements.json");
 const OUT_CATALOG = path.join(
   ROOT,
   "src/core/components/functional/sdui/component-catalog.json"
@@ -24,7 +24,7 @@ const OUT_GAPS = path.join(
   "src/core/components/functional/sdui/component-catalog.gaps.json"
 );
 
-function extractTagNameFromSource(filePath) {
+export function extractTagNameFromSource(filePath) {
   if (!fs.existsSync(filePath)) return undefined;
   const src = fs.readFileSync(filePath, "utf8");
   const literal = src.match(/@customElement\(\s*["'`]([^"'`]+)["'`]\s*\)/);
@@ -176,8 +176,12 @@ export function cemToCatalog(cem, { resolveTagName } = {}) {
   };
 }
 
-function runCem() {
+export function runCem(
+  configPath = path.join(ROOT, "custom-elements-manifest.config.mjs"),
+) {
   fs.mkdirSync(CEM_DIR, { recursive: true });
+  const pkgPath = path.join(ROOT, "package.json");
+  const pkgBefore = fs.readFileSync(pkgPath, "utf8");
   const cemBin = path.join(
     ROOT,
     "node_modules/@custom-elements-manifest/analyzer/cem.js"
@@ -188,16 +192,13 @@ function runCem() {
       cemBin,
       "analyze",
       "--config",
-      path.join(ROOT, "custom-elements-manifest.config.mjs"),
+      configPath,
     ],
     { cwd: ROOT, stdio: ["ignore", "pipe", "pipe"] }
   );
-  // Ne pas laisser cem écrire customElements dans package.json
-  const pkgPath = path.join(ROOT, "package.json");
-  const pkg = JSON.parse(fs.readFileSync(pkgPath, "utf8"));
-  if (pkg.customElements) {
-    delete pkg.customElements;
-    fs.writeFileSync(pkgPath, JSON.stringify(pkg, null, 2) + "\n");
+  // cem may rewrite package.json (customElements): keep it as it was
+  if (fs.readFileSync(pkgPath, "utf8") !== pkgBefore) {
+    fs.writeFileSync(pkgPath, pkgBefore);
   }
 }
 

package/scripts/github-export.mjs

@@ -0,0 +1,150 @@
+#!/usr/bin/env node
+/**
+ * Prépare une arborescence « lib publique » pour GitHub :
+ * - sans site doc (src/docs, index.html, …)
+ * - sans sonic-captcha (altcha.supersoniks.org)
+ * - sans scripts / jobs GitLab internes
+ *
+ * Usage :
+ *   node scripts/github-export.mjs [répertoire_cible]
+ *
+ * Par défaut : ../concorde-github-export
+ */
+import fs from "fs";
+import path from "path";
+import { fileURLToPath } from "url";
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const ROOT = path.resolve(__dirname, "..");
+const IGNORE_FILE = path.join(ROOT, ".github-export-ignore");
+const DEFAULT_OUT = path.resolve(ROOT, "../concorde-github-export");
+
+const EXPORT_FILTER_KEY =
+  /(?:^|\/)(?:docs|mock-api|undefined)(?:\/|$)|captcha|\.demo\.|\.stories\.|TestUtils|\/prism\/|\/search\/|\/tailwind\/|\/header\/|\/navigation\/|\/layout\.ts$|\/code\.ts$|^code$|^docs$|^layout$/;
+
+function readIgnorePatterns() {
+  if (!fs.existsSync(IGNORE_FILE)) return [];
+  return fs
+    .readFileSync(IGNORE_FILE, "utf8")
+    .split("\n")
+    .map((l) => l.trim())
+    .filter((l) => l && !l.startsWith("#"));
+}
+
+function globToRegExp(pattern) {
+  const escaped = pattern
+    .replace(/[.+^${}()|[\]\\]/g, "\\$&")
+    .replace(/\*\*/g, "{{GLOBSTAR}}")
+    .replace(/\*/g, "[^/]*")
+    .replace(/\{\{GLOBSTAR\}\}/g, ".*");
+  return new RegExp(`^${escaped}$`);
+}
+
+function shouldExclude(relativePath, patterns) {
+  const normalized = relativePath.replace(/\\/g, "/");
+  for (const pattern of patterns) {
+    const re = globToRegExp(pattern);
+    if (re.test(normalized)) return true;
+    if (normalized.startsWith(pattern.endsWith("/") ? pattern : `${pattern}/`))
+      return true;
+    if (normalized === pattern) return true;
+  }
+  return false;
+}
+
+function listAllFiles(dir, base = dir) {
+  const out = [];
+  for (const name of fs.readdirSync(dir, { withFileTypes: true })) {
+    if (name.name === ".git" || name.name === "node_modules") continue;
+    const full = path.join(dir, name.name);
+    const rel = path.relative(base, full).replace(/\\/g, "/");
+    if (name.isDirectory()) {
+      if (shouldExclude(rel, ignorePatterns)) continue;
+      out.push(...listAllFiles(full, base));
+    } else {
+      if (!shouldExclude(rel, ignorePatterns)) out.push(rel);
+    }
+  }
+  return out;
+}
+
+function copyFile(rel, destRoot) {
+  const src = path.join(ROOT, rel);
+  const dest = path.join(destRoot, rel);
+  fs.mkdirSync(path.dirname(dest), { recursive: true });
+  fs.copyFileSync(src, dest);
+}
+
+function patchUiTs(destRoot) {
+  const uiPath = path.join(
+    destRoot,
+    "src/core/components/ui/ui.ts",
+  );
+  if (!fs.existsSync(uiPath)) return;
+  const content = fs.readFileSync(uiPath, "utf8");
+  const patched = content.replace(
+    /import "\.\/captcha\/captcha";\n?/,
+    "",
+  );
+  fs.writeFileSync(uiPath, patched);
+}
+
+function exportHaystack(value) {
+  if (typeof value === "string") return value;
+  if (value && typeof value === "object") {
+    return `${value.types ?? ""}${value.default ?? ""}`;
+  }
+  return String(value);
+}
+
+function filterPackageJson(destRoot) {
+  const pkgPath = path.join(destRoot, "package.json");
+  if (!fs.existsSync(pkgPath)) return;
+  const pkg = JSON.parse(fs.readFileSync(pkgPath, "utf8"));
+  if (pkg.exports && typeof pkg.exports === "object") {
+    const next = {};
+    for (const [key, value] of Object.entries(pkg.exports)) {
+      const hay = `${key}:${exportHaystack(value)}`;
+      if (EXPORT_FILTER_KEY.test(hay)) continue;
+      if (exportHaystack(value).includes("/captcha/")) continue;
+      if (exportHaystack(value).includes("/docs/")) continue;
+      if (exportHaystack(value).includes(".demo.")) continue;
+      if (exportHaystack(value).includes(".stories.")) continue;
+      next[key] = value;
+    }
+    pkg.exports = next;
+  }
+  fs.writeFileSync(pkgPath, JSON.stringify(pkg, null, 2) + "\n");
+}
+
+function patchReadme(destRoot) {
+  const readmePath = path.join(destRoot, "README.md");
+  if (!fs.existsSync(readmePath)) return;
+  let text = fs.readFileSync(readmePath, "utf8");
+  text = text.replace(
+    /Le dépôt GitHub est un \*\*miroir\*\* synchronisé depuis GitLab \(voir \[OPEN_SOURCE_CHECKPOINT\.md\]\(OPEN_SOURCE_CHECKPOINT\.md\)\)\./,
+    "Le dépôt GitHub est un **export filtré** depuis GitLab (librairie uniquement, sans site doc ni composant captcha — voir [OPEN_SOURCE_CHECKPOINT.md](OPEN_SOURCE_CHECKPOINT.md)).",
+  );
+  fs.writeFileSync(readmePath, text);
+}
+
+const ignorePatterns = readIgnorePatterns();
+const outDir = path.resolve(process.argv[2] || DEFAULT_OUT);
+
+if (fs.existsSync(outDir)) {
+  fs.rmSync(outDir, { recursive: true, force: true });
+}
+fs.mkdirSync(outDir, { recursive: true });
+
+const files = listAllFiles(ROOT);
+for (const rel of files) {
+  copyFile(rel, outDir);
+}
+
+patchUiTs(outDir);
+filterPackageJson(outDir);
+patchReadme(outDir);
+
+console.log(`✅ Export GitHub : ${files.length} fichiers → ${outDir}`);
+console.log("   Exclusions : .github-export-ignore");
+console.log("   Patches    : ui.ts (sans captcha), package.json exports");

package/scripts/post-build-docs.js

@@ -1,57 +1,62 @@
 import fs from "fs";
 import path from "path";
 import { generateCrawlDocs } from "./generate-crawl-docs.mjs";
+import { generateCem } from "./generate-cem.mjs";
 
 const src = path.resolve("src");
 const docs = path.resolve("docs");
-/**
- * lister les fichiers d'un répertoire de manière récursive
- */
+
 function listFiles(dir) {
   let files = [];
   let list = fs.readdirSync(dir);
   list.forEach((file) => {
-    const path = dir + "/" + file;
-    let stat = fs.statSync(path);
+    const filePath = dir + "/" + file;
+    let stat = fs.statSync(filePath);
     if (stat && stat.isDirectory()) {
-      files = files.concat(listFiles(path));
+      files = files.concat(listFiles(filePath));
     } else {
-      files.push(path);
+      files.push(filePath);
     }
   });
   return files;
 }
 
+function ensureCopy(absSrc, absDest) {
+  fs.mkdirSync(path.dirname(absDest), { recursive: true });
+  fs.copyFileSync(absSrc, absDest);
+}
+
 /**
- * copier toues les fichiers .md présents dans le répertpoire src et ses sous dossiers de manière récurcive vers dossier docs en respectant l'arborescence qu'il y avait dans le dossier src
- * il faut creer le chemin si il n'existe pas avant de copier le fichier
+ * Publish markdown at the public pathname (`/docs/…/page.md`, `/core/…/page.md`)
+ * and keep a `/src/…` copy for non-md assets (and legacy MD until 301).
  */
 function copyDocs() {
-  let files = listFiles(src);
+  const files = listFiles(src);
   for (const file of files) {
-    if (file.indexOf(".md") != -1 || file.indexOf(".json") != -1) {
-      let idx = file.indexOf("src/");
-      let dest = file.substring(idx);
-      let split = dest.split("/");
-      let dir = path.resolve(docs);
-      for (let i = 0; i < split.length - 1; i++) {
-        dir += "/" + split[i];
-        if (!fs.existsSync(dir)) {
-          fs.mkdirSync(dir);
-        }
-      }
-      dir += "/" + split[split.length - 1];
-      fs.copyFileSync(file, dir);
+    const isMd = /\.mdx?$/i.test(file);
+    const isJson = file.endsWith(".json");
+    if (!isMd && !isJson) continue;
+
+    const idx = file.indexOf("src/");
+    if (idx === -1) continue;
+    const underSrc = file.substring(idx); // src/docs/…/x.md
+    const withoutSrc = underSrc.replace(/^src\//, ""); // docs/…/x.md
+
+    // Always keep under docs/src/… (json + legacy md paths)
+    ensureCopy(file, path.join(docs, underSrc));
+
+    // Public MD URLs: /docs/… and /core/… (+ .md)
+    if (isMd && /^(docs|core)\//.test(withoutSrc)) {
+      ensureCopy(file, path.join(docs, withoutSrc));
     }
   }
 }
+
 copyDocs();
+// Custom Elements Manifest, served at /custom-elements.json
+fs.copyFileSync(generateCem(), path.join(docs, "custom-elements.json"));
 generateCrawlDocs();
 
-/**
- * Par rapport au projet, regarder si ../concorde_doc existe et si oui, remplacer le contenu de /concorde_doc par celui de contenu dans docs/ en dehors des fichiers spécifiques à git .git .gitignore .gitattributes)
- */
-
 function copyToConcordeDoc() {
   let concordeDoc = path.resolve("../concorde_doc");
 
@@ -60,15 +65,17 @@ function copyToConcordeDoc() {
   }
 
   try {
-    /* first remove all files/foilders in concorde_doc  except .git folder and .gitignore file */
     let files = listFiles(concordeDoc);
     for (const file of files) {
       if (file.indexOf(".git") == -1) {
         fs.unlinkSync(file);
       }
     }
+    const legacyCrawl = path.join(concordeDoc, "crawl");
+    if (fs.existsSync(legacyCrawl)) {
+      fs.rmSync(legacyCrawl, { recursive: true, force: true });
+    }
 
-    /*then copy the new files*/
     files = listFiles(docs);
     for (const file of files) {
       let idx = file.indexOf("docs/") + 5;
@@ -95,29 +102,9 @@ function copyToConcordeDoc() {
     console.warn(
       "copyToConcordeDoc: skipped (",
       err.code || err.message,
-      ") — docs/ and docs/crawl/ were still generated.",
+      ") — docs/ pathnames (HTML + MD + hubs) were still generated.",
     );
   }
 }
 
 copyToConcordeDoc();
-
-/**
- * dans ../concorde_doc, faire un git  add, commit et push
- */
-/* 
-function gitConcordeDoc() {
-  let concordeDoc = path.resolve("../concorde_doc");
-  if (fs.existsSync(concordeDoc)) {
-    exec("cd " + concordeDoc + " && git add . && git commit -m 'update doc' && git push", (error, stdout, stderr) => {
-      if (error) {
-        console.error(`exec error: ${error}`);
-        return;
-      }
-      console.log(`stdout: ${stdout}`);
-      console.error(`stderr: ${stderr}`);
-    });
-  }
-}
-
-gitConcordeDoc(); */

package/scripts/pre-publish.mjs

@@ -4,8 +4,9 @@ import fs from "fs";
 const requiredFiles = [
   'concorde-core.bundle.js',
   'concorde-core.es.js',
-  'ai/AGENTS.md',
+  'ai/skills/concorde/SKILL.md',
   'scripts/ai-init.mjs',
+  'custom-elements.json',
 ];
 
 let allFilesExist = true;