@palate/cli 0.0.0-stage → 0.0.2

package/src/refs.mjs

@@ -0,0 +1,299 @@
+// `palate refs`: the librarian. Shelf → filter → rank → diversify → three directions, then fetch only
+// those images so the agent can look at them. Deterministic and offline once the catalogue is synced.
+import fs from "node:fs";
+import path from "node:path";
+import { CACHE_DIR, readCatalogs, readNotes, addNote } from "./catalog/store.mjs";
+import { fontClass, parseColor, luminance } from "./catalog/classify.mjs";
+import { curatedRecords } from "./catalog/curated.mjs";
+import { capture, measuresFor } from "./capture.mjs";
+import { readIndex } from "./catalog/index-build.mjs";
+
+const onShelf = (r, shelf) => (r.shelves ?? [r.shelf]).includes(shelf);
+
+// Once palate has captured a site, its own measurements (theme, fonts) improve that record for future picks.
+function withCapturedFacts(r) {
+  if (!r.site) return r;
+  const m = measuresFor(r.site);
+  if (!m) return r;
+  const bg = m.backgrounds?.[0];
+  const rgb = bg && parseColor(bg);
+  return {
+    ...r,
+    theme: rgb ? (luminance(rgb) > 0.4 ? "light" : "dark") : r.theme,
+    fonts: r.fonts?.length ? r.fonts : m.fonts,
+    typeClasses: r.typeClasses?.length ? r.typeClasses : [...new Set((m.fonts ?? []).map(fontClass).filter(Boolean))],
+    measured: m,
+  };
+}
+// Curated rows that point at a catalogued page (e.g. a vibrant.design entry) merge into that record.
+export function allRecords() {
+  // Shipped index first, then anything synced locally (same ids replace the shipped copy).
+  const byId = new Map(readIndex().map((r) => [r.id, r]));
+  for (const r of readCatalogs()) byId.set(r.id, { ...byId.get(r.id), ...r });
+  const catalog = [...byId.values()].map(withCapturedFacts);
+  const norm = (u) => u.replace(/[?#].*$/, "").replace(/\/$/, "");
+  const byUrl = new Map(catalog.map((r) => [norm(r.url), r]));
+  const curated = [];
+  for (const c of curatedRecords()) {
+    const hit = byUrl.get(norm(c.url));
+    if (hit) {
+      hit.curated = true;
+      hit.description = [hit.description, c.description].filter(Boolean).join(" ");
+    } else if (/vibrant\.design/.test(c.url)) {
+      curated.push({ ...c, capture: false }); // not synced yet: no live capture of the gallery page itself
+    } else curated.push(c);
+  }
+  return [...catalog, ...curated];
+}
+
+// Shelves that can stand in for each other when one is thin.
+const NEIGHBOURS = {
+  cta: ["hero"],
+  hero: ["website"],
+  features: ["bento", "how-it-works"],
+  bento: ["features"],
+  "how-it-works": ["features"],
+  "data-table": ["dashboard", "app-shell"],
+  dashboard: ["data-table", "app-shell"],
+  settings: ["app-shell", "contact"],
+  "app-shell": ["dashboard", "data-table"],
+  canvas: ["app-shell"],
+  docs: ["changelog"],
+  "product-page": ["collections"],
+  "logo-cloud": ["testimonials", "stats"],
+  testimonials: ["logo-cloud"],
+  stats: ["features"],
+  blogs: ["changelog"],
+  changelog: ["blogs"],
+  pricing: ["website"],
+  faq: ["pricing"],
+  website: ["hero"],
+};
+
+// Words people use for screens → catalogue shelves.
+const SHELF_ALIASES = {
+  landing: "website", homepage: "website", "landing-page": "website", home: "website",
+  "logo-strip": "logo-cloud", logos: "logo-cloud", reviews: "testimonials", quotes: "testimonials",
+  signup: "login-and-signup", login: "login-and-signup", auth: "login-and-signup",
+  blog: "blogs", "release-notes": "changelog", footer: "footer", "call-to-action": "cta",
+};
+
+const words = (s) => (Array.isArray(s) ? s.join(" ") : String(s ?? "")).toLowerCase().match(/[a-z0-9]+/g) ?? [];
+const STOP = new Set(["the", "and", "for", "with", "that", "this", "from", "into", "your", "our", "page", "section", "design", "website"]);
+
+function docText(r) {
+  return [r.name, r.description, r.siteTitle, r.siteDescription, r.siteName, r.industry, r.style, r.colorTags, r.interaction, r.audience, r.features, r.keywords, r.fonts].flat().join(" ");
+}
+
+// Plain BM25 over the catalogue documents.
+function bm25Index(records) {
+  const docs = records.map((r) => words(docText(r)).filter((w) => !STOP.has(w)));
+  const df = new Map();
+  for (const d of docs) for (const w of new Set(d)) df.set(w, (df.get(w) ?? 0) + 1);
+  const avg = docs.reduce((n, d) => n + d.length, 0) / Math.max(1, docs.length);
+  const N = docs.length;
+  return (qWords, i) => {
+    const d = docs[i];
+    const tf = new Map();
+    for (const w of d) tf.set(w, (tf.get(w) ?? 0) + 1);
+    let s = 0;
+    for (const w of qWords) {
+      const f = tf.get(w);
+      if (!f) continue;
+      const idf = Math.log(1 + (N - df.get(w) + 0.5) / (df.get(w) + 0.5));
+      s += idf * ((f * 2.2) / (f + 1.2 * (0.25 + 0.75 * (d.length / avg))));
+    }
+    return s;
+  };
+}
+
+// What the codebase implies: light or dark base, and the class of its main typeface.
+export function projectProfile(dir) {
+  const profile = {};
+  const cssFiles = [];
+  const walk = (d, depth) => {
+    if (depth < 0 || !fs.existsSync(d)) return;
+    for (const e of fs.readdirSync(d, { withFileTypes: true })) {
+      if (["node_modules", ".git", "dist", ".palate"].includes(e.name)) continue;
+      const p = path.join(d, e.name);
+      if (e.isDirectory()) walk(p, depth - 1);
+      else if (/\.(css|scss)$/.test(e.name) && !/\.module\./.test(e.name)) cssFiles.push(p);
+    }
+  };
+  walk(path.join(dir, "src"), 3);
+  walk(path.join(dir, "app"), 3);
+  walk(path.join(dir, "styles"), 2);
+  for (const f of cssFiles) {
+    const css = fs.readFileSync(f, "utf8");
+    // The first background-like token in :root decides the base theme.
+    // Only the light (default) block: stop at the first `.dark {` or `[data-theme=dark] {` rule.
+    const root = css.split(/\n\s*(?:\.dark|\[data-theme=["']?dark["']?\])\s*\{/)[0];
+    const bg = /--(?:color-)?(?:background|bg|paper|surface|canvas)\s*:\s*([^;]+);/i.exec(root);
+    const rgb = bg && parseColor(bg[1]);
+    if (rgb && !profile.theme) profile.theme = luminance(rgb) > 0.4 ? "light" : "dark";
+    const font = /--font-(?:sans|body|display|heading)?[\w-]*\s*:\s*([^;]+);|font-family:\s*([^;]+);/i.exec(css);
+    const name = font && (font[1] ?? font[2]).split(",")[0].replace(/["']/g, "").trim();
+    if (name && !name.startsWith("var(") && !profile.font) {
+      profile.font = name;
+      profile.typeClass = fontClass(name);
+    }
+  }
+  const html = fs.existsSync(path.join(dir, "index.html")) ? fs.readFileSync(path.join(dir, "index.html"), "utf8") : "";
+  const gf = /fonts\.googleapis\.com\/css2\?family=([^"&:]+)/.exec(html);
+  if (gf && !profile.font) {
+    profile.font = decodeURIComponent(gf[1]).replace(/\+/g, " ");
+    profile.typeClass = fontClass(profile.font);
+  }
+  return profile;
+}
+
+const TYPE_NEAR = {
+  "geometric-sans": ["grotesk", "humanist-sans"],
+  grotesk: ["geometric-sans", "humanist-sans"],
+  "humanist-sans": ["grotesk", "geometric-sans"],
+  serif: ["display"],
+  display: ["serif", "geometric-sans"],
+  mono: ["grotesk"],
+};
+
+function jaccard(a, b) {
+  const A = new Set(a);
+  const B = new Set(b);
+  if (!A.size || !B.size) return 0;
+  let inter = 0;
+  for (const x of A) if (B.has(x)) inter++;
+  return inter / (A.size + B.size - inter);
+}
+
+export function findRefs(query, { dir, count = 3 } = {}) {
+  const all = allRecords();
+  if (!all.length) throw new Error("No references yet. Run: palate catalog sync vibrant");
+  const notes = readNotes();
+  const profile = dir ? projectProfile(dir) : {};
+  const theme = query.theme ?? profile.theme;
+
+  // 1. Shelf (with neighbours as a fallback when the shelf is thin).
+  const want = query.shelf ? SHELF_ALIASES[query.shelf] ?? query.shelf : undefined;
+  let shelves = want ? [want] : [];
+  // Interaction posts (shelf "micro") only appear when asked for.
+  const browsable = want === "micro" ? all : all.filter((r) => r.shelf !== "micro");
+  let pool = want ? browsable.filter((r) => onShelf(r, want)) : browsable;
+  if (want && pool.length < 12) {
+    shelves = [want, ...(NEIGHBOURS[want] ?? [])];
+    pool = browsable.filter((r) => shelves.some((sh) => onShelf(r, sh)));
+  }
+  if (!pool.length) pool = browsable;
+
+  // 2. Filter hard conflicts (theme), unless that empties the shelf.
+  const themed = theme ? pool.filter((r) => !r.theme || r.theme === theme || r.theme === "mid") : pool;
+  if (themed.length >= Math.min(6, pool.length)) pool = themed;
+
+  // 3. Rank.
+  const qRelevance = words([query.industry, query.audience, query.needs, query.words].join(" ")).filter((w) => !STOP.has(w) && w.length > 2);
+  const qTone = words(query.tone).filter((w) => w.length > 2);
+  const bm25 = bm25Index(pool);
+  const maxViews = Math.max(1, ...pool.map((r) => r.views ?? 0));
+  const scored = pool.map((r, i) => {
+    const relevance = bm25(qRelevance, i);
+    const toneHits = qTone.filter((w) => words([r.style, r.colorTags, r.description]).includes(w)).length;
+    const tone = qTone.length ? toneHits / qTone.length : 0;
+    let compat = 0.5;
+    if (theme && r.theme) compat = r.theme === theme ? 1 : 0.15;
+    if (profile.typeClass && r.typeClasses?.length) {
+      compat += r.typeClasses.includes(profile.typeClass) ? 0.5 : r.typeClasses.some((c) => TYPE_NEAR[profile.typeClass]?.includes(c)) ? 0.25 : 0;
+    }
+    const noteBoost = Math.min(0.3, (notes[r.id]?.length ?? 0) * 0.1);
+    const quality = Math.log1p(r.views ?? 0) / Math.log1p(maxViews) + (r.description?.length > 60 ? 0.2 : 0) + (r.curated ? 0.6 : 0) + noteBoost;
+    return { r, relevance, tone, compat, quality };
+  });
+  const maxRel = Math.max(1e-9, ...scored.map((s) => s.relevance));
+  for (const s of scored) s.score = 0.4 * (s.relevance / maxRel) + 0.25 * (s.compat / 1.5) + 0.2 * s.tone + 0.15 * Math.min(1, s.quality);
+  scored.sort((a, b) => b.score - a.score);
+
+  // 4. Diversify: maximal marginal relevance over tags and description words.
+  const feats = (r) => [...r.style, ...r.colorTags, ...r.interaction, ...(r.typeClasses ?? []), r.theme ?? "", ...words(r.description).filter((w) => w.length > 4)];
+  const picked = [];
+  const candidates = scored.slice(0, 40);
+  while (picked.length < count && candidates.length) {
+    let best = 0;
+    let bestVal = -Infinity;
+    candidates.forEach((c, i) => {
+      const sim = picked.length ? Math.max(...picked.map((p) => jaccard(feats(p.r), feats(c.r)))) : 0;
+      const val = 0.7 * c.score - 0.3 * sim;
+      if (val > bestVal) {
+        bestVal = val;
+        best = i;
+      }
+    });
+    picked.push(candidates.splice(best, 1)[0]);
+  }
+
+  const why = (s) => {
+    const parts = [];
+    if (s.relevance > 0) parts.push(`matches ${qRelevance.filter((w) => words(docText(s.r)).includes(w)).slice(0, 5).join(", ") || "the brief"}`);
+    if (theme && s.r.theme === theme) parts.push(`${theme} base like this project`);
+    if (s.tone > 0) parts.push(`tone: ${qTone.filter((w) => words([s.r.style, s.r.colorTags, s.r.description]).includes(w)).join(", ")}`);
+    if (profile.typeClass && s.r.typeClasses?.includes(profile.typeClass)) parts.push(`same type class (${profile.typeClass})`);
+    if (notes[s.r.id]?.length) parts.push(`${notes[s.r.id].length} note(s) from earlier palate runs`);
+    return parts.join("; ") || "closest fit on this shelf";
+  };
+
+  return { profile: { ...profile, theme }, shelves, poolSize: pool.length, picked: picked.map((s) => ({ ...s, why: why(s) })), runnersUp: scored.filter((s) => !picked.includes(s)).slice(0, 5) };
+}
+
+// Returns a local image for the record: the catalogue's preview, or a live capture of the site.
+export async function fetchImage(record) {
+  if (record.capture && record.site) {
+    const res = await capture(record.site, { full: false, shelf: record.shelf });
+    record.measured = res.measures[1440];
+    return res.files[0];
+  }
+  if (!record.image) return undefined;
+  const ext = /\.(png|jpe?g|webp)/i.exec(record.image)?.[1] ?? "jpg";
+  const file = path.join(CACHE_DIR, "images", `${record.id.replace(/[^a-z0-9]+/gi, "_")}.${ext}`);
+  if (fs.existsSync(file)) return file;
+  const res = await fetch(record.image, { headers: { "user-agent": "palate-catalog/0.1" } });
+  if (!res.ok) return undefined;
+  fs.mkdirSync(path.dirname(file), { recursive: true });
+  fs.writeFileSync(file, Buffer.from(await res.arrayBuffer()));
+  return file;
+}
+
+export async function refsReport(query, opts) {
+  const res = findRefs(query, opts);
+  const out = [`# References for ${query.shelf ?? "any shelf"}`, ""];
+  out.push(`Project: ${res.profile.theme ?? "unknown"} base${res.profile.font ? `, ${res.profile.font} (${res.profile.typeClass ?? "unclassified"})` : ""}. Searched ${res.poolSize} designs on ${res.shelves.join(" + ") || "all shelves"}.`, "");
+  let n = 0;
+  for (const s of res.picked) {
+    n++;
+    const img = await fetchImage(s.r).catch(() => undefined);
+    out.push(`## Direction ${n}: ${s.r.name}`);
+    out.push(`- id: ${s.r.id} · live site: ${s.r.site ?? "n/a"} · source: ${s.r.url}`);
+    out.push(`- what it is: ${s.r.description}`);
+    out.push(`- style: ${[...s.r.style, ...s.r.colorTags].join(", ") || "n/a"} · interaction: ${s.r.interaction.join(", ") || "n/a"} · fonts: ${s.r.fonts.join(", ") || "n/a"} · ${s.r.theme ?? "theme unknown"}`);
+    out.push(`- why it was picked: ${s.why}`);
+    if (readNotes()[s.r.id]?.length) out.push(`- notes: ${readNotes()[s.r.id].map((x) => x.text).join(" | ")}`);
+    if (s.r.measured) {
+      const m = s.r.measured;
+      out.push(`- measured live: headline ${m.headline.size}px ${m.headline.family} ${m.headline.weight} (tracking ${m.headline.tracking}) · fonts ${m.fonts.join(", ")} · scale ${m.typeScale.join("/")} · body ${m.bodySize}px · radii ${m.radii.join(", ") || "none"}`);
+    }
+    out.push(`- image: ${img ?? "not available"}`);
+    out.push("");
+  }
+  if (res.runnersUp.length) out.push(`Also close: ${res.runnersUp.map((s) => `${s.r.id} (${s.r.name})`).join(" · ")}`, "");
+  out.push("Open the three images with your image-reading tool. Pick one direction, and write in the Decision Record exactly what you borrow (structure, type treatment, density, signature move) and what you won't copy (their words, logos, assets).");
+  out.push("Afterwards, leave a one-line note on the reference you used: palate refs note <id> \"what worked\"");
+  return out.join("\n");
+}
+
+export function catalogStats() {
+  const all = allRecords();
+  const by = (k) => Object.entries(all.reduce((m, r) => ((m[r[k] ?? "none"] = (m[r[k] ?? "none"] ?? 0) + 1), m), {})).sort((a, b) => b[1] - a[1]);
+  return [
+    `${all.length} designs in the local catalogue`,
+    `Shelves: ${by("shelf").map(([k, v]) => `${k} ${v}`).join(" · ")}`,
+    `Themes: ${by("theme").map(([k, v]) => `${k} ${v}`).join(" · ")}`,
+  ].join("\n");
+}
+
+export { addNote };

package/src/robots.mjs

@@ -0,0 +1,145 @@
+// Polite looking for `palate research`: robots.txt (RFC 9309) for every page palate captures, one request at a
+// time per host with a few seconds between them, and only public http(s) pages. palate looks at a page the way a
+// designer browsing would, on the user's machine and only when they ask for a design; anything a site asks bots
+// not to read is skipped, never worked around.
+import { setTimeout as sleep } from "node:timers/promises";
+
+// How palate names itself in robots.txt groups and in the browser's user agent.
+export const AGENT = "palate-research";
+export const UA_SUFFIX = `${AGENT}/0.1 (design research a person asked for; one page at a time; respects robots.txt)`;
+export const HOST_GAP_MS = 3000; // between two requests to one host
+export const MAX_CRAWL_DELAY_S = 30; // a site asking for more than this is skipped rather than kept waiting
+
+// ---- parsing (RFC 9309: groups by user-agent, longest match wins, Allow wins a tie, * and $ in paths)
+export function parseRobots(text) {
+  const groups = [];
+  let current = null;
+  let lastWasAgent = false;
+  for (const raw of String(text ?? "").split(/\r?\n/)) {
+    const line = raw.replace(/#.*$/, "").trim();
+    const m = line.match(/^([A-Za-z-]+)\s*:\s*(.*)$/);
+    if (!m) continue;
+    const key = m[1].toLowerCase();
+    const value = m[2].trim();
+    if (key === "user-agent") {
+      if (!current || !lastWasAgent) groups.push((current = { agents: [], rules: [], delay: null }));
+      current.agents.push(value.toLowerCase());
+      lastWasAgent = true;
+      continue;
+    }
+    lastWasAgent = false;
+    if (!current) continue;
+    if (key === "allow" || key === "disallow") {
+      if (key === "disallow" && !value) continue; // "Disallow:" with nothing allows everything
+      current.rules.push({ allow: key === "allow", path: value });
+    } else if (key === "crawl-delay") {
+      const n = Number(value);
+      if (Number.isFinite(n)) current.delay = n;
+    }
+  }
+  return groups;
+}
+
+// The group that applies to palate: the most specific user-agent that names it, else "*".
+export function groupFor(groups, agents = [AGENT, "palate"]) {
+  const names = agents.map((a) => a.toLowerCase());
+  const own = groups.filter((g) => g.agents.some((a) => a !== "*" && names.some((n) => n.startsWith(a) || a.startsWith(n))));
+  if (own.length) return { rules: own.flatMap((g) => g.rules), delay: own.find((g) => g.delay != null)?.delay ?? null };
+  const star = groups.filter((g) => g.agents.includes("*"));
+  return { rules: star.flatMap((g) => g.rules), delay: star.find((g) => g.delay != null)?.delay ?? null };
+}
+
+const patternRe = (p) => new RegExp("^" + p.replace(/[.+?^{}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*").replace(/\\\$$|\$$/, "$"));
+
+// Is this path (with its query) allowed by these rules?
+export function allowedBy(rules, pathAndQuery) {
+  let best = null;
+  for (const r of rules) {
+    if (!r.path) continue;
+    let p = r.path;
+    try {
+      p = decodeURI(p);
+    } catch {}
+    if (!patternRe(p).test(pathAndQuery)) continue;
+    const len = r.path.replace(/[*$]/g, "").length;
+    if (!best || len > best.len || (len === best.len && r.allow && !best.allow)) best = { len, allow: r.allow };
+  }
+  return best ? best.allow : true;
+}
+
+// ---- public pages only
+const PRIVATE = /^(localhost|.*\.local|.*\.internal|.*\.localhost|127\.\d+\.\d+\.\d+|10\.\d+\.\d+\.\d+|192\.168\.\d+\.\d+|172\.(1[6-9]|2\d|3[01])\.\d+\.\d+|169\.254\.\d+\.\d+|0\.0\.0\.0|\[?::1\]?|\[?f[cd][0-9a-f]{2}:.*)$/i;
+export function publicUrl(raw, { allowLocal = false } = {}) {
+  let u;
+  try {
+    u = new URL(/^[a-z]+:\/\//i.test(raw) ? raw : `https://${raw}`);
+  } catch {
+    return { ok: false, why: "not a URL" };
+  }
+  if (!/^https?:$/.test(u.protocol)) return { ok: false, why: "only http(s) pages" };
+  if (u.username || u.password) return { ok: false, why: "URLs with credentials are never opened" };
+  if (!allowLocal && PRIVATE.test(u.hostname)) return { ok: false, why: "not a public page" };
+  u.hash = "";
+  return { ok: true, url: u.toString(), host: u.host, origin: u.origin };
+}
+
+// ---- fetching robots.txt, once per origin per run
+// 2xx: parse. 4xx: no rules (allowed). 5xx, a timeout or no answer: the site is treated as fully disallowed.
+export async function robotsFor(origin, { cache = new Map(), timeout = 8000, fetchImpl = fetch } = {}) {
+  if (cache.has(origin)) return cache.get(origin);
+  const p = (async () => {
+    try {
+      const res = await fetchImpl(`${origin}/robots.txt`, { redirect: "follow", signal: AbortSignal.timeout(timeout), headers: { "user-agent": UA_SUFFIX } });
+      if (res.status >= 200 && res.status < 300) {
+        const text = (await res.text()).slice(0, 500_000);
+        return { status: res.status, ...groupFor(parseRobots(text)) };
+      }
+      if (res.status >= 400 && res.status < 500) return { status: res.status, rules: [], delay: null };
+      return { status: res.status, unreachable: true, rules: [{ allow: false, path: "/" }], delay: null };
+    } catch (e) {
+      return { status: 0, unreachable: true, error: String(e?.message ?? e).slice(0, 120), rules: [{ allow: false, path: "/" }], delay: null };
+    }
+  })();
+  cache.set(origin, p);
+  return p;
+}
+
+// May palate look at this URL? { ok, why, delay } — delay in ms between requests to its host.
+export async function mayLook(raw, opts = {}) {
+  const pub = publicUrl(raw, opts);
+  if (!pub.ok) return { ok: false, why: pub.why };
+  const robots = await robotsFor(pub.origin, opts);
+  const u = new URL(pub.url);
+  if (robots.unreachable) return { ok: false, url: pub.url, host: pub.host, why: `robots.txt could not be read (${robots.status || robots.error}); skipped, as RFC 9309 asks` };
+  if (!allowedBy(robots.rules, u.pathname + u.search)) return { ok: false, url: pub.url, host: pub.host, why: "robots.txt disallows this page" };
+  if (robots.delay != null && robots.delay > MAX_CRAWL_DELAY_S) return { ok: false, url: pub.url, host: pub.host, why: `robots.txt asks for ${robots.delay}s between requests; skipped` };
+  return { ok: true, url: pub.url, host: pub.host, delay: Math.max(HOST_GAP_MS, (robots.delay ?? 0) * 1000) };
+}
+
+// Runs jobs in parallel across hosts and one at a time per host, at least `gap` ms apart on one host.
+// jobs: [{ host, run: async () => result, gap? }] → results in the same order.
+export async function politeRun(jobs, { concurrency = 3, gap = HOST_GAP_MS } = {}) {
+  const byHost = new Map();
+  jobs.forEach((j, i) => byHost.set(j.host, [...(byHost.get(j.host) ?? []), { ...j, i }]));
+  const queues = [...byHost.values()];
+  const out = new Array(jobs.length);
+  let next = 0;
+  const worker = async () => {
+    while (next < queues.length) {
+      const q = queues[next++];
+      for (let k = 0; k < q.length; k++) {
+        if (k) await sleep(q[k].gap ?? gap);
+        try {
+          out[q[k].i] = await q[k].run();
+        } catch (e) {
+          out[q[k].i] = { error: String(e?.message ?? e) };
+        }
+      }
+    }
+  };
+  await Promise.all(Array.from({ length: Math.min(concurrency, queues.length) }, worker));
+  return out;
+}
+
+// A page that answers with a bot wall or a CAPTCHA: palate skips it and never tries to get past it.
+export const BOT_WALL = /captcha|recaptcha|hcaptcha|turnstile|challenges\.cloudflare\.com|cf-chl|just a moment\.\.\.|verify (that )?you are (a )?human|are you a robot|access denied|attention required|request blocked|bot detection|unusual traffic|px-captcha|datadome/i;

package/src/sections.mjs

@@ -0,0 +1,210 @@
+// The section archetypes library: palate's own vetted layouts for landing-page sections, one Markdown file each in
+// kb/sections/*.md (YAML frontmatter + body), written by us (never copied from a paid kit). A planner picks one
+// archetype per section (no two adjacent alike), the builder follows its recipe, and an independent judge checks the
+// built section against the archetype's "Floor checks". kb/sections/_finish.md is the shared finishing standard
+// (shadows, buttons, hairlines, radius nesting, textures, image treatment, type details, phone rules); files that
+// start with "_" are not sections.
+//   loadSections()                    every archetype, frontmatter + body + sections
+//   matchSections({ kind, tags, exclude, avoidShapes, imagery, limit })
+//                                     ranked by fits overlap minus avoid overlap
+//   sectionBrief(id)                  one archetype as compact Markdown for a builder
+//   floorChecks(id)                   the judge's yes/no questions
+//   finishStandard()                  the contents of _finish.md
+import fs from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import YAML from "yaml";
+import { TAGS, loadMoves, sectionsOf } from "./moves.mjs";
+import { KB_ROOT } from "./paths.mjs";
+
+const KB_DIR = process.env.PALATE_KB ?? KB_ROOT;
+export const SECTIONS_DIR = path.join(KB_DIR, "sections");
+export const FINISH_FILE = path.join(SECTIONS_DIR, "_finish.md");
+
+export const KINDS = ["hero", "demo", "features", "story", "gallery", "stats", "process", "pricing", "proof", "closing", "faq", "footer", "comparison", "about", "signup", "docs"];
+// The structural key the planner compares to keep neighbouring sections from looking alike.
+export const SHAPES = ["centred-stack", "split", "full-bleed-media", "object-first", "editorial-columns", "bento", "cards-row", "list-ruled", "stacked-rows", "table", "marquee", "single-statement", "timeline", "masonry", "form-led"];
+// What the section is built around. none, type and texture can always be made in code; product-ui, photo and
+// illustration need assets; any means it works with whichever of those the project has (but needs at least one).
+export const IMAGERY = ["none", "product-ui", "photo", "illustration", "texture", "type", "any"];
+export const ALWAYS_AVAILABLE = ["none", "type", "texture"];
+export const ASSET_IMAGERY = ["product-ui", "photo", "illustration"];
+export const DENSITIES = ["airy", "balanced", "dense"];
+export const EFFORTS = ["s", "m", "l"];
+export const REQUIRED = ["id", "name", "kind", "shape", "layout", "fits", "imagery", "density", "effort"];
+export const LISTS = ["fits", "avoid", "needs"]; // must be present as keys (avoid and needs may be empty)
+export const HEADINGS = ["When it's right", "Structure", "Finish", "Content", "States and motion", "Pitfalls", "Floor checks"];
+export const MIN_FLOOR_CHECKS = 5;
+
+const list = (v) => (Array.isArray(v) ? v : v == null || v === "" ? [] : [v]).map((x) => String(x).trim()).filter(Boolean);
+const csv = (v) => list(v).flatMap((x) => x.split(",")).map((x) => x.trim()).filter(Boolean);
+
+// The questions under "## Floor checks": one per list item ("- …", "* …" or "1. …"); continuation lines are joined.
+export function checksOf(text = "") {
+  const out = [];
+  for (const line of text.split("\n")) {
+    const item = line.match(/^\s*(?:[-*]|\d+[.)])\s+(.+?)\s*$/);
+    if (item) out.push(item[1]);
+    else if (out.length && line.trim()) out[out.length - 1] += ` ${line.trim()}`;
+  }
+  return out;
+}
+
+export function parseSection(src, file = "") {
+  const m = src.replace(/\r\n/g, "\n").match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/);
+  if (!m) throw new Error(`${path.basename(file)}: no frontmatter`);
+  const meta = YAML.parse(m[1]) ?? {};
+  const body = m[2].trim();
+  const sections = sectionsOf(body);
+  return {
+    ...meta,
+    fits: list(meta.fits),
+    avoid: list(meta.avoid),
+    needs: list(meta.needs),
+    moves: list(meta.moves),
+    present: Object.keys(meta),
+    body,
+    sections,
+    checks: checksOf(sections["Floor checks"]),
+    file,
+  };
+}
+
+let cache = null;
+export function loadSections({ fresh = false } = {}) {
+  if (cache && !fresh) return cache;
+  if (!fs.existsSync(SECTIONS_DIR)) return (cache = []);
+  cache = fs
+    .readdirSync(SECTIONS_DIR)
+    .filter((f) => f.endsWith(".md") && !f.startsWith("_"))
+    .sort()
+    .map((f) => {
+      const file = path.join(SECTIONS_DIR, f);
+      return parseSection(fs.readFileSync(file, "utf8"), file);
+    });
+  return cache;
+}
+
+export function getSection(id) {
+  return loadSections().find((s) => s.id === id) ?? null;
+}
+
+let movesKnown = null;
+const knownMoves = () => (movesKnown ??= new Set(loadMoves().map((m) => m.id)));
+
+// Problems with one archetype (missing fields, unknown words, missing headings, too few floor checks, no wireframe);
+// [] when it is sound. `moves` (optional) must name real moves in kb/moves.
+export function lintSection(s, { moves = knownMoves() } = {}) {
+  const p = [];
+  for (const k of REQUIRED) if (s[k] == null || s[k] === "") p.push(`missing ${k}`);
+  for (const k of LISTS) if (!(s.present ?? []).includes(k)) p.push(`missing ${k}`);
+  if (s.id && !/^[a-z0-9]+(-[a-z0-9]+)*$/.test(s.id)) p.push(`id ${s.id} is not kebab-case`);
+  if (s.file && s.id && path.basename(s.file, ".md") !== s.id) p.push(`file name should be ${s.id}.md`);
+  if (s.kind && !KINDS.includes(s.kind)) p.push(`unknown kind ${s.kind}`);
+  if (s.shape && !SHAPES.includes(s.shape)) p.push(`unknown shape ${s.shape}`);
+  if (s.imagery && !IMAGERY.includes(s.imagery)) p.push(`unknown imagery ${s.imagery}`);
+  if (s.density && !DENSITIES.includes(s.density)) p.push(`density must be airy, balanced or dense`);
+  if (s.effort && !EFFORTS.includes(s.effort)) p.push(`effort must be s, m or l`);
+  if (!s.fits?.length) p.push("fits is empty");
+  for (const t of [...(s.fits ?? []), ...(s.avoid ?? [])]) if (!TAGS.includes(t)) p.push(`unknown tag ${t}`);
+  for (const t of s.fits ?? []) if ((s.avoid ?? []).includes(t)) p.push(`${t} is in both fits and avoid`);
+  for (const id of s.moves ?? []) if (!moves.has(id)) p.push(`move ${id} does not exist`);
+  for (const h of HEADINGS) if (!s.sections?.[h]) p.push(`missing section "${h}"`);
+  if (s.sections?.Structure && !/```[\s\S]*?\n[\s\S]*?```/.test(s.sections.Structure)) p.push("Structure has no ASCII wireframe in a ``` block");
+  const checks = s.checks ?? [];
+  if (checks.length < MIN_FLOOR_CHECKS) p.push(`only ${checks.length} floor checks (need ${MIN_FLOOR_CHECKS}+)`);
+  for (const q of checks) if (!q.trim().endsWith("?")) p.push(`floor check is not a question: "${q.slice(0, 60)}"`);
+  return p;
+}
+
+// A query tag matches a section's tag exactly, or as a family ("playful" ~ "playful-kids") at half weight.
+const near = (q, t) => q !== t && (t.startsWith(`${q}-`) || q.startsWith(`${t}-`));
+const hits = (query, tags) => {
+  const exact = query.filter((q) => tags.includes(q));
+  const partial = query.filter((q) => !exact.includes(q) && tags.some((t) => near(q, t)));
+  return { exact, partial };
+};
+
+/**
+ * Ranks archetypes for one section of a plan. Score = fits overlap (1 per exact tag, 0.5 per family match) minus
+ * avoid overlap (1.5 per exact tag, 0.75 per family match), plus a little for a focused archetype (its fits mostly
+ * matched). Filters:
+ *   kind         one kind or several ("hero", "story,gallery")
+ *   exclude      archetype ids to leave out (used elsewhere on the page)
+ *   avoidShapes  shapes to leave out (the neighbouring sections' shapes, so no two adjacent sections look alike)
+ *   imagery      what the project can supply ("photo", ["product-ui", "photo"]); archetypes built around imagery it
+ *                lacks are dropped. none, type and texture are always available; any needs at least one of product-ui,
+ *              photo or illustration. Omit to keep everything.
+ * With tags, archetypes that score below zero are dropped unless nothing else is left. limit 0 = all.
+ */
+export function matchSections({ kind, tags = [], exclude = [], avoidShapes = [], limit = 3, imagery } = {}) {
+  const q = [...new Set(csv(tags).map((t) => t.toLowerCase()))];
+  const kinds = csv(kind);
+  const skip = new Set(csv(exclude));
+  const noShape = new Set(csv(avoidShapes));
+  const have = imagery == null ? null : new Set([...ALWAYS_AVAILABLE, ...csv(imagery)]);
+  if (have && ASSET_IMAGERY.some((k) => have.has(k))) have.add("any");
+  let rows = loadSections()
+    .filter((s) => !skip.has(s.id))
+    .filter((s) => !kinds.length || kinds.includes(s.kind))
+    .filter((s) => !noShape.has(s.shape))
+    .filter((s) => !have || have.has(s.imagery))
+    .map((s) => {
+      const f = hits(q, s.fits);
+      const a = hits(q, s.avoid);
+      const fit = f.exact.length + 0.5 * f.partial.length;
+      const focus = s.fits.length ? fit / s.fits.length : 0;
+      const score = q.length ? Math.round((fit - 1.5 * a.exact.length - 0.75 * a.partial.length + 0.25 * focus) * 100) / 100 : 0;
+      const parts = [];
+      if (fit) parts.push(`fits ${[...f.exact, ...f.partial.map((t) => `~${t}`)].join(", ")}`);
+      if (a.exact.length || a.partial.length) parts.push(`but avoid ${[...a.exact, ...a.partial.map((t) => `~${t}`)].join(", ")}`);
+      parts.push(`${s.kind}, ${s.shape}, imagery ${s.imagery}, effort ${s.effort}`);
+      return { id: s.id, name: s.name, kind: s.kind, shape: s.shape, layout: s.layout, imagery: s.imagery, density: s.density, effort: s.effort, score, reason: parts.join("; "), section: s };
+    });
+  if (q.length) {
+    const ok = rows.filter((r) => r.score >= 0);
+    if (ok.length) rows = ok;
+  }
+  rows.sort((a, b) => b.score - a.score || a.id.localeCompare(b.id));
+  const n = Math.max(0, Number(limit) || 0);
+  return n ? rows.slice(0, n) : rows;
+}
+
+/** The judge's questions for one archetype ([] for an unknown id). YES = good. */
+export function floorChecks(id) {
+  return getSection(id)?.checks ?? [];
+}
+
+/** One archetype (or several, comma-separated or an array) as compact Markdown for a builder. */
+export function sectionBrief(ids) {
+  const out = [];
+  for (const id of csv(ids)) {
+    const s = getSection(id);
+    if (!s) {
+      out.push(`### ${id}\n(no such section archetype)`);
+      continue;
+    }
+    out.push(
+      [
+        `### ${s.name} (${s.id}) · ${s.kind}, ${s.shape}`,
+        `- layout: ${s.layout}`,
+        `- imagery: ${s.imagery} · density: ${s.density} · effort: ${s.effort}`,
+        s.needs.length ? `- needs: ${s.needs.join("; ")}` : null,
+        `- fits: ${s.fits.join(", ")}${s.avoid.length ? ` · avoid: ${s.avoid.join(", ")}` : ""}`,
+        s.moves.length ? `- pairs with moves: ${s.moves.join(", ")}` : null,
+        `- shared recipes (shadows, button craft, hairlines, textures, type, phone rules): kb/sections/_finish.md; the page's button language: kb/sections/_buttons.md`,
+        "",
+        s.body,
+      ]
+        .filter((l) => l != null)
+        .join("\n"),
+    );
+  }
+  return out.join("\n\n");
+}
+
+/** The shared finishing standard (kb/sections/_finish.md) without its frontmatter; "" when it is missing. */
+export function finishStandard() {
+  if (!fs.existsSync(FINISH_FILE)) return "";
+  return fs.readFileSync(FINISH_FILE, "utf8").replace(/\r\n/g, "\n").replace(/^---\n[\s\S]*?\n---\n?/, "").trim();
+}