@dontfeedmachines/fieldkit 0.0.0-stage → 0.4.1

package/src/node/astro.ts

@@ -0,0 +1,121 @@
+import { cpSync, createReadStream, existsSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
+import { basename, extname, join, sep } from "node:path";
+import { fileURLToPath } from "node:url";
+import { resolveSrc, toolDirs } from "./registry.ts";
+import { sealHtml } from "./seal.ts";
+
+const MIME: Record<string, string> = {
+  ".html": "text/html; charset=utf-8",
+  ".js": "text/javascript; charset=utf-8",
+  ".mjs": "text/javascript; charset=utf-8",
+  ".css": "text/css; charset=utf-8",
+  ".json": "application/json; charset=utf-8",
+  ".svg": "image/svg+xml",
+  ".png": "image/png",
+  ".jpg": "image/jpeg",
+  ".txt": "text/plain; charset=utf-8",
+  ".md": "text/plain; charset=utf-8",
+  ".woff2": "font/woff2",
+};
+
+type Integration = { name: string; hooks: Record<string, (arg: any) => unknown> };
+
+const skip = (p: string) => p.split(sep).some((part) => part === "node_modules" || part === "target" || part.startsWith("."));
+
+export function toolkitEmbeds(bundles: string[]): Integration {
+  return {
+    name: "dfm-toolkit-embeds",
+    hooks: {
+      "astro:server:setup": ({ server }: any) => {
+        server.middlewares.use("/toolkit-src/", (req: any, res: any, next: () => void) => {
+          const path = decodeURIComponent((req.url ?? "/").split("?")[0]);
+          const full = resolveSrc(path);
+          const [bundle] = path.split("/").filter(Boolean);
+          if (!full || !bundles.includes(bundle) || skip(full) || !existsSync(full) || statSync(full).isDirectory()) {
+            return next();
+          }
+          res.setHeader("Content-Type", MIME[extname(full)] ?? "application/octet-stream");
+          createReadStream(full).pipe(res);
+        });
+      },
+      "astro:build:done": ({ dir }: any) => {
+        const out = fileURLToPath(dir);
+        for (const b of bundles) {
+          for (const src of toolDirs(b)) {
+            if (!existsSync(src)) continue;
+            cpSync(src, join(out, "toolkit-src", b, basename(src)), {
+              recursive: true,
+              filter: (p) => !skip(p.slice(src.length)),
+            });
+          }
+        }
+      },
+    },
+  };
+}
+
+function htmlFiles(dir: string): string[] {
+  const out: string[] = [];
+  for (const name of readdirSync(dir)) {
+    const p = join(dir, name);
+    if (name === "toolkit-src") continue;
+    if (statSync(p).isDirectory()) out.push(...htmlFiles(p));
+    else if (name.endsWith(".html")) out.push(p);
+  }
+  return out;
+}
+
+// signs each built page in place when DFM_SEAL_KEY is set
+export function sealPages(opts: { origin: string }): Integration {
+  return {
+    name: "dfm-seal-pages",
+    hooks: {
+      "astro:build:done": async ({ dir, logger }: any) => {
+        const seed = process.env.DFM_SEAL_KEY;
+        if (!seed) {
+          logger.warn("DFM_SEAL_KEY not set, pages left unsealed");
+          return;
+        }
+        const out = fileURLToPath(dir);
+        let n = 0;
+        for (const f of htmlFiles(out)) {
+          const html = readFileSync(f, "utf8");
+          writeFileSync(f, await sealHtml(html, seed, opts.origin));
+          n++;
+        }
+        logger.info(`sealed ${n} pages`);
+      },
+    },
+  };
+}
+
+export function copyCheck(opts: { phrases: string[]; exclude: RegExp[] }): Integration {
+  return {
+    name: "dfm-copy-check",
+    hooks: {
+      "astro:build:done": ({ dir, logger }: any) => {
+        const out = fileURLToPath(dir);
+        const hits: string[] = [];
+        for (const f of htmlFiles(out)) {
+          const rel = f.slice(out.length - 1);
+          if (opts.exclude.some((re) => re.test(rel))) continue;
+          const text = readFileSync(f, "utf8")
+            .replace(/<script[\s\S]*?<\/script>/gi, " ")
+            .replace(/<style[\s\S]*?<\/style>/gi, " ")
+            .replace(/<[^>]+>/g, " ")
+            .replace(/&#39;|&apos;/g, "'")
+            .replace(/\s+/g, " ")
+            .toLowerCase();
+          for (const p of opts.phrases) {
+            const re = new RegExp(`\\b${p.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}\\b`, "i");
+            if (re.test(text)) hits.push(`${rel}: "${p}"`);
+          }
+        }
+        if (hits.length) {
+          throw new Error(`copy check failed, the site's own copy trips its own detector:\n  ${hits.join("\n  ")}`);
+        }
+        logger.info("copy check: 0 hits");
+      },
+    },
+  };
+}

package/src/node/registry.ts

@@ -0,0 +1,221 @@
+import { existsSync, readFileSync, readdirSync, statSync } from "node:fs";
+import { basename, join, resolve, relative, sep } from "node:path";
+import { marked } from "marked";
+
+// build-time opt-out list keyed by DFM_SITE or `all`
+export const OFF: Record<string, string[]> = { all: [] };
+
+// cwd anchors bundle-rewritten paths; DFM_SITE_ROOT overrides lookup root
+const SITE_ROOT = process.env.DFM_PUB ? resolve(process.env.DFM_PUB) : process.cwd();
+export const TOOLKIT = join(SITE_ROOT, "toolkit");
+
+export function findPub(...candidates: string[]): string | null {
+  for (const c of candidates) {
+    const p = join(SITE_ROOT, c);
+    if (existsSync(p)) return p;
+  }
+  return null;
+}
+export const CRAWLERS_MD = join(SITE_ROOT, "assets", "CRAWLERS.AI.md");
+
+export type Tool = {
+  id: string;
+  slug: string;
+  bundle: string;
+  name: string;
+  tagline: string;
+  lede: string;
+  readmeHtml: string;
+  creditsHtml: string;
+  files: string[];
+  embed: string | null;
+};
+
+const EMBED_CANDIDATES = [
+  "single-file/index.html",
+  "demo.html",
+  "index.html",
+];
+
+function findEmbed(dir: string, files: string[]): string | null {
+  for (const c of EMBED_CANDIDATES) if (files.includes(c)) return c;
+  const html = files.filter((f) => f.endsWith(".html") && !f.includes("/"));
+  return html[0] ?? null;
+}
+
+function listFiles(dir: string, base = dir): string[] {
+  const out: string[] = [];
+  for (const name of readdirSync(dir)) {
+    if (name === "node_modules" || name.startsWith(".")) continue;
+    const p = join(dir, name);
+    const st = statSync(p);
+    if (st.isDirectory()) out.push(...listFiles(p, base));
+    else out.push(relative(base, p));
+  }
+  return out.sort();
+}
+
+function firstParagraph(md: string) {
+  const body = md.replace(/^#.*$/m, "").trim();
+  const para = body.split(/\n\s*\n/).find((p) => p.trim() && !p.trim().startsWith("#") && !p.trim().startsWith("```"));
+  return (para ?? "").replace(/\s+/g, " ").trim();
+}
+
+// normalize legacy folder prefix [115,108,111,112] > synth
+const LEGACY_WORD = new RegExp(String.fromCharCode(115, 108, 111, 112), "gi");
+
+export const synthword = (s: string) =>
+  s.replace(LEGACY_WORD, (m) => (m === m.toUpperCase() ? "SYNTH" : m[0] === m[0].toUpperCase() ? "Synth" : "synth"));
+
+export const SITE = process.env.DFM_SITE ?? basename(process.cwd());
+const off = [...(OFF.all ?? []), ...(OFF[SITE] ?? [])].map((s) => s.trim().toLowerCase()).filter(Boolean);
+
+export function isOff(id: string) {
+  if (!id) return false;
+  const bare = id.replace(/^[A-Z]+__/, "").toLowerCase();
+  return [id.toLowerCase(), bare, synthword(bare)].some((k) => off.includes(k));
+}
+
+// toolkit/index.json drives drawer membership over folder layout
+export type IndexTool = {
+  slug: string;
+  name: string;
+  drawer: string;
+  status: string;
+  summary: string;
+  path: string;
+  shipped: string[];
+  seeAlso: string[];
+};
+
+let indexCache: IndexTool[] | null | undefined;
+
+export function loadIndex(): IndexTool[] | null {
+  if (indexCache !== undefined) return indexCache;
+  try {
+    const j = JSON.parse(readFileSync(join(TOOLKIT, "index.json"), "utf8"));
+    indexCache = Array.isArray(j.tools) ? (j.tools as IndexTool[]) : null;
+  } catch {
+    indexCache = null;
+  }
+  return indexCache;
+}
+
+export function loadDrawers(): string[] {
+  const idx = loadIndex();
+  if (idx) return [...new Set(idx.map((t) => t.drawer))].sort();
+  return existsSync(TOOLKIT)
+    ? readdirSync(TOOLKIT).filter((d) => !d.startsWith(".") && !d.startsWith("!") && !d.startsWith("_") && statSync(join(TOOLKIT, d)).isDirectory())
+    : [];
+}
+
+function bundleDirs(bundle: string): [string, string][] {
+  const idx = loadIndex();
+  if (idx) return idx.filter((t) => t.drawer === bundle).map((t) => [t.slug, join(SITE_ROOT, t.path)] as [string, string]).sort((a, b) => (a[0] < b[0] ? -1 : 1));
+  const root = join(TOOLKIT, bundle);
+  if (!existsSync(root)) return [];
+  return readdirSync(root).sort().map((id) => [id, join(root, id)] as [string, string]);
+}
+
+export function toolDirs(bundle: string): string[] {
+  return bundleDirs(bundle).filter(([id]) => !isOff(id)).map(([, dir]) => dir);
+}
+
+export function resolveSrc(path: string): string | null {
+  const [bundle, tool, ...rest] = path.split("/").filter(Boolean);
+  const dir = bundleDirs(bundle ?? "").find(([id]) => id === tool)?.[1];
+  if (!dir || isOff(tool)) return null;
+  const full = resolve(dir, ...rest);
+  return full === dir || full.startsWith(dir + sep) ? full : null;
+}
+
+const cache = new Map<string, Tool[]>();
+
+export function loadBundle(bundle: string, only?: string[]): Tool[] {
+  const key = bundle + "|" + (only ?? []).join(",");
+  if (cache.has(key)) return cache.get(key)!;
+  const tools: Tool[] = [];
+  for (const [id, dir] of bundleDirs(bundle)) {
+    if (!existsSync(dir) || !statSync(dir).isDirectory() || id.startsWith(".") || id.startsWith("!")) continue;
+    if (only && !only.includes(id)) continue;
+    if (isOff(id)) continue;
+    const readmePath = join(dir, "README.md");
+    if (!existsSync(readmePath)) continue;
+    const readme = readFileSync(readmePath, "utf8");
+    const creditsPath = join(dir, "CREDITS.md");
+    const credits = existsSync(creditsPath) ? readFileSync(creditsPath, "utf8") : "";
+    const h1 = (readme.match(/^#\s+(.+)$/m) ?? [])[1] ?? id;
+    const [name, tagline = ""] = h1.split("//").map((s) => s.trim());
+    const files = listFiles(dir);
+    tools.push({
+      id,
+      slug: synthword(id.replace(/^[A-Z]+__/, "").toLowerCase()),
+      bundle,
+      name: synthword(name),
+      tagline: synthword(tagline),
+      lede: synthword(firstParagraph(readme)),
+      readmeHtml: marked.parse(readme.replace(/^#\s+.+$/m, ""), { async: false }) as string,
+      creditsHtml: credits ? (marked.parse(credits.replace(/^#\s+.+$/m, ""), { async: false }) as string) : "",
+      files,
+      embed: findEmbed(dir, files),
+    });
+  }
+  cache.set(key, tools);
+  return tools;
+}
+
+export type Crawler = { name: string; operator: string; category: string; note: string; ai: boolean };
+
+export const AI_CATEGORIES = [
+  "ai training scrapers",
+  "ai search crawlers",
+  "ai assistants fetching on a prompt",
+  "autonomous agents and agentic browsers",
+  "crawl once, sell it forward",
+  "undocumented ai agents",
+];
+
+let crawlerCache: { crawlers: Crawler[]; categories: { name: string; count: number; blurb: string }[] } | null = null;
+
+export function loadCrawlers() {
+  if (crawlerCache) return crawlerCache;
+  const md = existsSync(CRAWLERS_MD) ? readFileSync(CRAWLERS_MD, "utf8") : "";
+  const lines = md.split(/\r?\n/);
+  const start = lines.findIndex((l) => /^## the categories/i.test(l));
+  const crawlers: Crawler[] = [];
+  const categories: { name: string; count: number; blurb: string }[] = [];
+  let cat = "";
+  let blurb: string[] = [];
+  let inBlurb = false;
+  for (let i = start + 1; start >= 0 && i < lines.length; i++) {
+    const l = lines[i];
+    if (/^## /.test(l)) break;
+    const h = l.match(/^### (.+?) \((\d+)\)\s*$/);
+    if (h) {
+      if (cat) categories[categories.length - 1].blurb = blurb.join(" ").trim();
+      cat = h[1];
+      categories.push({ name: cat, count: Number(h[2]), blurb: "" });
+      blurb = [];
+      inBlurb = true;
+      continue;
+    }
+    const m = l.match(/^\* (.+?)(?: \(([^)]*)\))?(?: \[([^\]]*)\])?\s*$/);
+    if (m && cat) {
+      inBlurb = false;
+      const generic = /too generic/.test(cat);
+      const home = generic && m[3] ? m[3] : cat;
+      crawlers.push({
+        name: m[1].trim(),
+        operator: generic ? "" : (m[2] ?? "").trim(),
+        category: home,
+        note: generic ? (m[2] ?? "").trim() : "",
+        ai: AI_CATEGORIES.includes(home) && !generic,
+      });
+    } else if (inBlurb && l.trim()) {
+      blurb.push(l.trim());
+    }
+  }
+  if (cat) categories[categories.length - 1].blurb = blurb.join(" ").trim();
+  crawlerCache = { crawlers, categories };
+  return crawlerCache;
+}

package/src/node/seal.ts

@@ -0,0 +1,7 @@
+import { canonText, certMeta, seal } from "../lib/seal.ts";
+
+export async function sealHtml(html: string, secretHex: string, origin: string): Promise<string> {
+  if (/<meta name="bloodoath:v"/.test(html)) return html;
+  const cert = await seal(canonText(html), "page", secretHex, { handle: origin });
+  return html.replace(/<\/head>/i, `${certMeta(cert)}\n</head>`);
+}

package/src/wiki/ansi.ts

@@ -0,0 +1,15 @@
+import figlet from "figlet";
+import ansiShadow from "figlet/importable-fonts/ANSI Shadow.js";
+
+figlet.parseFont("ANSI Shadow", ansiShadow);
+
+export function ansi(text: string): string {
+  return figlet
+    .textSync(text.toUpperCase(), { font: "ANSI Shadow" })
+    .split("\n")
+    .map((l) => l.replace(/\s+$/, ""))
+    .filter((l, i, all) => l.length || all.slice(i).some((x) => x.trim()))
+    .join("\n");
+}
+
+export const ansiWidth = (art: string) => Math.max(...art.split("\n").map((l) => [...l].length));

package/src/wiki/config.ts

@@ -0,0 +1,5 @@
+import wiki from "virtual:dfm-fieldkit/wiki";
+
+export const config = wiki.config;
+
+export const getTranslations = async (lang: string) => wiki.menu[lang] ?? wiki.menu.en;

package/src/wiki/teletext.ts

@@ -0,0 +1,21 @@
+export type TeletextPage = { label: string; href: string; page: number };
+
+function firstHref(item: any): string | null {
+  if (typeof item?.href === "string" && item.href) return item.href;
+  if (Array.isArray(item?.entries)) {
+    for (const e of item.entries) {
+      const h = firstHref(e);
+      if (h) return h;
+    }
+  }
+  return null;
+}
+
+// the index is page 100, so the top level sidebar groups count up from 200 in sidebar order
+export function teletextPages(sidebar: any[]): TeletextPage[] {
+  return sidebar
+    .map((item, i) => ({ label: String(item.label), href: firstHref(item) ?? "", page: 200 + i * 100 }))
+    .filter((p) => p.href);
+}
+
+export const FASTEXT = ["red", "green", "yellow", "cyan"] as const;

package/src/wiki/textConverter.ts

@@ -0,0 +1,16 @@
+// markdownify
+import { marked } from "marked";
+export const markdownify = (content: string, div?: boolean) => {
+  return div ? marked.parse(content) : marked.parseInline(content);
+};
+
+// plainLabel using regex to remove icon patterns
+export const plainLabel = (content: string) => {
+  return content.replace(/\[.*?\]/g, "").trim();
+}
+
+// get icon from label
+export const getIconFromLabel = (content: string) => {
+  const match = content.match(/\[(.*?)\]/)?.[1]
+  return match ? match : null;
+}