@xemahq/biome-sdk 0.1.1

package/src/builder/lib/content-walker.ts

@@ -0,0 +1,383 @@
+// ═══════════════════════════════════════════════════════════════════════════
+// Content walker — resolves a manifest's content directories to absolute
+// file paths. Used by:
+//   - server-side seeders (enumerate + parse files at boot)
+//   - in-biome install functions (when they need to register imperative
+//     content alongside declarative content)
+//
+// All paths returned are absolute. Paths declared in xema-biome.json are
+// resolved relative to the biome package root (the directory containing
+// the manifest). Symbolic links are followed; hidden files (basename
+// starting with `.`) are skipped.
+// ═══════════════════════════════════════════════════════════════════════════
+
+import type { Dirent } from 'node:fs';
+import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+import * as YAML from 'js-yaml';
+
+import {
+  contentKindToDir,
+  isServerBiomeManifest,
+  type BiomeManifest,
+} from '../../host';
+
+/**
+ * Content kinds backed by a directory of YAML/Markdown files. Kept as a
+ * free-form string alias because callers pass directory-token strings
+ * from `xema-biome.json` (the closed enum surface is `ContributionKind`
+ * — this helper bridges to the on-disk directory layout). Unknown tokens
+ * fail fast in `contentKindToDir`.
+ */
+export type ContentKind = string;
+
+/** File extensions per content kind (used to filter walk results). */
+const CONTENT_KIND_EXTENSIONS: Readonly<Record<ContentKind, readonly string[]>> = {
+  agents: ['.md'],
+  workspaceManifests: ['.yaml', '.yml'],
+  workspaceManifestTemplates: ['.hbs', '.yaml', '.yml'],
+  toolProfiles: ['.yaml', '.yml'],
+  mcpCatalog: ['.yaml', '.yml'],
+  mcpTools: ['.yaml', '.yml', '.json'],
+  // OpenCode skills are folders containing SKILL.md — walker returns the
+  // SKILL.md path; callers can dirname() if they need the folder.
+  openCodeSkills: ['.md'],
+  openCodeTools: ['.ts', '.js', '.mjs'],
+  openCodePlugins: ['.ts', '.js', '.mjs'],
+  roleCapabilities: ['.yaml', '.yml'],
+  workflowConfig: ['.yaml', '.yml'],
+  deliverableSpecs: ['.yaml', '.yml', '.md', '.json'],
+  artifactTypes: ['.yaml', '.yml'],
+  biomeInstallSchema: ['.json'],
+  icons: ['.svg'],
+  projectKits: ['.yaml', '.yml'],
+  provisioningScaffolds: ['.yaml', '.yml', '.json'],
+};
+
+/**
+ * Resolve a biome's package root from `import.meta.url` (the biome's
+ * src/index.ts). Walks upward until it finds a `xema-biome.json` —
+ * works for both the source layout (`biomes/<tier>/<id>/src/index.ts`)
+ * and the built layout (`biomes/<tier>/<id>/dist/index.js`).
+ */
+export function resolvePluginRoot(importMetaUrl: string): string {
+  let dir = dirname(fileURLToPath(importMetaUrl));
+  // Walk up to 5 levels max to avoid runaway. Real layout never needs more than 2.
+  for (let i = 0; i < 5; i++) {
+    if (existsSync(join(dir, 'xema-biome.json'))) {
+      return dir;
+    }
+    const parent = dirname(dir);
+    if (parent === dir) break;
+    dir = parent;
+  }
+  throw new Error(
+    `resolvePluginRoot: could not locate xema-biome.json walking up from ${importMetaUrl}`,
+  );
+}
+
+/**
+ * Resolve the absolute directory for a given content kind, or `null` if
+ * the manifest does not ship that kind. Web manifests have no content
+ * directories — callers always pass server manifests; we return `null`
+ * for web manifests instead of throwing so seeders that scan a mixed
+ * tree don't need their own target check.
+ *
+ * Path is conventional: `<biomeRoot>/<contentKindToDir(kind)>/`.
+ */
+export function resolveContentDir(
+  biomeRoot: string,
+  manifest: BiomeManifest,
+  kind: ContentKind,
+): string | null {
+  if (!isServerBiomeManifest(manifest)) return null;
+  const ships = manifest.xema.ships?.content ?? [];
+  if (!ships.includes(kind)) return null;
+  return resolve(biomeRoot, contentKindToDir(kind));
+}
+
+export interface ContentFile {
+  /** Absolute path to the file. */
+  readonly absolutePath: string;
+  /** File basename (e.g. `build.md`). */
+  readonly basename: string;
+  /** Slug derived by stripping the extension. */
+  readonly slug: string;
+}
+
+/**
+ * Walk every biome under the given scan roots for a single content kind.
+ *
+ * Each scan root is a directory containing biome folders (e.g.
+ * `biomes/kernel`, `biomes/platform`). For each subdirectory:
+ *   1. Parse `xema-biome.json` (skip if missing).
+ *   2. Resolve the manifest's `content[kind]` path (skip if not declared).
+ *   3. `walkContent()` that directory for the kind's extensions.
+ *
+ * Returns a flat list, each entry tagged with the originating biome id
+ * so seeders can attribute the row in DB upserts. Order is stable:
+ * scan-roots are visited in the supplied order; within a root, biomes
+ * are visited alphabetically.
+ *
+ * Used by every per-service seeder (mcp-catalog, tool-profile,
+ * agent-session-profile, …) so the FS-discovery path stays DRY.
+ */
+export function enumerateContentAcrossBiomeTree(
+  scanRoots: readonly string[],
+  kind: ContentKind,
+): Array<ContentFile & { biomeId: string; biomeRoot: string }> {
+  const out: Array<ContentFile & { biomeId: string; biomeRoot: string }> = [];
+  for (const root of scanRoots) {
+    if (!existsSync(root)) continue;
+    let entries: Dirent[];
+    try {
+      entries = readdirSync(root, { withFileTypes: true }) as Dirent[];
+    } catch {
+      continue;
+    }
+    const subdirs = entries
+      .filter((e) => e.isDirectory())
+      .map((e) => e.name)
+      .sort();
+    for (const dirName of subdirs) {
+      const biomeRoot = join(root, dirName);
+      const manifestPath = join(biomeRoot, 'xema-biome.json');
+      if (!existsSync(manifestPath)) continue;
+      let raw: unknown;
+      try {
+        raw = JSON.parse(readFileSync(manifestPath, 'utf-8'));
+      } catch {
+        continue;
+      }
+      // We intentionally do NOT call `parseBiomeManifest` here — this
+      // helper is used during seeder boot, where a malformed manifest
+      // should be flagged by biome-host's own validation rather than
+      // crashing every consumer. Callers receive only well-formed entries
+      // because resolveContentDir returns null for missing/invalid kinds
+      // and for non-server manifests.
+      const shape = raw as {
+        xema?: {
+          id?: string;
+          target?: string;
+          ships?: { content?: readonly string[] };
+        };
+      };
+      const biomeId = shape?.xema?.id;
+      if (typeof biomeId !== 'string') continue;
+      if (shape.xema?.target !== 'server') continue;
+      const dir = resolveContentDir(biomeRoot, raw as BiomeManifest, kind);
+      if (!dir) continue;
+      for (const file of walkContent(dir, kind)) {
+        out.push({ ...file, biomeId, biomeRoot });
+      }
+    }
+  }
+  return out;
+}
+
+export interface BiomeContentBinding {
+  /** Biome id from `xema-biome.json` `xema.id`. */
+  readonly biomeId: string;
+  /** Absolute path to the biome package root (containing the manifest). */
+  readonly biomeRoot: string;
+  /** Absolute path to the resolved content directory for the requested kind. */
+  readonly contentDir: string;
+}
+
+/**
+ * Walk every biome under the given scan roots and return a binding for
+ * each biome that declares the requested content kind. Unlike
+ * `enumerateContentAcrossBiomeTree`, this does NOT list files — it
+ * returns the directory roots so callers with sub-directory conventions
+ * (e.g. `<contentDir>/actions/*.yaml` vs `<contentDir>/workflows/*.yaml`)
+ * can do their own walking.
+ *
+ * Order is stable: scan roots in supplied order; biomes alphabetically
+ * within each root. Biomes missing `xema-biome.json` or not declaring
+ * the kind are silently skipped.
+ */
+export function enumerateBiomesWithContent(
+  scanRoots: readonly string[],
+  kind: ContentKind,
+): BiomeContentBinding[] {
+  const out: BiomeContentBinding[] = [];
+  for (const root of scanRoots) {
+    if (!existsSync(root)) continue;
+    let entries: Dirent[];
+    try {
+      entries = readdirSync(root, { withFileTypes: true }) as Dirent[];
+    } catch {
+      continue;
+    }
+    const subdirs = entries
+      .filter((e) => e.isDirectory())
+      .map((e) => e.name)
+      .sort();
+    for (const dirName of subdirs) {
+      const biomeRoot = join(root, dirName);
+      const manifestPath = join(biomeRoot, 'xema-biome.json');
+      if (!existsSync(manifestPath)) continue;
+      let raw: unknown;
+      try {
+        raw = JSON.parse(readFileSync(manifestPath, 'utf-8'));
+      } catch {
+        continue;
+      }
+      const shape = raw as {
+        xema?: { id?: string; target?: string };
+      };
+      const biomeId = shape?.xema?.id;
+      if (typeof biomeId !== 'string') continue;
+      if (shape.xema?.target !== 'server') continue;
+      const contentDir = resolveContentDir(biomeRoot, raw as BiomeManifest, kind);
+      if (!contentDir) continue;
+      if (!existsSync(contentDir)) continue;
+      out.push({ biomeId, biomeRoot, contentDir });
+    }
+  }
+  return out;
+}
+
+/**
+ * Resolve server-biome scan roots. Two-tier lookup — fail-fast on
+ * misconfiguration, no silent fallbacks:
+ *
+ *   1. `XEMA_BIOME_SCAN_ROOTS_SERVER` env (CSV of absolute paths) —
+ *      explicit, required in any non-monorepo environment (prod, CI,
+ *      Docker). Every consuming service MUST set it in its Helm /
+ *      compose config; missing env in those environments is a critical
+ *      misconfiguration that crashes the seeder loudly so it's caught
+ *      at first deploy, not after silent under-seeding.
+ *   2. Monorepo dev fallback — walk up from `cwd` looking for a
+ *      `pnpm-workspace.yaml` next to a `biomes/` tree. Active only
+ *      when running from inside the source repo.
+ *
+ * Throwing on missing config is intentional. We intentionally do NOT
+ * fall back to a container path like `/app/biomes/` — that would
+ * silently mask a missing env var in production, which is precisely
+ * the failure mode this contract exists to prevent.
+ */
+export function resolveDefaultBiomeScanRoots(cwd: string = process.cwd()): string[] {
+  const envOverride = process.env.XEMA_BIOME_SCAN_ROOTS_SERVER;
+  if (envOverride && envOverride.length > 0) {
+    return envOverride
+      .split(',')
+      .map((p) => p.trim())
+      .filter((p) => p.length > 0);
+  }
+
+  // Monorepo dev fallback.
+  let cursor = cwd;
+  for (let i = 0; i < 8; i++) {
+    if (
+      existsSync(join(cursor, 'biomes')) &&
+      existsSync(join(cursor, 'pnpm-workspace.yaml'))
+    ) {
+      return [join(cursor, 'biomes', 'kernel'), join(cursor, 'biomes', 'platform')];
+    }
+    const parent = dirname(cursor);
+    if (parent === cursor) break;
+    cursor = parent;
+  }
+  throw new Error(
+    `resolveDefaultBiomeScanRoots: XEMA_BIOME_SCAN_ROOTS_SERVER is unset and no ` +
+      `monorepo workspace was found walking up from ${cwd}. Every service that ` +
+      `consumes biome content (artifact-store-api, deliverable-specs-api, ` +
+      `workflow-engine-api, llm-registry-api, biome-host-api) MUST declare ` +
+      `XEMA_BIOME_SCAN_ROOTS_SERVER in its Helm + docker-compose env; this ` +
+      `error means a deployment is missing that config.`,
+  );
+}
+
+/**
+ * Walk a content directory for a given kind, returning every file whose
+ * extension matches the kind. Returns [] if the directory does not exist
+ * (so callers do not need to existsSync first). Throws if the directory
+ * exists but is not readable.
+ */
+export function walkContent(contentDir: string, kind: ContentKind): ContentFile[] {
+  if (!existsSync(contentDir)) return [];
+  const exts = CONTENT_KIND_EXTENSIONS[kind];
+  if (!exts) {
+    throw new Error(
+      `walkContent: unknown legacy content kind "${kind}". ` +
+        `Extend CONTENT_KIND_EXTENSIONS or migrate the caller to the xema.contributions protocol.`,
+    );
+  }
+  const out: ContentFile[] = [];
+
+  const stack: string[] = [contentDir];
+  while (stack.length > 0) {
+    const current = stack.pop();
+    if (!current) break;
+    let entries: Dirent[];
+    try {
+      entries = readdirSync(current, { withFileTypes: true }) as Dirent[];
+    } catch (err) {
+      throw new Error(
+        `walkContent: failed to read ${current}: ${(err as Error).message}`,
+      );
+    }
+    for (const entry of entries) {
+      if (entry.name.startsWith('.')) continue;
+      const full = join(current, entry.name);
+      if (entry.isDirectory()) {
+        // Skills are folder-shaped (SKILL.md inside) — recurse.
+        if (kind === 'openCodeSkills' || kind === 'deliverableSpecs') {
+          stack.push(full);
+        }
+        continue;
+      }
+      if (!entry.isFile() && !entry.isSymbolicLink()) continue;
+      // Defensive: resolve symlinks to a real file before checking extension.
+      try {
+        const stat = statSync(full);
+        if (!stat.isFile()) continue;
+      } catch {
+        continue;
+      }
+      if (!exts.some((ext) => entry.name.endsWith(ext))) continue;
+      const slug = stripExtension(entry.name);
+      out.push({ absolutePath: full, basename: entry.name, slug });
+    }
+  }
+  // Stable ordering for reproducible seed runs.
+  out.sort((a, b) => a.absolutePath.localeCompare(b.absolutePath));
+  return out;
+}
+
+function stripExtension(basename: string): string {
+  const dot = basename.lastIndexOf('.');
+  if (dot <= 0) return basename;
+  return basename.slice(0, dot);
+}
+
+// ── YAML/Markdown parsing ──────────────────────────────────────────────
+
+const FRONTMATTER_REGEX = /^---\r?\n([\s\S]*?)\r?\n---\r?\n?([\s\S]*)$/;
+
+export function readYamlFile(path: string): unknown {
+  const raw = readFileSync(path, 'utf-8');
+  return YAML.load(raw);
+}
+
+/**
+ * Parse a markdown file with YAML frontmatter. Returns `null` for the
+ * frontmatter half if the file has none — callers should fail fast in
+ * that case (every Xema agent .md file MUST have frontmatter).
+ */
+export function readMarkdownWithFrontmatter(path: string): {
+  frontmatter: unknown;
+  body: string;
+} {
+  const raw = readFileSync(path, 'utf-8');
+  const match = raw.match(FRONTMATTER_REGEX);
+  if (!match) {
+    return { frontmatter: null, body: raw };
+  }
+  const yamlPart = match[1] ?? '';
+  const body = match[2] ?? '';
+  return { frontmatter: YAML.load(yamlPart), body };
+}