@usegraft/compiler 0.1.1 → 1.0.0-beta.0

package/README.md

@@ -0,0 +1,44 @@
+# @usegraft/compiler
+
+> Validate authored MDX against your collections and project it into the content index. The bridge that makes file-authored content queryable.
+
+Part of [Graft](https://github.com/AndersonDesign1/graft), a CMS built so an AI agent is the primary operator.
+
+## Install
+
+```bash
+npm i @usegraft/compiler
+```
+
+Most projects use `graft compile` from [`@usegraft/cli`](https://www.npmjs.com/package/@usegraft/cli) rather than calling this directly.
+
+## Compile
+
+```ts
+import { compile, compileStatic } from "@usegraft/compiler";
+
+// into Postgres
+const result = await compile({ contentDir, collections, db, branchId: "main" });
+
+// or into a SQLite artifact, with no database at all
+const result = await compileStatic({ contentDir, collections, indexPath: ".graft/index.db" });
+```
+
+Both return a `ChangeSet`: what was added, changed and removed, plus the git SHA the tree was compiled from. Projection is a content-hash diff, not a delete-and-rebuild, so unchanged rows are untouched and removals are a soft delete.
+
+## What it refuses, and why
+
+`readDocs` validates every authored file before anything reaches the index:
+
+- Frontmatter against the collection schema.
+- Slug uniqueness within a collection.
+- Files sitting outside a registered collection folder.
+- **Executable MDX.** `{…}` expressions, `import`, `export` and spread attributes are refused, because rendering evaluates them as JavaScript on the server.
+
+Every offending document is reported at once rather than the first found, because an author fixing them one at a time learns the rule slowly.
+
+Set `export const mdxTrust = "full"` in `graft.config.ts` to allow full MDX. That is correct only where every author of the repository has commit access, since code review then really is the control. It defaults to `"restricted"`, matching `MdxBody`, so a document that compiles is a document that renders.
+
+---
+
+MIT. [Repository](https://github.com/AndersonDesign1/graft) · [Changelog](https://github.com/AndersonDesign1/graft/blob/main/packages/compiler/CHANGELOG.md)

package/dist/index.d.ts

@@ -1,6 +1,10 @@
 import { AnyCollection } from '@usegraft/core';
+import { MdxTrust } from '@usegraft/mdx-safety';
 import { Database, ChangeSet } from '@usegraft/db';
 
+/** Kebab-case: URL-safe, unambiguous in paths, and stable as a primary-key part. */
+/** URL-safe kebab-case: the only shape a slug may take. */
+declare const SLUG_RE: RegExp;
 /** One asset reference found in a document's frontmatter. */
 interface DocAsset {
     /** The field the reference came from. */
@@ -21,6 +25,51 @@ interface ProjectedDoc {
 }
 declare function parseDocument(raw: string, collection: AnyCollection, sourcePath: string): ProjectedDoc;
 
+/** How THIS surface tells a caller to do the thing the error is about. */
+interface ContentHints {
+    /** How to see what collections exist, e.g. "see list_collections". */
+    listCollections?: string;
+    /** How to create a document, e.g. "author it with write_content". */
+    authorDocument?: string;
+}
+/** Every `.md`/`.mdx` file under `dir`, recursively, in stable order. */
+declare function walkContentFiles(dir: string): string[];
+/** Resolve a registered collection, or throw a miss the caller can act on. */
+declare function requireCollection(collections: Record<string, AnyCollection>, name: string, hints?: ContentHints): AnyCollection;
+/** Parse every document in one collection's folder. Empty if it does not exist. */
+declare function readCollectionDocs(contentDir: string, collectionName: string, collection: AnyCollection): ProjectedDoc[];
+/** Find one document by slug, or throw DOCUMENT_NOT_FOUND listing the real ones. */
+declare function findDoc(contentDir: string, collectionName: string, collection: AnyCollection, slug: string, hints?: ContentHints): ProjectedDoc;
+
+interface ContainOptions {
+    /** What the path names, for the error message (default: "path"). */
+    label?: string;
+    /**
+     * Follow symlinks instead of refusing them. Only for paths the operator
+     * typed, never for anything a request or a manifest supplied.
+     */
+    allowSymlinks?: boolean;
+    /**
+     * Accept an absolute path, provided it still resolves inside the root.
+     *
+     * For a local agent naming a file on its own machine an absolute path is the
+     * natural form, and containment is what matters, not the spelling. Leave this
+     * off for anything arriving in a request field, where "relative to the root"
+     * is the contract and an absolute path means the caller misunderstood — or is
+     * probing.
+     */
+    allowAbsolute?: boolean;
+}
+/**
+ * Resolve `path` against `root` and return it only if it genuinely lives there.
+ *
+ * Refuses absolute paths, NUL bytes, anything that escapes the root once
+ * normalised, and — unless `allowSymlinks` — any segment that is a symbolic
+ * link, whether or not its target escapes. A symlink inside the tree is a
+ * redirection the caller did not get to choose, so the honest answer is no.
+ */
+declare function resolveContained(root: string, path: string, options?: ContainOptions): string;
+
 interface CompileOptions {
     contentDir: string;
     collections: Record<string, AnyCollection>;
@@ -38,6 +87,12 @@ interface CompileOptions {
      * unless this is set (legitimate after a collection rename/delete).
      */
     pruneUnknown?: boolean;
+    /**
+     * How much of MDX authored bodies may be. Defaults to "restricted", matching
+     * MdxBody, so a document that compiles is a document that renders. From
+     * `export const mdxTrust` in graft.config.ts.
+     */
+    mdxTrust?: MdxTrust;
 }
 interface CompileResult {
     count: number;
@@ -47,8 +102,16 @@ interface CompileResult {
     /** The commit the content tree was compiled from, when resolvable. */
     gitSha: string | null;
 }
+interface ReadDocsOptions {
+    /**
+     * How much of MDX authored bodies may be. Defaults to "restricted", which is
+     * also MdxBody's default, so a document that compiles is a document that
+     * renders. Set "full" only where every author has commit access.
+     */
+    mdxTrust?: MdxTrust;
+}
 /** Read + validate the content tree into normalized docs. Pure of the database. */
-declare function readDocs(contentDir: string, collections: Record<string, AnyCollection>): ProjectedDoc[];
+declare function readDocs(contentDir: string, collections: Record<string, AnyCollection>, options?: ReadDocsOptions): ProjectedDoc[];
 declare function compile(options: CompileOptions): Promise<CompileResult>;
 interface CompileStaticOptions {
     contentDir: string;
@@ -57,6 +120,12 @@ interface CompileStaticOptions {
     indexPath: string;
     /** As in CompileOptions: omit to auto-resolve, null to skip. */
     gitSha?: string | null;
+    /**
+     * How much of MDX authored bodies may be. Defaults to "restricted", matching
+     * MdxBody, so a document that compiles is a document that renders. From
+     * `export const mdxTrust` in graft.config.ts.
+     */
+    mdxTrust?: MdxTrust;
 }
 /**
  * The static-mode sibling of `compile`: same validate step (readDocs), but the
@@ -73,8 +142,18 @@ declare function resolveGitSha(cwd: string): string | null;
  * a Studio or MCP write served from one fails deep inside fs with an opaque
  * errno; authored content being files is the whole model, so the fix is to run
  * the writing surface where the checkout is writable.
+ *
+ * Takes a root and a relative path rather than a resolved one, and contains it
+ * here, because this sink previously trusted whatever path it was handed and
+ * `.greptile/rules.md` told reviewers that every sink in this package did its
+ * own containment. Studio resolved carefully; MCP's `write_content` did not,
+ * and a slug of `"../../escaped"` wrote outside the content tree — the exact
+ * bug the rule claimed was impossible. A guarantee that lives in each caller is
+ * one a new caller does not inherit.
+ *
+ * Returns the resolved path, so a caller that needs it does not re-derive it.
  */
-declare function writeDocumentFile(fullPath: string, raw: string): void;
+declare function writeDocumentFile(root: string, sourcePath: string, raw: string): string;
 /**
  * The bytes to write for a document.
  *
@@ -85,4 +164,4 @@ declare function writeDocumentFile(fullPath: string, raw: string): void;
  */
 declare function composeDocument(existingRaw: string | undefined, data: Record<string, unknown>, body: string): string;
 
-export { type CompileOptions, type CompileResult, type CompileStaticOptions, type DocAsset, type ProjectedDoc, compile, compileStatic, composeDocument, parseDocument, readDocs, resolveGitSha, writeDocumentFile };
+export { type CompileOptions, type CompileResult, type CompileStaticOptions, type ContainOptions, type ContentHints, type DocAsset, type ProjectedDoc, type ReadDocsOptions, SLUG_RE, compile, compileStatic, composeDocument, findDoc, parseDocument, readCollectionDocs, readDocs, requireCollection, resolveContained, resolveGitSha, walkContentFiles, writeDocumentFile };

package/dist/index.js

@@ -49,18 +49,121 @@ function collectAssets(collection, data) {
   return assets;
 }
 
-// src/compile.ts
-import { execFileSync } from "child_process";
+// src/content-files.ts
 import { existsSync, readdirSync, readFileSync, statSync } from "fs";
 import { join, relative, sep } from "path";
 import { GraftError as GraftError2 } from "@usegraft/contracts";
+function walkContentFiles(dir) {
+  const out = [];
+  for (const name of readdirSync(dir)) {
+    const full = join(dir, name);
+    if (statSync(full).isDirectory()) out.push(...walkContentFiles(full));
+    else if (/\.mdx?$/.test(name)) out.push(full);
+  }
+  return out.sort();
+}
+function requireCollection(collections, name, hints = {}) {
+  const collection = collections[name];
+  if (collection) return collection;
+  const registered = Object.keys(collections);
+  const known = registered.join(", ") || "(none registered)";
+  const where = hints.listCollections ? ` (${hints.listCollections})` : "";
+  throw new GraftError2({
+    code: "COLLECTION_NOT_FOUND",
+    message: `No collection named "${name}" is registered`,
+    fix: `Use one of the registered collections: ${known}${where}, or add defineCollection({ name: "${name}", \u2026 }) to the schema.`,
+    details: { collection: name, registered }
+  });
+}
+function readCollectionDocs(contentDir, collectionName, collection) {
+  const dir = join(contentDir, collectionName);
+  if (!existsSync(dir) || !statSync(dir).isDirectory()) return [];
+  const docs = [];
+  for (const file of walkContentFiles(dir)) {
+    const sourcePath = relative(contentDir, file).split(sep).join("/");
+    docs.push(parseDocument(readFileSync(file, "utf8"), collection, sourcePath));
+  }
+  return docs.sort((a, b) => a.slug.localeCompare(b.slug));
+}
+function findDoc(contentDir, collectionName, collection, slug, hints = {}) {
+  const docs = readCollectionDocs(contentDir, collectionName, collection);
+  const doc = docs.find((candidate) => candidate.slug === slug);
+  if (doc) return doc;
+  const slugs = docs.map((candidate) => candidate.slug);
+  const author = hints.authorDocument ? ` ${hints.authorDocument}` : "";
+  throw new GraftError2({
+    code: "DOCUMENT_NOT_FOUND",
+    message: `No document "${slug}" in collection "${collectionName}"`,
+    fix: slugs.length > 0 ? `Existing slugs: ${slugs.join(", ")}. Use one of those,${author || " or create it."}` : `Collection "${collectionName}" has no documents yet \u2014${author || " create the first one."}`,
+    details: { collection: collectionName, slug, existing: slugs }
+  });
+}
+
+// src/paths.ts
+import { existsSync as existsSync2, lstatSync } from "fs";
+import { isAbsolute, normalize, resolve, sep as sep2 } from "path";
+import { GraftError as GraftError3 } from "@usegraft/contracts";
+function refuse(label, path, why, fix) {
+  throw new GraftError3({
+    code: "INPUT_VALIDATION_FAILED",
+    message: `${label} "${path}" ${why}.`,
+    fix,
+    details: { path }
+  });
+}
+function resolveContained(root, path, options = {}) {
+  const label = options.label ?? "path";
+  const rootAbs = resolve(root);
+  if (!path || path.includes("\0")) {
+    refuse(
+      label,
+      path,
+      "is empty or contains a NUL byte",
+      "Pass a relative path with no NUL bytes."
+    );
+  }
+  if (isAbsolute(path) && options.allowAbsolute !== true) {
+    refuse(label, path, "is absolute", `Pass a path relative to ${rootAbs}.`);
+  }
+  const candidate = isAbsolute(path) ? resolve(normalize(path)) : resolve(rootAbs, normalize(path));
+  if (candidate !== rootAbs && !candidate.startsWith(rootAbs + sep2)) {
+    refuse(
+      label,
+      path,
+      "resolves outside the permitted directory",
+      `Pass a path inside ${rootAbs}.`
+    );
+  }
+  if (options.allowSymlinks !== true) {
+    let cursor = rootAbs;
+    for (const segment of candidate.slice(rootAbs.length).split(sep2).filter(Boolean)) {
+      cursor = resolve(cursor, segment);
+      if (existsSync2(cursor) && lstatSync(cursor).isSymbolicLink()) {
+        refuse(
+          label,
+          path,
+          "passes through a symbolic link",
+          "Symbolic links are refused because they redirect a path somewhere the caller did not name. Replace the link with a real file, or read the target directly."
+        );
+      }
+    }
+  }
+  return candidate;
+}
+
+// src/compile.ts
+import { execFileSync } from "child_process";
+import { existsSync as existsSync3, readFileSync as readFileSync2, statSync as statSync2 } from "fs";
+import { relative as relative2, sep as sep3 } from "path";
+import { GraftError as GraftError4 } from "@usegraft/contracts";
+import { findExecutableMdx, UncheckableMdxError } from "@usegraft/mdx-safety";
 import {
   projectBranchContent,
   projectStaticContent
 } from "@usegraft/db";
-function readDocs(contentDir, collections) {
-  if (!existsSync(contentDir) || !statSync(contentDir).isDirectory()) {
-    throw new GraftError2({
+function readDocs(contentDir, collections, options = {}) {
+  if (!existsSync3(contentDir) || !statSync2(contentDir).isDirectory()) {
+    throw new GraftError4({
       code: "CONTENT_DIR_NOT_FOUND",
       message: `Content directory not found: ${contentDir}`,
       fix: `Create it (documents live at <contentDir>/<collection>/<slug>.mdx, e.g. content/pages/home.mdx) or pass the correct \`contentDir\` to compile().`,
@@ -69,14 +172,17 @@ function readDocs(contentDir, collections) {
   }
   const docs = [];
   const seen = /* @__PURE__ */ new Map();
-  for (const file of walk(contentDir)) {
-    const sourcePath = relative(contentDir, file).split(sep).join("/");
+  const executable = [];
+  const unreadable = [];
+  const mdxTrust = options.mdxTrust ?? "restricted";
+  for (const file of walkContentFiles(contentDir)) {
+    const sourcePath = relative2(contentDir, file).split(sep3).join("/");
     const segments = sourcePath.split("/");
     const collectionName = segments.length > 1 ? segments[0] ?? "" : "";
     const collection = collections[collectionName];
     if (!collection) {
       const known = Object.keys(collections).join(", ") || "(none registered)";
-      throw new GraftError2({
+      throw new GraftError4({
         code: "COLLECTION_NOT_FOUND",
         message: segments.length > 1 ? `No collection registered for "${collectionName}" (${sourcePath})` : `${sourcePath} sits at the content root; documents must live in a collection folder`,
         fix: segments.length > 1 ? `Register it \u2014 defineCollection({ name: "${collectionName}", fields: { \u2026 } }) and pass it to compile() \u2014 or move the file into one of: ${known}.` : `Move it to <contentDir>/<collection>/${sourcePath}. Registered collections: ${known}.`,
@@ -84,18 +190,34 @@ function readDocs(contentDir, collections) {
       });
     }
     if (collection.authority === "db-authoritative") {
-      throw new GraftError2({
+      throw new GraftError4({
         code: "AUTHORITY_MISMATCH",
         message: `${sourcePath} is a file, but collection "${collectionName}" is db-authoritative \u2014 its records live in Postgres, not in the content tree.`,
         fix: `Remove the file and write the data through the collection's functions instead (insertRecord via a defineFunction mutation). Files are only for file-authoritative collections.`,
         details: { sourcePath, collection: collectionName, authority: collection.authority }
       });
     }
-    const doc = parseDocument(readFileSync(file, "utf8"), collection, sourcePath);
+    const doc = parseDocument(readFileSync2(file, "utf8"), collection, sourcePath);
+    if (mdxTrust !== "full") {
+      try {
+        const found = findExecutableMdx(doc.body);
+        if (found.length > 0) {
+          const first = found[0];
+          executable.push({
+            sourcePath,
+            found: found.length,
+            first: first?.line === void 0 ? "" : `line ${first.line}`
+          });
+        }
+      } catch (error) {
+        if (!(error instanceof UncheckableMdxError)) throw error;
+        unreadable.push({ sourcePath, reason: error.message });
+      }
+    }
     const key = `${doc.collection}/${doc.slug}`;
     const existing = seen.get(key);
     if (existing) {
-      throw new GraftError2({
+      throw new GraftError4({
         code: "SLUG_NOT_UNIQUE",
         message: `Slug "${doc.slug}" in collection "${doc.collection}" is used by both ${existing} and ${sourcePath}`,
         fix: `Give one a unique \`slug\` in frontmatter, or rename the file.`,
@@ -105,12 +227,33 @@ function readDocs(contentDir, collections) {
     seen.set(key, sourcePath);
     docs.push(doc);
   }
+  if (unreadable.length > 0) {
+    const listed = unreadable.map((u) => `${u.sourcePath} (${u.reason})`).join(", ");
+    throw new GraftError4({
+      code: "INPUT_VALIDATION_FAILED",
+      message: `${unreadable.length} document(s) could not be checked for executable MDX: ${listed}.`,
+      fix: "Fix the MDX syntax so it parses. Unparseable source is refused rather than indexed, because the renderer's parser is not this one and the difference between them is where executable source would hide.",
+      details: { unreadable }
+    });
+  }
+  if (executable.length > 0) {
+    const listed = executable.slice(0, 10).map((e) => e.first === "" ? e.sourcePath : `${e.sourcePath} (${e.first})`).join(", ");
+    const more = executable.length > 10 ? ` \u2026and ${executable.length - 10} more` : "";
+    throw new GraftError4({
+      code: "INPUT_VALIDATION_FAILED",
+      message: `Executable MDX in ${executable.length} document(s): ${listed}${more}.`,
+      fix: 'Rendering evaluates `{\u2026}` and `import` as JavaScript on the server, and MdxBody refuses them by default, so these documents would fail per-request rather than here. Write prose, Markdown and components with literal attributes. If every author of this repository has commit access, set `export const mdxTrust = "full"` in graft.config.ts and pass trust="full" to MdxBody. Code review is the control in that case, which is what ADR 0004 assumes.',
+      details: { documents: executable.length, offenders: executable.slice(0, 20) }
+    });
+  }
   return docs.sort(
     (a, b) => a.collection === b.collection ? a.slug.localeCompare(b.slug) : a.collection.localeCompare(b.collection)
   );
 }
 async function compile(options) {
-  const docs = readDocs(options.contentDir, options.collections);
+  const docs = readDocs(options.contentDir, options.collections, {
+    mdxTrust: options.mdxTrust
+  });
   const gitSha = options.gitSha === void 0 ? resolveGitSha(options.contentDir) : options.gitSha;
   const changes = await projectBranchContent(
     options.db,
@@ -132,7 +275,9 @@ async function compile(options) {
   return { count: docs.length, docs, changes, gitSha };
 }
 async function compileStatic(options) {
-  const docs = readDocs(options.contentDir, options.collections);
+  const docs = readDocs(options.contentDir, options.collections, {
+    mdxTrust: options.mdxTrust
+  });
   const gitSha = options.gitSha === void 0 ? resolveGitSha(options.contentDir) : options.gitSha;
   const changes = await projectStaticContent(
     docs.map((doc) => ({
@@ -158,30 +303,23 @@ function resolveGitSha(cwd) {
     return null;
   }
 }
-function walk(dir) {
-  const out = [];
-  for (const name of readdirSync(dir)) {
-    const full = join(dir, name);
-    if (statSync(full).isDirectory()) out.push(...walk(full));
-    else if (/\.mdx?$/.test(name)) out.push(full);
-  }
-  return out.sort();
-}
 
 // src/serialize.ts
 import { mkdirSync, writeFileSync } from "fs";
 import { dirname } from "path";
-import { GraftError as GraftError3 } from "@usegraft/contracts";
+import { GraftError as GraftError5 } from "@usegraft/contracts";
 import matter2 from "gray-matter";
 var READ_ONLY_CODES = /* @__PURE__ */ new Set(["EROFS", "EACCES", "EPERM"]);
-function writeDocumentFile(fullPath, raw) {
+function writeDocumentFile(root, sourcePath, raw) {
+  const fullPath = resolveContained(root, sourcePath, { label: "document" });
   try {
     mkdirSync(dirname(fullPath), { recursive: true });
     writeFileSync(fullPath, raw);
+    return fullPath;
   } catch (error) {
     const code = error.code ?? "";
     if (!READ_ONLY_CODES.has(code)) throw error;
-    throw new GraftError3({
+    throw new GraftError5({
       code: "CONTENT_TREE_READ_ONLY",
       message: `Cannot write ${fullPath}: the filesystem refused it (${code}).`,
       fix: "Authored content lives in files, so writing needs a writable checkout. Run this surface locally (`graft studio` / `graft mcp`) or in a container with the project mounted read-write; a serverless deployment's filesystem is read-only and should serve reads only.",
@@ -280,11 +418,17 @@ function composeDocument(existingRaw, data, body) {
   return frontmatter + separator + body + (needsNewline ? newline : "");
 }
 export {
+  SLUG_RE,
   compile,
   compileStatic,
   composeDocument,
+  findDoc,
   parseDocument,
+  readCollectionDocs,
   readDocs,
+  requireCollection,
+  resolveContained,
   resolveGitSha,
+  walkContentFiles,
   writeDocumentFile
 };

package/package.json

@@ -1,6 +1,21 @@
 {
   "name": "@usegraft/compiler",
-  "version": "0.1.1",
+  "version": "1.0.0-beta.0",
+  "description": "Validate authored MDX against your collections and project it into the content index.",
+  "keywords": [
+    "agent",
+    "ai",
+    "cms",
+    "compiler",
+    "content-index",
+    "graft",
+    "headless-cms",
+    "mcp",
+    "mdx",
+    "typescript",
+    "validation"
+  ],
+  "homepage": "https://github.com/AndersonDesign1/graft#readme",
   "license": "MIT",
   "repository": {
     "type": "git",
@@ -24,9 +39,10 @@
     "access": "public"
   },
   "dependencies": {
-    "@usegraft/contracts": "0.1.1",
-    "@usegraft/core": "0.1.1",
-    "@usegraft/db": "0.1.1",
+    "@usegraft/contracts": "1.0.0-beta.0",
+    "@usegraft/core": "1.0.0-beta.0",
+    "@usegraft/db": "1.0.0-beta.0",
+    "@usegraft/mdx-safety": "1.0.0-beta.0",
     "gray-matter": "^4.0.3"
   },
   "engines": {
@@ -36,7 +52,6 @@
     "build": "tsup src/index.ts --format esm --dts --clean",
     "dev": "tsup src/index.ts --format esm --watch",
     "typecheck": "tsc --noEmit",
-    "test": "vitest run --passWithNoTests",
-    "lint": "oxlint ."
+    "test": "vitest run --passWithNoTests"
   }
 }