@backmatter/core 0.0.17 → 0.0.19

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@backmatter/core",
-  "version": "0.0.17",
+  "version": "0.0.19",
   "description": "Experimental Backmatter SDK. APIs will break. Not ready for production.",
   "type": "module",
   "publishConfig": {

package/src/block.ts

@@ -39,9 +39,17 @@ export function defineBlock<const D extends BlockDef>(def: D): D {
   if (def.component === undefined) {
     throw new Error("block.component is required");
   }
+  if (RESERVED_BLOCK_PROP in def.fields) {
+    throw new Error(
+      `block.fields.${RESERVED_BLOCK_PROP} is reserved: it carries the external source record`,
+    );
+  }
   return def;
 }
 
+/** Prop name the renderer uses for an external model's record. */
+export const RESERVED_BLOCK_PROP = "data";
+
 type Simplify<T> = { -readonly [K in keyof T]: T[K] } & {};
 
 type ScalarOf<K extends FieldKind> = K extends "number"
@@ -58,8 +66,11 @@ type IsRequired<F> = F extends { required?: infer R }
     : false
   : false;
 
-/** TypeScript shape of a block's props, derived from `field.*` calls. */
-export type InferBlockProps<Fields extends Record<string, FieldDef>> = Simplify<
+/**
+ * TypeScript shape of a block's props, derived from `field.*` calls. Pass the
+ * record type as `Data` for blocks on an `external` model to type `props.data`.
+ */
+export type InferBlockProps<Fields extends Record<string, FieldDef>, Data = undefined> = Simplify<
   {
     [N in keyof Fields as IsRequired<Fields[N]> extends true ? N : never]: ScalarOf<
       Fields[N]["kind"]
@@ -68,7 +79,7 @@ export type InferBlockProps<Fields extends Record<string, FieldDef>> = Simplify<
     [N in keyof Fields as IsRequired<Fields[N]> extends true ? never : N]?: ScalarOf<
       Fields[N]["kind"]
     >;
-  }
+  } & ([Data] extends [undefined] ? {} : { data: Data })
 >;
 
 export function serializeBlock(block: BlockDef): BlockSnapshot {

package/src/config.ts

@@ -16,6 +16,8 @@ export type BackmatterDirs = {
   layouts?: string;
   /** Folder with one `defineObject` default export per file. Default: "objects". */
   objects?: string;
+  /** Folder with one `defineSource` default export per external model type. Default: "sources". */
+  sources?: string;
 };
 
 export type BackmatterConfig = {
@@ -57,6 +59,7 @@ export const DEFAULT_DIRS: Required<BackmatterDirs> = {
   mdx: "mdx",
   layouts: "layouts",
   objects: "objects",
+  sources: "sources",
 };
 export const DEFAULT_LOCALE = "en";
 export const CLOUD_API_URL = "https://api.backmatter.app";
@@ -190,6 +193,7 @@ export function resolveConfig(
       mdx: config.dirs?.mdx ?? DEFAULT_DIRS.mdx,
       layouts: config.dirs?.layouts ?? DEFAULT_DIRS.layouts,
       objects: config.dirs?.objects ?? DEFAULT_DIRS.objects,
+      sources: config.dirs?.sources ?? DEFAULT_DIRS.sources,
     },
   };
 }

package/src/content-committed.ts

@@ -40,6 +40,22 @@ export function listCacheTags(type?: string): string[] {
   return type ? [BACKMATTER_CACHE_TAG, `backmatter:${type}`] : [BACKMATTER_CACHE_TAG];
 }
 
+/**
+ * Cache tags a commit should expire. A normal commit expires the touched
+ * documents and their type lists. A full rebuild expires the global tag,
+ * which every cached entry carries.
+ */
+export function revalidateTagsForCommit(event: Pick<ContentCommittedEvent, "docs" | "full">): string[] {
+  if (event.full) return [BACKMATTER_CACHE_TAG];
+  const tags = new Set<string>();
+  for (const doc of event.docs ?? []) {
+    for (const tag of contentCacheTags(doc.locale, doc.type, doc.slug)) {
+      if (tag !== BACKMATTER_CACHE_TAG) tags.add(tag);
+    }
+  }
+  return [...tags];
+}
+
 /** Site paths the host should `revalidatePath` for these docs. */
 export function pathsForCommittedDocs(
   models: readonly RouteModel[],

package/src/index.ts

@@ -24,7 +24,9 @@ export {
   defaultModelPath,
   DEFAULT_MODEL_PATH,
   DEFAULT_PAGE_PATH,
+  DEFAULT_TEMPLATE_SLUG,
   IMPLICIT_SINGLETON_SLUG,
+  isTemplateSlug,
   serializeModel,
   serializeSchema,
   parseSchemaSnapshot,
@@ -40,8 +42,12 @@ export {
   defineBlock,
   serializeBlock,
   parseBlockSnapshot,
+  RESERVED_BLOCK_PROP,
 } from "./block";
 
+export type { SourceDef, SourceRef } from "./source";
+export { defineSource } from "./source";
+
 export type { DocFormat } from "./doc-format";
 export { modelFormat, parseDoc, serializeDoc } from "./doc-format";
 
@@ -118,21 +124,30 @@ export {
 export type { BackmatterDoc, BackmatterDocSummary } from "./doc";
 export { createQuery } from "./query";
 export type { Query, ModelQuery, TypedDoc, Where, Include, DocLoader } from "./query";
-export type { PreviewCheckout, PreviewTicket } from "./preview-session";
+export type { PreviewCheckout, PreviewSessionClaims, PreviewTicket } from "./preview-session";
 export {
   PREVIEW_COOKIE,
+  PREVIEW_SESSION_COOKIE,
+  PREVIEW_SESSION_QUERY,
+  PREVIEW_SESSION_STORAGE_KEY,
   PREVIEW_ENABLE_PATH,
   PREVIEW_QUERY,
   PREVIEW_CHECKOUT_QUERY,
   previewLocationWithCheckout,
   isSafeRedirectPath,
+  mintPreviewSession,
   mintPreviewTicket,
+  peekPreviewSessionWorkspace,
   previewMiddlewareDecision,
   readPreviewCookie,
+  readPreviewSession,
   readPreviewTicket,
+  scopePreviewAccess,
+  takePreviewSessionFromSearch,
   writePreviewCookie,
   PREVIEW_TICKET_TTL_MS,
   PREVIEW_COOKIE_TTL_MS,
+  PREVIEW_SESSION_TTL_MS,
 } from "./preview-session";
 export { signPayload, verifyPayload } from "./signed-token";
 export { getDocFromApi, listDocsFromApi } from "./api-loader";
@@ -147,5 +162,6 @@ export {
   contentCacheTags,
   listCacheTags,
   pathsForCommittedDocs,
+  revalidateTagsForCommit,
   revalidateUrl,
 } from "./content-committed";

package/src/model.ts

@@ -33,12 +33,51 @@ export type ModelDef = {
    * in a schema may set this.
    */
   indexSlug?: string;
+  /**
+   * Slugs belong to an outside system (a catalog). The host provides
+   * `sources/{type}.ts`; a record that does not exist there is a 404. A
+   * document at the `template` slug renders every record without its own file.
+   */
+  external?: boolean;
+  /** Reserved slug of the template document. Default `_template` when `external`. */
+  template?: string;
   fields: Record<string, FieldDef>;
 };
 
 /** What `defineModel` accepts: `path` falls back to the folder convention. */
 export type ModelInput = Omit<ModelDef, "path"> & { path?: string };
 
+export const DEFAULT_TEMPLATE_SLUG = "_template";
+
+/** True when `slug` is the template document of an external model. */
+export function isTemplateSlug(
+  model: Pick<ModelDef, "type" | "external" | "template"> | null | undefined,
+  slug: string,
+): boolean {
+  if (!model?.external) return false;
+  return slug === (model.template ?? DEFAULT_TEMPLATE_SLUG);
+}
+
+/** Validate `external` / `template` and return the template slug to store, if any. */
+export function resolveTemplateSlug(
+  def: { external?: boolean; template?: string; singleton?: boolean; indexSlug?: string },
+  label = "model",
+): string | undefined {
+  if (def.template !== undefined && !def.external) {
+    throw new Error(`${label}.template requires external: true`);
+  }
+  if (!def.external) return undefined;
+  if (def.singleton) throw new Error(`${label}.external cannot be combined with singleton`);
+  if (def.indexSlug !== undefined) {
+    throw new Error(`${label}.external cannot be combined with indexSlug`);
+  }
+  if (def.template === undefined) return DEFAULT_TEMPLATE_SLUG;
+  if (typeof def.template !== "string" || !def.template.trim()) {
+    throw new Error(`${label}.template must be a non-empty string`);
+  }
+  return def.template.trim();
+}
+
 /** Default repo path when a model does not declare one. */
 export const DEFAULT_MODEL_PATH = "content/{locale}/{type}/{slug}.mdx";
 export const DEFAULT_PAGE_PATH = "content/{locale}/{type}/{slug}.json";
@@ -55,6 +94,8 @@ export type ModelSnapshot = {
   singleton?: boolean;
   format?: DocFormat;
   indexSlug?: string;
+  external?: boolean;
+  template?: string;
   fields: Record<string, FieldSnapshot>;
 };
 
@@ -91,12 +132,17 @@ export type SchemaSnapshot = {
 };
 
 type WithPath<D extends ModelInput> = D extends { path: string } ? D : D & { path: string };
+type WithTemplate<D extends ModelInput> = D extends { external: true }
+  ? D extends { template: string }
+    ? D
+    : D & { template: string }
+  : D;
 
 /**
  * Validates the definition at runtime and preserves its literal types.
  * `path` defaults to `content/{locale}/{type}/{slug}.mdx`.
  */
-export function defineModel<const D extends ModelInput>(def: D): WithPath<D> {
+export function defineModel<const D extends ModelInput>(def: D): WithTemplate<WithPath<D>> {
   if (!def.type || typeof def.type !== "string") {
     throw new Error("model.type is required");
   }
@@ -123,7 +169,11 @@ export function defineModel<const D extends ModelInput>(def: D): WithPath<D> {
   if (!def.fields || typeof def.fields !== "object") {
     throw new Error("model.fields is required");
   }
-  const withPath = (def.path === undefined ? { ...def, path } : { ...def }) as WithPath<D>;
+  const template = resolveTemplateSlug(def);
+  const withPath = (def.path === undefined ? { ...def, path } : { ...def }) as WithTemplate<
+    WithPath<D>
+  >;
+  if (template !== undefined) (withPath as { template?: string }).template = template;
   if (singleton && def.route === undefined) {
     return { ...withPath, route: false };
   }
@@ -173,6 +223,10 @@ export function serializeModel(model: ModelDef): ModelSnapshot {
   if (model.singleton) snapshot.singleton = true;
   if (model.format === "page") snapshot.format = "page";
   if (model.indexSlug !== undefined) snapshot.indexSlug = model.indexSlug.trim();
+  if (model.external) {
+    snapshot.external = true;
+    snapshot.template = model.template ?? DEFAULT_TEMPLATE_SLUG;
+  }
   return snapshot;
 }
 
@@ -264,6 +318,22 @@ export function parseSchemaSnapshot(input: unknown): SchemaSnapshot {
       }
       snapshot.indexSlug = m.indexSlug.trim();
     }
+    if (m.external !== undefined && typeof m.external !== "boolean") {
+      throw new Error(`schema.models[${i}].external must be a boolean`);
+    }
+    const template = resolveTemplateSlug(
+      {
+        external: m.external === true,
+        template: m.template as string | undefined,
+        singleton,
+        indexSlug: snapshot.indexSlug,
+      },
+      `schema.models[${i}]`,
+    );
+    if (template !== undefined) {
+      snapshot.external = true;
+      snapshot.template = template;
+    }
     return snapshot;
   });
 

package/src/node/cli.ts

@@ -30,7 +30,7 @@ export type CliArgs = {
 
 const USAGE = `Usage:
   backmatter init                          scaffold backmatter.config.ts, the catch-all route and .backmatter/
-  backmatter generate  [--root <dir>]      write .backmatter/index.ts (typed cms), blocks.ts, mdx.ts and layouts.ts
+  backmatter generate  [--root <dir>]      write .backmatter/index.ts (typed cms), blocks.ts, mdx.ts, layouts.ts and sources.ts
   backmatter validate  [--root <dir>] [--models <file>] [--json]
   backmatter schema    [--root <dir>] [--models <file>] [--out <dir>]
 
@@ -239,6 +239,7 @@ export async function runGenerate(args: CliArgs): Promise<GenerateResult> {
     mdxDir: config.dirs.mdx,
     layoutsDir: config.dirs.layouts,
     objectsDir: config.dirs.objects,
+    sourcesDir: config.dirs.sources,
   });
 }
 
@@ -272,9 +273,10 @@ import { richtextComponents } from "${REGISTRY_SPECIFIER}/richtext";
 import { blockLoaders } from "${REGISTRY_SPECIFIER}/blocks";
 import { blockFallbacks } from "${REGISTRY_SPECIFIER}/block-fallbacks";
 import { layouts } from "${REGISTRY_SPECIFIER}/layouts";
+import { sources } from "${REGISTRY_SPECIFIER}/sources";
 
 const components = { ...richtextComponents, ...mdxComponents };
-const route = createCatchAll(cms, { components, blocks: blockLoaders, fallbacks: blockFallbacks, layouts });
+const route = createCatchAll(cms, { components, blocks: blockLoaders, fallbacks: blockFallbacks, layouts, sources });
 
 export const generateStaticParams = route.generateStaticParams;
 
@@ -348,6 +350,15 @@ export const config = {
 `;
 }
 
+function renderCmsProxyRoute(): string {
+  return `// api/backmatter/cms/[...path]/route.ts — same-origin proxy for the preview session.
+import { createCmsProxy } from "@backmatter/next/cms-proxy";
+import { cms } from "${REGISTRY_SPECIFIER}";
+
+export const { GET, POST, PUT, DELETE } = createCmsProxy(cms);
+`;
+}
+
 function renderRevalidateRoute(): string {
   return `// api/backmatter/revalidate/route.ts — content.committed from the BACKMATTER server.
 import { createRevalidateRoute } from "@backmatter/next/revalidate";
@@ -402,6 +413,16 @@ export async function runInit(args: CliArgs): Promise<string[]> {
       notes.push(`kept existing ${path.join(appDir, "api/backmatter/revalidate/route.ts")}`);
     }
 
+    const cmsDir = path.join(root, appDir, "api", "backmatter", "cms", "[...path]");
+    const cmsFile = path.join(cmsDir, "route.ts");
+    if (!existsSync(cmsFile)) {
+      await mkdir(cmsDir, { recursive: true });
+      await writeFile(cmsFile, renderCmsProxyRoute());
+      notes.push(`wrote ${path.join(appDir, "api/backmatter/cms/[...path]/route.ts")}`);
+    } else {
+      notes.push(`kept existing ${path.join(appDir, "api/backmatter/cms/[...path]/route.ts")}`);
+    }
+
     const previewDir = path.join(root, appDir, "api", "backmatter", "preview");
     const previewFile = path.join(previewDir, "route.ts");
     if (!existsSync(previewFile)) {
@@ -450,7 +471,7 @@ export async function runInit(args: CliArgs): Promise<string[]> {
     notes.push("no @backmatter/next dependency: skipped the catch-all route");
   }
 
-  for (const dir of ["models", "content", "blocks", "richtext", "mdx", "layouts"]) {
+  for (const dir of ["models", "content", "blocks", "richtext", "mdx", "layouts", "sources"]) {
     await mkdir(path.join(root, dir), { recursive: true });
   }
 

package/src/node/discover.ts

@@ -101,6 +101,11 @@ export function listLayoutFiles(root: string, layoutsDir = "layouts"): ModelFile
 export function listObjectFiles(root: string, objectsDir = "objects"): ModelFile[] {
   return listModelFiles(root, objectsDir);
 }
+
+/** `sources/{type}.ts`: one `defineSource` default export per external model type. */
+export function listSourceDefFiles(root: string, sourcesDir = "sources"): ModelFile[] {
+  return listModelFiles(root, sourcesDir);
+}
 export function listBlockFiles(
   root: string,
   blocksDir = "blocks",

package/src/node/eval-model.ts

@@ -1,15 +1,18 @@
 // packages/core/src/node/eval-model.ts
-// Sandbox-eval a convention `models/{type}.ts` blob (GitHub import). Only
-// `defineModel` / `field` from @backmatter/core or @backmatter/next are in scope —
-// never import() the customer checkout.
+// Sandbox-eval convention model and object files (GitHub import). The allowed
+// surface is the same one `discoverModels` / `discoverObjects` run: `defineModel`,
+// `defineObject`, and `field` from @backmatter/core or @backmatter/next, plus
+// `import name from "../objects/<name>"`. Never import() the customer checkout.
 import { runInNewContext } from "node:vm";
 import { defineModel, type ModelDef } from "../model";
+import { defineObject, type ObjectDef } from "../object";
 import { field } from "../fields";
 
 const SOURCE_EXT = /\.(ts|tsx|js|jsx|mjs)$/;
 const IGNORED = /(\.test\.|\.spec\.|\.d\.ts$|\.stories\.)/;
 const ALLOWED_IMPORT = /^\s*import\s+\{([^}]+)\}\s+from\s+["'](@backmatter\/core|@backmatter\/next)["']\s*;?\s*$/gm;
 const ALLOWED_NAMES = new Set(["defineModel", "field"]);
+const OBJECT_NAMES = new Set(["defineObject", "field"]);
 const FORBIDDEN = [
   { re: /\brequire\s*\(/, name: "require" },
   { re: /\bimport\s*\(/, name: "import" },
@@ -44,38 +47,61 @@ function stemOf(repoPath: string): string {
   return name.replace(SOURCE_EXT, "");
 }
 
-function stripAllowedImports(source: string): string {
+function stripAllowedImports(
+  source: string,
+  allowed: Set<string> = ALLOWED_NAMES,
+  label = "defineModel and field",
+): string {
+  ALLOWED_IMPORT.lastIndex = 0;
   const names = new Set<string>();
   const stripped = source.replace(ALLOWED_IMPORT, (_full, specifiers: string) => {
     for (const part of specifiers.split(",")) {
       const name = part.trim();
       if (!name) continue;
-      if (/\bas\b/.test(name) || !ALLOWED_NAMES.has(name)) {
-        throw new Error(`import "${name}" is not allowed; only defineModel and field`);
+      if (/\bas\b/.test(name) || !allowed.has(name)) {
+        throw new Error(`import "${name}" is not allowed; only ${label}`);
       }
       names.add(name);
     }
     return "";
   });
   if (/^\s*import\b/m.test(stripped)) {
-    throw new Error("import is not allowed except defineModel/field from @backmatter/core or @backmatter/next");
+    throw new Error(
+      `import is not allowed except ${label} from @backmatter/core or @backmatter/next`,
+    );
   }
   return stripped;
 }
 
-function assertNoForbidden(source: string): void {
+/** `import name from "../objects/<file>"` — the only local import a model may use. */
+function takeObjectImports(source: string): {
+  source: string;
+  bindings: Array<{ local: string; stem: string }>;
+} {
+  const bindings: Array<{ local: string; stem: string }> = [];
+  const re =
+    /^\s*import\s+([A-Za-z_$][\w$]*)\s+from\s+["']\.\.\/objects\/([A-Za-z][A-Za-z0-9]*)(?:\.(?:ts|tsx|js|jsx|mjs))?["']\s*;?\s*$/gm;
+  const next = source.replace(re, (_full, local: string, stem: string) => {
+    bindings.push({ local, stem });
+    return "";
+  });
+  return { source: next, bindings };
+}
+
+function assertNoForbidden(source: string, kind = "model"): void {
   for (const { re, name } of FORBIDDEN) {
-    if (re.test(source)) throw new Error(`${name} is not allowed in model files`);
+    re.lastIndex = 0;
+    if (re.test(source)) throw new Error(`${name} is not allowed in ${kind} files`);
   }
 }
 
-function rewriteDefaultExport(source: string): string {
+function rewriteDefaultExport(source: string, what = "defineModel({ ... })"): string {
   if (!/\bexport\s+default\b/.test(source)) {
-    throw new Error("must export default defineModel({ ... })");
+    throw new Error(`must export default ${what}`);
   }
   const rewritten = source.replace(/\bexport\s+default\s+/, "return ");
   if (/\bexport\b/.test(rewritten)) {
-    throw new Error("must export default defineModel({ ... })");
+    throw new Error(`must export default ${what}`);
   }
   return rewritten;
 }
@@ -85,18 +111,75 @@ function transpile(source: string): string {
   return transpiler.transformSync(source);
 }
 
+function isObjectDef(value: unknown): value is ObjectDef {
+  if (!value || typeof value !== "object") return false;
+  const object = value as Record<string, unknown>;
+  return typeof object.name === "string" && typeof object.fields === "object" && !("type" in object);
+}
+
+/**
+ * Evaluate one `objects/{name}.ts` blob. Same rule as `discoverObjects`:
+ * the file stem must equal `object.name`.
+ */
+export function evalObjectSource(source: string, stem: string): ObjectDef {
+  const body = rewriteDefaultExport(
+    stripAllowedImports(source, OBJECT_NAMES, "defineObject and field"),
+    "defineObject({ ... })",
+  );
+  assertNoForbidden(body, "object");
+  const wrapped = `(function () {\n${body}\n})()`;
+  const js = transpile(wrapped);
+  const sandbox = Object.create(null) as {
+    defineObject: typeof defineObject;
+    field: typeof field;
+  };
+  sandbox.defineObject = defineObject;
+  sandbox.field = field;
+  Object.freeze(sandbox);
+
+  let result: unknown;
+  try {
+    result = runInNewContext(js, sandbox, {
+      timeout: EVAL_TIMEOUT_MS,
+      filename: `${stem}.js`,
+    });
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    throw new Error(`object "${stem}" failed to evaluate: ${message}`);
+  }
+  if (!isObjectDef(result)) {
+    throw new Error("must export default defineObject({ ... })");
+  }
+  if (result.name !== stem) {
+    throw new Error(
+      `defines name "${result.name}" but the file is named "${stem}"; rename one so they match`,
+    );
+  }
+  return result;
+}
+
 /**
  * Evaluate one convention model file. `stem` is the file name without extension
- * and must equal `model.type`.
+ * and must equal `model.type`. `objects` supplies `import x from "../objects/<file>"`.
  */
-export function evalModelSource(source: string, stem: string): ModelDef {
-  const body = rewriteDefaultExport(stripAllowedImports(source));
+export function evalModelSource(
+  source: string,
+  stem: string,
+  objects: Readonly<Record<string, ObjectDef>> = {},
+): ModelDef {
+  const taken = takeObjectImports(source);
+  const body = rewriteDefaultExport(stripAllowedImports(taken.source));
   assertNoForbidden(body);
   const wrapped = `(function () {\n${body}\n})()`;
   const js = transpile(wrapped);
-  const sandbox = Object.create(null) as { defineModel: typeof defineModel; field: typeof field };
+  const sandbox = Object.create(null) as Record<string, unknown>;
   sandbox.defineModel = defineModel;
   sandbox.field = field;
+  for (const binding of taken.bindings) {
+    const object = objects[binding.stem];
+    if (!object) throw new Error(`import "../objects/${binding.stem}" was not found`);
+    sandbox[binding.local] = object;
+  }
   Object.freeze(sandbox);
 
   let result: unknown;
@@ -125,6 +208,11 @@ export function evalModelsFromTree(
   files: Array<{ path: string; source: string }>,
   modelsDir = "models",
 ): ModelDef[] {
+  const objects: Record<string, ObjectDef> = {};
+  for (const file of files.filter((f) => isModelSourcePath(f.path, "objects"))) {
+    const stem = stemOf(file.path);
+    objects[stem] = evalObjectSource(file.source, stem);
+  }
   const models: ModelDef[] = [];
   const types = new Set<string>();
   const sorted = files
@@ -132,7 +220,7 @@ export function evalModelsFromTree(
     .sort((a, b) => a.path.localeCompare(b.path));
   for (const file of sorted) {
     const stem = stemOf(file.path);
-    const model = evalModelSource(file.source, stem);
+    const model = evalModelSource(file.source, stem, objects);
     if (types.has(model.type)) {
       throw new Error(`duplicate model type "${model.type}" (${file.path})`);
     }