@markbrutx/promptbook-core 0.1.0

package/src/fs.ts

@@ -0,0 +1,21 @@
+import type { FsAdapter } from "./types.js";
+
+/**
+ * Default filesystem adapter backed by `node:fs/promises`.
+ *
+ * The import is lazy so that simply importing the core does not pull in any
+ * Node-only module: callers in Deno/Bun (or tests) can inject their own
+ * {@link FsAdapter} and never touch this path. The module is resolved once
+ * per adapter (when `nodeFs()` is called), not on every read.
+ */
+export function nodeFs(): FsAdapter {
+  const fsModule = import("node:fs/promises");
+  return {
+    async readFile(path: string): Promise<string> {
+      return (await fsModule).readFile(path, "utf8");
+    },
+    async readDir(path: string): Promise<string[]> {
+      return (await fsModule).readdir(path);
+    },
+  };
+}

package/src/guards.ts

@@ -0,0 +1,11 @@
+import type { ContextValue } from "./types.js";
+
+/** True for a plain object (a YAML/JSON mapping); excludes arrays and null. */
+export function isMapping(value: unknown): value is Record<string, unknown> {
+  return !!value && typeof value === "object" && !Array.isArray(value);
+}
+
+/** True for a scalar that is a valid {@link ContextValue} (string/number/boolean). */
+export function isContextValue(value: unknown): value is ContextValue {
+  return typeof value === "string" || typeof value === "number" || typeof value === "boolean";
+}

package/src/index.ts

@@ -0,0 +1,84 @@
+export type {
+  Annotation,
+  AnnotationAnchor,
+  AnnotationStatus,
+  AnnotationTarget,
+} from "./annotations.js";
+export {
+  ANNOTATION_QUEUE_DIR,
+  ANNOTATION_QUEUE_FILE,
+  parseInbox,
+  serializeAnnotationLine,
+  serializeInbox,
+} from "./annotations.js";
+export type { SerializeBookOptions } from "./bundle.js";
+export { serializeBook, serializeBookExpression, serializeBookJson } from "./bundle.js";
+export { defaultAssertions } from "./eval/assertions.js";
+export { evaluate } from "./eval/evaluate.js";
+export { loadFixtures } from "./eval/load-fixtures.js";
+export type {
+  Assertion,
+  AssertionFn,
+  AssertionRegistry,
+  AssertionResult,
+  EvalInput,
+  EvalReport,
+  Fixture,
+  FixtureResult,
+  ModelAdapter,
+  ModelRequest,
+  ModelResponse,
+  ModelUsage,
+} from "./eval/types.js";
+export { parseFrontmatter } from "./frontmatter.js";
+export { nodeFs } from "./fs.js";
+export { interpolate } from "./interpolate.js";
+export { lint } from "./lint/lint.js";
+export type { FragmentReference } from "./lint/references.js";
+export { iterateReferences } from "./lint/references.js";
+export type {
+  BannedTokensOptions,
+  DanglingReferenceOptions,
+  DeadRuleOptions,
+  ExampleBalanceOptions,
+  LanguageDirectivePositionOptions,
+  TokenBudgetOptions,
+  UnusedFragmentOptions,
+} from "./lint/rules/index.js";
+export {
+  bannedTokens,
+  type DefaultRulesOptions,
+  danglingReference,
+  deadRule,
+  defaultRules,
+  estimateTokensByChars,
+  exampleBalance,
+  languageDirectivePosition,
+  tokenBudget,
+  unusedFragment,
+} from "./lint/rules/index.js";
+export type { LintFinding, LintInput, LintReport, LintRule, LintScope, Severity } from "./lint/types.js";
+export { loadPrompts } from "./load.js";
+export { resolve } from "./resolve.js";
+export { resolveBook } from "./resolve-book.js";
+export type {
+  AddTrace,
+  CodePrompt,
+  CodePromptSample,
+  Composition,
+  Context,
+  ContextValue,
+  ForbidTrace,
+  Fragment,
+  FsAdapter,
+  PromptBook,
+  ReplaceTrace,
+  ResolveInput,
+  ResolveResult,
+  Rule,
+  RuleAction,
+  RuleTrace,
+  Trace,
+  UnmatchedAxis,
+  When,
+} from "./types.js";

package/src/interpolate.ts

@@ -0,0 +1,27 @@
+import type { Context } from "./types.js";
+
+/** Matches `${path}` placeholders, capturing an optional leading `\` escape. */
+const VAR_RE = /(\\?)\$\{([^}]+)\}/g;
+
+/**
+ * Substitute `${path}` placeholders in `body` using `context`.
+ *
+ * - A missing key resolves to an empty string and triggers `onMissing(key)`.
+ *   Interpolation never throws.
+ * - `\${path}` is an escape and renders the literal `${path}`.
+ * - Lookup is by flat key, matching the flat {@link Context} shape.
+ */
+export function interpolate(body: string, context: Context, onMissing: (key: string) => void): string {
+  return body.replace(VAR_RE, (_match, backslash: string, expr: string) => {
+    if (backslash) {
+      return `\${${expr}}`;
+    }
+    const key = expr.trim();
+    const value = context[key];
+    if (value === undefined) {
+      onMissing(key);
+      return "";
+    }
+    return String(value);
+  });
+}

package/src/lint/lint.ts

@@ -0,0 +1,32 @@
+import { defaultRules } from "./rules/index.js";
+import type { LintFinding, LintInput, LintReport, LintRule } from "./types.js";
+
+/**
+ * Run lint rules over an input and aggregate the findings.
+ *
+ * Book-scope rules always run. Resolved-scope rules run only when
+ * `input.result` is present, so calling `lint({ book })` lints structure only.
+ * The engine has no intelligence of its own: it dispatches by scope and counts
+ * severities. All judgement lives in the rules.
+ */
+export function lint(input: LintInput, rules: LintRule[] = defaultRules()): LintReport {
+  const findings: LintFinding[] = [];
+  for (const rule of rules) {
+    if (rule.scope === "resolved" && input.result === undefined) {
+      continue;
+    }
+    findings.push(...rule.check(input));
+  }
+
+  let errorCount = 0;
+  let warningCount = 0;
+  for (const finding of findings) {
+    if (finding.severity === "error") {
+      errorCount += 1;
+    } else if (finding.severity === "warning") {
+      warningCount += 1;
+    }
+  }
+
+  return { findings, errorCount, warningCount };
+}

package/src/lint/references.ts

@@ -0,0 +1,50 @@
+import type { PromptBook } from "../types.js";
+
+/** A single mention of a fragment id, with enough context to report it. */
+export interface FragmentReference {
+  id: string;
+  composition: string;
+  /** Where in a composition the fragment id is mentioned. */
+  role: "base" | "order" | "add" | "after" | "replace-from" | "replace-to" | "forbid" | "rule-order";
+  /** Index of the rule the reference came from, when it came from a rule. */
+  ruleIndex?: number;
+}
+
+/**
+ * Yield every place a fragment id is referenced across all compositions: base
+ * and explicit order lists, and each rule's add/after/replace/forbid/order.
+ * Shared by the `unused-fragment` and `dangling-reference` rules so both see
+ * the same reference surface.
+ */
+export function* iterateReferences(book: PromptBook): Generator<FragmentReference> {
+  for (const composition of book.compositions.values()) {
+    const composed = composition.name;
+    for (const id of composition.base) {
+      yield { id, composition: composed, role: "base" };
+    }
+    if (composition.order) {
+      for (const id of composition.order) {
+        yield { id, composition: composed, role: "order" };
+      }
+    }
+    for (const rule of composition.rules) {
+      const ruleIndex = rule.index;
+      for (const id of rule.add ?? []) {
+        yield { id, composition: composed, role: "add", ruleIndex };
+      }
+      if (rule.after !== undefined) {
+        yield { id: rule.after, composition: composed, role: "after", ruleIndex };
+      }
+      for (const [from, to] of Object.entries(rule.replace ?? {})) {
+        yield { id: from, composition: composed, role: "replace-from", ruleIndex };
+        yield { id: to, composition: composed, role: "replace-to", ruleIndex };
+      }
+      for (const id of rule.forbid ?? []) {
+        yield { id, composition: composed, role: "forbid", ruleIndex };
+      }
+      for (const id of rule.order ?? []) {
+        yield { id, composition: composed, role: "rule-order", ruleIndex };
+      }
+    }
+  }
+}

package/src/lint/rules/banned-tokens.ts

@@ -0,0 +1,46 @@
+import type { LintFinding, LintRule, Severity } from "../types.js";
+
+export interface BannedTokensOptions {
+  /** Substrings or patterns that must not appear in the assembled text. */
+  tokens?: (string | RegExp)[];
+  severity?: Severity;
+}
+
+// Em-dash: a common stylistic ban, and deliberately generic (not domain-specific).
+const EM_DASH = "\u2014";
+const DEFAULT_BANNED: (string | RegExp)[] = [EM_DASH];
+
+/**
+ * `banned-tokens` (resolved): the assembled text must not contain any of the
+ * configured substrings or patterns. The default bans the em-dash. Patterns
+ * should not use the global flag, since each token is tested once.
+ */
+export function bannedTokens(options: BannedTokensOptions = {}): LintRule {
+  const tokens = options.tokens ?? DEFAULT_BANNED;
+  const severity = options.severity ?? "error";
+  return {
+    id: "banned-tokens",
+    description: "Resolved prompt must not contain banned substrings or patterns.",
+    scope: "resolved",
+    check(input) {
+      const result = input.result;
+      if (result === undefined) {
+        return [];
+      }
+      const text = result.text;
+      const findings: LintFinding[] = [];
+      for (const token of tokens) {
+        const matched = typeof token === "string" ? text.includes(token) : token.test(text);
+        if (matched) {
+          const label = typeof token === "string" ? JSON.stringify(token) : String(token);
+          findings.push({
+            ruleId: "banned-tokens",
+            severity,
+            message: `text contains banned token ${label}`,
+          });
+        }
+      }
+      return findings;
+    },
+  };
+}

package/src/lint/rules/dangling-reference.ts

@@ -0,0 +1,43 @@
+import { iterateReferences } from "../references.js";
+import type { LintFinding, LintRule, Severity } from "../types.js";
+
+export interface DanglingReferenceOptions {
+  severity?: Severity;
+}
+
+/**
+ * `dangling-reference` (book): any reference (base, order, or a rule's
+ * add/replace/forbid/order/after) that points at a fragment id which does not
+ * exist is a hard error — the prompt cannot assemble as written.
+ */
+export function danglingReference(options: DanglingReferenceOptions = {}): LintRule {
+  const severity = options.severity ?? "error";
+  return {
+    id: "dangling-reference",
+    description: "Every referenced fragment id must exist.",
+    scope: "book",
+    check(input) {
+      const findings: LintFinding[] = [];
+      for (const reference of iterateReferences(input.book)) {
+        if (input.book.fragments.has(reference.id)) {
+          continue;
+        }
+        const where =
+          reference.ruleIndex !== undefined
+            ? `rule #${reference.ruleIndex} (${reference.role})`
+            : reference.role;
+        const finding: LintFinding = {
+          ruleId: "dangling-reference",
+          severity,
+          message: `composition "${reference.composition}" references unknown fragment "${reference.id}" via ${where}`,
+          fragmentId: reference.id,
+        };
+        if (reference.ruleIndex !== undefined) {
+          finding.ruleIndex = reference.ruleIndex;
+        }
+        findings.push(finding);
+      }
+      return findings;
+    },
+  };
+}

package/src/lint/rules/dead-rule.ts

@@ -0,0 +1,147 @@
+import { valuesEqual } from "../../resolve-book.js";
+import type { Composition, When } from "../../types.js";
+import type { LintFinding, LintRule, Severity } from "../types.js";
+
+export interface DeadRuleOptions {
+  severity?: Severity;
+}
+
+/**
+ * `dead-rule` (book): catch rules that statically cannot do what they say.
+ * For each rule it flags a `replace` whose source is never present, an `add` of
+ * an already-present id, and an `order` naming an id that can never appear.
+ *
+ * `when` is taken into account: a prior rule only shapes the present-set a later
+ * rule sees if it is *guaranteed to co-fire* — i.e. its `when` is implied by the
+ * later rule's `when`. This keeps mutually-exclusive single-axis swaps (e.g.
+ * `tone=a|b|c` each replacing the same base fragment) from reading as dead,
+ * while still catching a replace whose source was unconditionally consumed by an
+ * earlier rule.
+ *
+ * v0 limitation: still a static check. "A rule that never fires under any
+ * context" (which requires enumerating contexts) remains out of scope.
+ */
+export function deadRule(options: DeadRuleOptions = {}): LintRule {
+  const severity = options.severity ?? "warning";
+  return {
+    id: "dead-rule",
+    description: "Rules should affect fragments that can be present (static check).",
+    scope: "book",
+    check(input) {
+      const findings: LintFinding[] = [];
+      for (const composition of input.book.compositions.values()) {
+        const everPresent = everPresentSet(composition);
+        composition.rules.forEach((rule, position) => {
+          const present = presentSetFor(composition.base, composition.rules, position, rule.when);
+          switch (rule.action) {
+            case "add":
+              for (const id of rule.add ?? []) {
+                if (present.has(id)) {
+                  findings.push({
+                    ruleId: "dead-rule",
+                    severity,
+                    message: `rule #${rule.index} in "${composition.name}" adds "${id}", which is already present`,
+                    fragmentId: id,
+                    ruleIndex: rule.index,
+                  });
+                }
+              }
+              break;
+            case "replace":
+              for (const from of Object.keys(rule.replace ?? {})) {
+                if (!present.has(from)) {
+                  findings.push({
+                    ruleId: "dead-rule",
+                    severity,
+                    message: `rule #${rule.index} in "${composition.name}" replaces "${from}", which is never present`,
+                    fragmentId: from,
+                    ruleIndex: rule.index,
+                  });
+                }
+              }
+              break;
+            case "order":
+              for (const id of rule.order ?? []) {
+                if (!everPresent.has(id)) {
+                  findings.push({
+                    ruleId: "dead-rule",
+                    severity,
+                    message: `rule #${rule.index} in "${composition.name}" orders unknown id "${id}"`,
+                    fragmentId: id,
+                    ruleIndex: rule.index,
+                  });
+                }
+              }
+              break;
+            case "forbid":
+              // `forbid` does not feed the static present-set in v0.
+              break;
+          }
+        });
+      }
+      return findings;
+    },
+  };
+}
+
+/**
+ * The present-set a rule sees: `base` mutated by every earlier rule that is
+ * guaranteed to co-fire (its `when` is implied by this rule's `when`).
+ */
+function presentSetFor(
+  base: readonly string[],
+  rules: Composition["rules"],
+  position: number,
+  currentWhen: When,
+): Set<string> {
+  const present = new Set<string>(base);
+  for (let i = 0; i < position; i++) {
+    const prior = rules[i];
+    if (!prior || !firesWhenever(prior.when, currentWhen)) {
+      continue;
+    }
+    if (prior.action === "add") {
+      for (const id of prior.add ?? []) {
+        present.add(id);
+      }
+    } else if (prior.action === "replace") {
+      for (const [from, to] of Object.entries(prior.replace ?? {})) {
+        if (present.has(from)) {
+          present.delete(from);
+          present.add(to);
+        }
+      }
+    }
+  }
+  return present;
+}
+
+/** Every id that can ever appear: base plus all add ids and all replace targets. */
+function everPresentSet(composition: Composition): Set<string> {
+  const present = new Set<string>(composition.base);
+  for (const rule of composition.rules) {
+    if (rule.action === "add") {
+      for (const id of rule.add ?? []) {
+        present.add(id);
+      }
+    } else if (rule.action === "replace") {
+      for (const to of Object.values(rule.replace ?? {})) {
+        present.add(to);
+      }
+    }
+  }
+  return present;
+}
+
+/** True if `prior` fires in every context where `current` fires — i.e. every
+ * constraint in `prior` is also required (same value) by `current`. An empty
+ * `prior` (unconditional) always co-fires. */
+function firesWhenever(prior: When, current: When): boolean {
+  for (const [key, value] of Object.entries(prior)) {
+    const actual = current[key];
+    if (actual === undefined || !valuesEqual(actual, value)) {
+      return false;
+    }
+  }
+  return true;
+}

package/src/lint/rules/example-balance.ts

@@ -0,0 +1,68 @@
+import type { LintRule, Severity } from "../types.js";
+
+export interface ExampleBalanceOptions {
+  /** Fragment `kind` treated as an example. */
+  kind?: string;
+  /** Tag prefix that groups examples, e.g. "case:". */
+  groupTagPrefix?: string;
+  /** Maximum allowed difference between the largest and smallest group. */
+  maxSkew?: number;
+  severity?: Severity;
+}
+
+/**
+ * `example-balance` (resolved): few-shot examples (fragments of the configured
+ * kind) are grouped by their `groupTagPrefix` tag (e.g. `case:positive`). When
+ * the largest group exceeds the smallest by more than `maxSkew`, attention is
+ * skewed and a finding is raised. Examples without a group tag are ignored, so
+ * the rule only compares groups that actually have examples.
+ */
+export function exampleBalance(options: ExampleBalanceOptions = {}): LintRule {
+  const kind = options.kind ?? "example";
+  const groupTagPrefix = options.groupTagPrefix ?? "case:";
+  const maxSkew = options.maxSkew ?? 1;
+  const severity = options.severity ?? "warning";
+  return {
+    id: "example-balance",
+    description: `Example groups (kind "${kind}", tag "${groupTagPrefix}*") should differ by at most ${maxSkew}.`,
+    scope: "resolved",
+    check(input) {
+      const result = input.result;
+      if (result === undefined) {
+        return [];
+      }
+      const counts = new Map<string, number>();
+      for (const id of result.trace.finalOrder) {
+        const fragment = input.book.fragments.get(id);
+        if (fragment?.kind !== kind) {
+          continue;
+        }
+        for (const tag of fragment.tags ?? []) {
+          if (tag.startsWith(groupTagPrefix)) {
+            counts.set(tag, (counts.get(tag) ?? 0) + 1);
+          }
+        }
+      }
+      if (counts.size < 2) {
+        return [];
+      }
+      const values = [...counts.values()];
+      const max = Math.max(...values);
+      const min = Math.min(...values);
+      if (max - min <= maxSkew) {
+        return [];
+      }
+      const summary = [...counts.entries()]
+        .sort(([a], [b]) => a.localeCompare(b))
+        .map(([group, count]) => `${group}=${count}`)
+        .join(", ");
+      return [
+        {
+          ruleId: "example-balance",
+          severity,
+          message: `example groups are imbalanced (${summary}); skew ${max - min} exceeds ${maxSkew}`,
+        },
+      ];
+    },
+  };
+}

package/src/lint/rules/index.ts

@@ -0,0 +1,47 @@
+import type { LintRule } from "../types.js";
+import { bannedTokens } from "./banned-tokens.js";
+import { danglingReference } from "./dangling-reference.js";
+import { deadRule } from "./dead-rule.js";
+import { exampleBalance } from "./example-balance.js";
+import { languageDirectivePosition } from "./language-directive-position.js";
+import { tokenBudget } from "./token-budget.js";
+import { unusedFragment } from "./unused-fragment.js";
+
+/** Options surfaced through {@link defaultRules}; per-rule factories take more. */
+export interface DefaultRulesOptions {
+  /** Override the token-budget ceiling. */
+  maxTokens?: number;
+  /** Override the banned-tokens list. */
+  bannedTokens?: (string | RegExp)[];
+}
+
+/**
+ * The built-in rule set, instantiated with optional overrides. Callers can
+ * also build their own array and pass it to `lint` to swap the set entirely.
+ */
+export function defaultRules(options: DefaultRulesOptions = {}): LintRule[] {
+  return [
+    tokenBudget(options.maxTokens !== undefined ? { maxTokens: options.maxTokens } : {}),
+    languageDirectivePosition(),
+    exampleBalance(),
+    bannedTokens(options.bannedTokens !== undefined ? { tokens: options.bannedTokens } : {}),
+    unusedFragment(),
+    danglingReference(),
+    deadRule(),
+  ];
+}
+
+export type { BannedTokensOptions } from "./banned-tokens.js";
+export { bannedTokens } from "./banned-tokens.js";
+export type { DanglingReferenceOptions } from "./dangling-reference.js";
+export { danglingReference } from "./dangling-reference.js";
+export type { DeadRuleOptions } from "./dead-rule.js";
+export { deadRule } from "./dead-rule.js";
+export type { ExampleBalanceOptions } from "./example-balance.js";
+export { exampleBalance } from "./example-balance.js";
+export type { LanguageDirectivePositionOptions } from "./language-directive-position.js";
+export { languageDirectivePosition } from "./language-directive-position.js";
+export type { TokenBudgetOptions } from "./token-budget.js";
+export { estimateTokensByChars, tokenBudget } from "./token-budget.js";
+export type { UnusedFragmentOptions } from "./unused-fragment.js";
+export { unusedFragment } from "./unused-fragment.js";

package/src/lint/rules/language-directive-position.ts

@@ -0,0 +1,51 @@
+import type { LintFinding, LintRule, Severity } from "../types.js";
+
+export interface LanguageDirectivePositionOptions {
+  /** Fragment `kind` that must sit at an edge. */
+  kind?: string;
+  /** How many leading/trailing positions count as an edge. */
+  edgeWindow?: number;
+  severity?: Severity;
+}
+
+/**
+ * `language-directive-position` (resolved): fragments of the configured kind
+ * must appear within the first or last `edgeWindow` positions of the final
+ * order. A language directive buried in the middle is easy for a model to
+ * miss, so it is flagged.
+ */
+export function languageDirectivePosition(options: LanguageDirectivePositionOptions = {}): LintRule {
+  const kind = options.kind ?? "language-directive";
+  const edgeWindow = options.edgeWindow ?? 2;
+  const severity = options.severity ?? "warning";
+  return {
+    id: "language-directive-position",
+    description: `Fragments of kind "${kind}" should sit within the first/last ${edgeWindow} positions.`,
+    scope: "resolved",
+    check(input) {
+      const result = input.result;
+      if (result === undefined) {
+        return [];
+      }
+      const order = result.trace.finalOrder;
+      const total = order.length;
+      const findings: LintFinding[] = [];
+      order.forEach((id, position) => {
+        if (input.book.fragments.get(id)?.kind !== kind) {
+          return;
+        }
+        const atStart = position < edgeWindow;
+        const atEnd = position >= total - edgeWindow;
+        if (!atStart && !atEnd) {
+          findings.push({
+            ruleId: "language-directive-position",
+            severity,
+            message: `fragment "${id}" (kind "${kind}") is at position ${position + 1} of ${total}, not within the first/last ${edgeWindow}`,
+            fragmentId: id,
+          });
+        }
+      });
+      return findings;
+    },
+  };
+}

package/src/lint/rules/token-budget.ts

@@ -0,0 +1,50 @@
+import type { LintRule, Severity } from "../types.js";
+
+/** Heuristic token estimate: ~4 characters per token. Not a real tokenizer. */
+export function estimateTokensByChars(text: string): number {
+  return Math.ceil(text.length / 4);
+}
+
+export interface TokenBudgetOptions {
+  /** Maximum estimated tokens before a finding is raised. */
+  maxTokens?: number;
+  /** Inject a real token counter; defaults to the chars/4 heuristic. */
+  estimateTokens?: (text: string) => number;
+  severity?: Severity;
+}
+
+const DEFAULT_MAX_TOKENS = 8000;
+
+/**
+ * `token-budget` (resolved): estimate the assembled text's token count and
+ * report when it exceeds `maxTokens`. The estimate is a deterministic
+ * heuristic (chars/4) so the core stays dependency-free; callers can inject a
+ * precise counter via `estimateTokens`.
+ */
+export function tokenBudget(options: TokenBudgetOptions = {}): LintRule {
+  const maxTokens = options.maxTokens ?? DEFAULT_MAX_TOKENS;
+  const estimate = options.estimateTokens ?? estimateTokensByChars;
+  const severity = options.severity ?? "warning";
+  return {
+    id: "token-budget",
+    description: `Resolved prompt should fit within ~${maxTokens} estimated tokens.`,
+    scope: "resolved",
+    check(input) {
+      const result = input.result;
+      if (result === undefined) {
+        return [];
+      }
+      const tokens = estimate(result.text);
+      if (tokens <= maxTokens) {
+        return [];
+      }
+      return [
+        {
+          ruleId: "token-budget",
+          severity,
+          message: `estimated ${tokens} tokens exceeds budget of ${maxTokens}`,
+        },
+      ];
+    },
+  };
+}