@wkronmiller/lisa 0.1.0

package/src/spec/extensions/benchmark.ts

@@ -0,0 +1,347 @@
+import { relative } from "path";
+
+import type { ParsedEnvironmentConfig, ParsedSpecDocument, SpecCheckEvidence, ValidationIssue } from "../types";
+import type {
+  ExtensionVerificationContext,
+  ExtensionVerificationResult,
+  ExtensionVerificationTarget,
+  ParsedImportBundle,
+  SpecExtension,
+} from "./types";
+
+import { normalizeRelativePath } from "../path-utils";
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function extractStringRecord(value: unknown): Record<string, string> {
+  if (!isRecord(value)) {
+    return {};
+  }
+
+  return Object.fromEntries(
+    Object.entries(value).filter((entry): entry is [string, string] => typeof entry[1] === "string" && entry[1].trim().length > 0),
+  );
+}
+
+function truncateOutput(output: string): string {
+  const trimmed = output.trim();
+  if (trimmed.length <= 240) {
+    return trimmed;
+  }
+
+  return `${trimmed.slice(0, 237)}...`;
+}
+
+function extractJsonPayload(text: string): string | undefined {
+  const trimmed = text.trim();
+  if (!trimmed) {
+    return undefined;
+  }
+
+  if ((trimmed.startsWith("{") && trimmed.endsWith("}")) || (trimmed.startsWith("[") && trimmed.endsWith("]"))) {
+    return trimmed;
+  }
+
+  const fencedMatch = trimmed.match(/```json\s*([\s\S]*?)```/i);
+  if (fencedMatch?.[1]) {
+    return fencedMatch[1].trim();
+  }
+
+  const firstBrace = trimmed.indexOf("{");
+  const lastBrace = trimmed.lastIndexOf("}");
+  if (firstBrace >= 0 && lastBrace > firstBrace) {
+    return trimmed.slice(firstBrace, lastBrace + 1).trim();
+  }
+
+  return undefined;
+}
+
+function parseJsonOutput(output: string): unknown {
+  const payload = extractJsonPayload(output);
+  if (!payload) {
+    throw new Error("Benchmark command did not emit JSON output.");
+  }
+
+  return JSON.parse(payload);
+}
+
+function extractNumberRecord(value: unknown): Record<string, number> {
+  if (!isRecord(value)) {
+    return {};
+  }
+
+  const entries = Object.entries(value)
+    .map(([key, entryValue]) => {
+      if (typeof entryValue === "number" && Number.isFinite(entryValue)) {
+        return [key, entryValue] as [string, number];
+      }
+
+      if (typeof entryValue === "string") {
+        const parsed = Number(entryValue.trim());
+        if (Number.isFinite(parsed)) {
+          return [key, parsed] as [string, number];
+        }
+      }
+
+      return undefined;
+    })
+    .filter((entry): entry is [string, number] => entry !== undefined);
+
+  return Object.fromEntries(entries);
+}
+
+function extractMetrics(payload: unknown, keys: string[]): Record<string, number> {
+  if (!isRecord(payload)) {
+    return {};
+  }
+
+  for (const key of keys) {
+    const value = payload[key];
+    const metrics = extractNumberRecord(value);
+    if (Object.keys(metrics).length > 0) {
+      return metrics;
+    }
+  }
+
+  return extractNumberRecord(payload);
+}
+
+function parseThreshold(threshold: string): { operator: string; value: number } {
+  const match = threshold.trim().match(/^(<=|>=|==|=|<|>)\s*(.+)$/);
+  if (!match) {
+    throw new Error(`Unsupported benchmark threshold: ${threshold}`);
+  }
+
+  const value = Number(match[2]);
+  if (!Number.isFinite(value)) {
+    throw new Error(`Benchmark threshold must compare against a numeric value: ${threshold}`);
+  }
+
+  return {
+    operator: match[1] === "=" ? "==" : match[1],
+    value,
+  };
+}
+
+function compareThreshold(current: number, operator: string, expected: number): boolean {
+  if (operator === "<=") {
+    return current <= expected;
+  }
+  if (operator === ">=") {
+    return current >= expected;
+  }
+  if (operator === "<") {
+    return current < expected;
+  }
+  if (operator === ">") {
+    return current > expected;
+  }
+  return current === expected;
+}
+
+function deriveRegressionDirection(operator: string): "lower" | "higher" | "exact" {
+  if (operator === "<=" || operator === "<") {
+    return "lower";
+  }
+  if (operator === ">=" || operator === ">") {
+    return "higher";
+  }
+  return "exact";
+}
+
+function compareRelativeRegression(
+  current: number,
+  baseline: number,
+  direction: "lower" | "higher" | "exact",
+  tolerancePct: number,
+): boolean {
+  if (direction === "lower") {
+    return current <= baseline * (1 + tolerancePct / 100);
+  }
+
+  if (direction === "higher") {
+    return current >= baseline * (1 - tolerancePct / 100);
+  }
+
+  return Math.abs(current - baseline) <= Math.abs(baseline) * (tolerancePct / 100);
+}
+
+function formatEnvironment(environment: ParsedEnvironmentConfig | undefined): string {
+  if (!environment?.name) {
+    return "(missing environment config)";
+  }
+
+  const runtime = environment.runtime ? Bun.YAML.stringify(environment.runtime).trim().replace(/\n/g, "; ") : "runtime: unspecified";
+  const resources = environment.resources ? Bun.YAML.stringify(environment.resources).trim().replace(/\n/g, "; ") : "resources: unspecified";
+  return `${environment.name} (${runtime}; ${resources})`;
+}
+
+function ensureImportedBenchmarkDraftStatuses(bundle: ParsedImportBundle): ValidationIssue[] {
+  const issues: ValidationIssue[] = [];
+
+  for (const draft of bundle.files) {
+    if (!draft.path.includes(".bench.")) {
+      continue;
+    }
+
+    if (!draft.content.includes("status: draft")) {
+      issues.push({
+        severity: "error",
+        path: draft.path,
+        message: "Imported benchmark sidecars must declare `status: draft`.",
+      });
+    }
+  }
+
+  return issues;
+}
+
+async function verifyBenchmarkSidecars(
+  target: ExtensionVerificationTarget,
+  context: ExtensionVerificationContext,
+): Promise<ExtensionVerificationResult | undefined> {
+  if (target.sidecars.length === 0) {
+    return undefined;
+  }
+
+  const evidence: SpecCheckEvidence[] = [];
+  const issues: string[] = [];
+
+  for (const sidecar of target.sidecars) {
+    const sidecarPath = normalizeRelativePath(relative(context.workspaceRoot, sidecar.path));
+    const required = sidecar.frontmatter.required === true;
+    const environmentName = typeof sidecar.frontmatter.environment === "string" ? sidecar.frontmatter.environment : undefined;
+    const environment = context.workspace.environments.find((entry) => entry.name === environmentName);
+    evidence.push({
+      type: "config",
+      detail: `Benchmark sidecar ${sidecar.id ?? sidecarPath} uses environment ${formatEnvironment(environment)}.`,
+    });
+
+    const command = typeof sidecar.frontmatter.command === "string" ? sidecar.frontmatter.command : undefined;
+    if (!command) {
+      if (required) {
+        issues.push(`Required benchmark sidecar ${sidecar.id ?? sidecarPath} does not declare a runnable command.`);
+      } else {
+        evidence.push({ type: "benchmark", detail: `Advisory benchmark sidecar ${sidecar.id ?? sidecarPath} has no runnable command.` });
+      }
+      continue;
+    }
+
+    const commandResult = context.runCommand(command, context.workspaceRoot);
+    if (commandResult.exitCode !== 0) {
+      const detail = `Benchmark command failed for ${sidecar.id ?? sidecarPath}: ${command}${commandResult.stderr || commandResult.stdout ? ` (${truncateOutput(commandResult.stderr || commandResult.stdout)})` : ""}`;
+      if (required) {
+        issues.push(detail);
+      } else {
+        evidence.push({ type: "benchmark", detail: `Advisory ${detail}` });
+      }
+      continue;
+    }
+
+    let payload: unknown;
+    try {
+      payload = parseJsonOutput(commandResult.stdout || commandResult.stderr);
+    } catch (error) {
+      const detail = error instanceof Error ? error.message : String(error);
+      if (required) {
+        issues.push(`Required benchmark sidecar ${sidecar.id ?? sidecarPath} produced invalid JSON output: ${detail}`);
+      } else {
+        evidence.push({ type: "benchmark", detail: `Advisory benchmark sidecar ${sidecar.id ?? sidecarPath} produced invalid JSON output: ${detail}` });
+      }
+      continue;
+    }
+
+    const metrics = extractMetrics(payload, ["metrics", "results", "current_metrics", "current"]);
+    const baselineMetrics = extractMetrics(payload, ["baseline_metrics", "baseline", "baselineResults"]);
+    const thresholds = extractStringRecord(sidecar.frontmatter.metrics);
+    const baselineLabel = typeof sidecar.frontmatter.baseline === "string" ? sidecar.frontmatter.baseline : undefined;
+    const tolerancePct = typeof sidecar.frontmatter.noise_tolerance_pct === "number" ? sidecar.frontmatter.noise_tolerance_pct : 0;
+    const failures: string[] = [];
+
+    for (const [metricName, thresholdText] of Object.entries(thresholds)) {
+      const currentValue = metrics[metricName];
+      if (currentValue === undefined) {
+        failures.push(`missing metric \`${metricName}\``);
+        continue;
+      }
+
+      let operator: string;
+      let value: number;
+      try {
+        ({ operator, value } = parseThreshold(thresholdText));
+      } catch (error) {
+        failures.push(error instanceof Error ? error.message : String(error));
+        continue;
+      }
+      if (!compareThreshold(currentValue, operator, value)) {
+        failures.push(`metric \`${metricName}\` = ${currentValue} violated ${thresholdText}`);
+      }
+
+      if (baselineLabel) {
+        const baselineValue = baselineMetrics[metricName];
+        if (baselineValue === undefined) {
+          failures.push(`baseline metric \`${metricName}\` missing for ${baselineLabel}`);
+          continue;
+        }
+
+        const regressionDirection = deriveRegressionDirection(operator);
+        if (!compareRelativeRegression(currentValue, baselineValue, regressionDirection, tolerancePct)) {
+          failures.push(
+            `metric \`${metricName}\` regressed versus baseline ${baselineLabel}: current ${currentValue}, baseline ${baselineValue}, tolerance ${tolerancePct}%`,
+          );
+        }
+      }
+    }
+
+    if (failures.length > 0) {
+      const detail = `${sidecar.id ?? sidecarPath} failed benchmark enforcement: ${failures.join("; ")}`;
+      if (required) {
+        issues.push(detail);
+      } else {
+        evidence.push({ type: "benchmark", detail: `Advisory ${detail}` });
+      }
+      continue;
+    }
+
+    evidence.push({
+      type: "benchmark",
+      detail: `Passed benchmark sidecar ${sidecar.id ?? sidecarPath} via \`${command}\`${baselineLabel ? ` against ${baselineLabel}` : ""}.`,
+    });
+  }
+
+  if (issues.length > 0) {
+    return {
+      status: "FAIL",
+      summary: "Required benchmark checks failed.",
+      evidence,
+      issues,
+    };
+  }
+
+  return {
+    evidence,
+    issues: [],
+  };
+}
+
+export function createBenchmarkExtension(): SpecExtension {
+  return {
+    kind: "benchmark",
+    sidecarMarkers: [".bench."],
+    buildImportPromptGuidance(): string[] {
+      return [
+        "If you find benchmark scripts, perf tests, or documented latency/throughput budgets, draft `.bench.<environment>.md` sidecars with `status: draft`.",
+        "When a benchmark sidecar references a new environment, also draft `.specs/environments/<environment>.yaml`.",
+        "Benchmark commands must emit JSON and should stay grounded in existing scripts or test helpers.",
+      ];
+    },
+    validateImportBundle(bundle: ParsedImportBundle): ValidationIssue[] {
+      return ensureImportedBenchmarkDraftStatuses(bundle);
+    },
+    async verify(target: ExtensionVerificationTarget, context: ExtensionVerificationContext): Promise<ExtensionVerificationResult | undefined> {
+      return verifyBenchmarkSidecars(target, context);
+    },
+  };
+}

package/src/spec/extensions/registry.ts

@@ -0,0 +1,59 @@
+import type { LoadedSpecWorkspace, ParsedSpecDocument, ValidationIssue } from "../types";
+import { createBenchmarkExtension } from "./benchmark";
+import type {
+  ExtensionImportContext,
+  ExtensionVerificationContext,
+  ExtensionVerificationResult,
+  ExtensionVerificationTarget,
+  ParsedImportBundle,
+  SpecExtension,
+} from "./types";
+
+const EXTENSIONS: SpecExtension[] = [createBenchmarkExtension()];
+
+export function getSpecExtensions(): SpecExtension[] {
+  return [...EXTENSIONS];
+}
+
+export function isRegisteredExtensionKind(kind: string): boolean {
+  return getSpecExtensions().some((extension) => extension.kind === kind);
+}
+
+export function resolveExtensionKindForPath(path: string): string | undefined {
+  return getSpecExtensions().find((extension) => (extension.sidecarMarkers ?? []).some((marker) => path.includes(marker)))?.kind;
+}
+
+export function buildImportPromptGuidance(context: ExtensionImportContext): string[] {
+  return getSpecExtensions().flatMap((extension) => extension.buildImportPromptGuidance?.(context) ?? []);
+}
+
+export function validateImportBundleWithExtensions(bundle: ParsedImportBundle, workspace: LoadedSpecWorkspace): ValidationIssue[] {
+  return getSpecExtensions().flatMap((extension) => extension.validateImportBundle?.(bundle, workspace) ?? []);
+}
+
+export async function verifySpecExtensions(
+  target: ExtensionVerificationTarget,
+  context: ExtensionVerificationContext,
+): Promise<ExtensionVerificationResult[]> {
+  const results: ExtensionVerificationResult[] = [];
+
+  for (const extension of getSpecExtensions()) {
+    const matchingSidecars = target.sidecars.filter((document) => document.extensionKind === extension.kind || document.kind === extension.kind);
+    if (matchingSidecars.length === 0) {
+      continue;
+    }
+
+    const result = await extension.verify?.({ ...target, sidecars: matchingSidecars }, context);
+    if (result) {
+      results.push(result);
+    }
+  }
+
+  return results;
+}
+
+export function getExtensionSidecarsForSpec(workspace: LoadedSpecWorkspace, specId: string): ParsedSpecDocument[] {
+  return workspace.documents
+    .filter((document) => document.kind !== "base" && document.frontmatter.extends === specId && document.status === "active")
+    .sort((left, right) => (left.id ?? left.path).localeCompare(right.id ?? right.path));
+}

package/src/spec/extensions/types.ts

@@ -0,0 +1,56 @@
+import type {
+  LoadedSpecWorkspace,
+  ParsedSpecDocument,
+  SpecCheckEvidence,
+  SpecCheckStatus,
+  SpecImportUncertainty,
+  ValidationIssue,
+} from "../types";
+
+export interface ExtensionCommandExecutionResult {
+  exitCode: number;
+  stdout: string;
+  stderr: string;
+}
+
+export interface ExtensionVerificationTarget {
+  specId: string;
+  path: string;
+  document: ParsedSpecDocument;
+  sidecars: ParsedSpecDocument[];
+}
+
+export interface ExtensionVerificationContext {
+  workspaceRoot: string;
+  workspace: LoadedSpecWorkspace;
+  runCommand(command: string, cwd: string): ExtensionCommandExecutionResult;
+}
+
+export interface ExtensionVerificationResult {
+  status?: SpecCheckStatus;
+  summary?: string;
+  evidence: SpecCheckEvidence[];
+  issues: string[];
+}
+
+export interface ExtensionImportContext {
+  workspace: LoadedSpecWorkspace;
+  selectedPaths: string[];
+}
+
+export interface ParsedImportBundle {
+  files: Array<{
+    path: string;
+    content: string;
+  }>;
+  notes: string[];
+  uncertainties: SpecImportUncertainty[];
+}
+
+export interface SpecExtension {
+  kind: string;
+  sidecarMarkers?: string[];
+  buildImportPromptGuidance?(context: ExtensionImportContext): string[];
+  validateImportBundle?(bundle: ParsedImportBundle, workspace: LoadedSpecWorkspace): ValidationIssue[];
+  verify?(target: ExtensionVerificationTarget, context: ExtensionVerificationContext): Promise<ExtensionVerificationResult | undefined>;
+}

package/src/spec/grammar/index.ts

@@ -0,0 +1,14 @@
+export type {
+  StatementKind,
+  ConstraintStrength,
+  RiskSeverity,
+  FileRef,
+  FieldLine,
+  ParsedStatement,
+  GrammarWarning,
+  GrammarParseResult,
+} from "./types";
+
+export { parseGrammarStatements } from "./parser";
+export { validateGrammarCrossReferences } from "./validator";
+export type { GrammarValidationOptions, GrammarValidationResult } from "./validator";