@wkronmiller/lisa 0.1.0

package/src/spec/validator.ts

@@ -0,0 +1,443 @@
+import { basename } from "path";
+
+import { isRegisteredExtensionKind } from "./extensions/registry";
+import { validateGrammarCrossReferences } from "./grammar";
+import { resolveConfiguredHarnessId } from "./config";
+import type {
+  LoadedSpecWorkspace,
+  ParsedEnvironmentConfig,
+  ParsedSpecConfig,
+  ParsedSpecDocument,
+  ValidationIssue,
+} from "./types";
+import { SPEC_STATUSES, SPEC_STAGES } from "./types";
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function asStringArray(value: unknown): string[] {
+  if (!Array.isArray(value)) {
+    return [];
+  }
+
+  return value.filter((entry): entry is string => typeof entry === "string" && entry.trim().length > 0);
+}
+
+function isStringArray(value: unknown): value is string[] {
+  return Array.isArray(value) && value.every((entry) => typeof entry === "string");
+}
+
+function hasNonEmptyStringArray(value: unknown): boolean {
+  return isStringArray(value) && value.some((entry) => entry.trim().length > 0);
+}
+
+function hasPlaceholderValue(values: string[]): boolean {
+  return values.some((entry) => /^TODO:\s*replace\b/i.test(entry.trim()));
+}
+
+function asStringRecord(value: unknown): Record<string, string> {
+  if (!isRecord(value)) {
+    return {};
+  }
+
+  return Object.fromEntries(
+    Object.entries(value).filter((entry): entry is [string, string] => typeof entry[1] === "string" && entry[1].trim().length > 0),
+  );
+}
+
+function isStringRecord(value: unknown): value is Record<string, string> {
+  return isRecord(value) && Object.values(value).every((entry) => typeof entry === "string");
+}
+
+function issue(path: string, message: string, severity: ValidationIssue["severity"] = "error"): ValidationIssue {
+  return { path, message, severity };
+}
+
+function isSafeRepoRelativePath(value: string): boolean {
+  const normalized = value.replace(/\\/g, "/").trim();
+  if (!normalized || normalized.startsWith("/") || /^[A-Za-z]:\//.test(normalized)) {
+    return false;
+  }
+
+  return !normalized.split("/").some((segment) => segment === ".." || segment.length === 0);
+}
+
+function isReservedLisaPath(value: string): boolean {
+  const normalized = value.replace(/\\/g, "/").trim().replace(/^\.\//, "");
+  return normalized === ".specs"
+    || normalized.startsWith(".specs/")
+    || normalized === ".lisa"
+    || normalized.startsWith(".lisa/");
+}
+
+function normalizeHeading(title: string): string {
+  return title.toLowerCase().replace(/\s+/g, " ").trim();
+}
+
+function hasSection(document: ParsedSpecDocument, expectedTitle: string): boolean {
+  const expected = normalizeHeading(expectedTitle);
+
+  for (const [index, section] of document.sections.entries()) {
+    if (normalizeHeading(section.title) !== expected) {
+      continue;
+    }
+
+    if (section.content.length > 0) {
+      return true;
+    }
+
+    for (let childIndex = index + 1; childIndex < document.sections.length; childIndex += 1) {
+      const childSection = document.sections[childIndex];
+      if (childSection.level <= section.level) {
+        break;
+      }
+
+      if (childSection.content.length > 0) {
+        return true;
+      }
+    }
+  }
+
+  return false;
+}
+
+function getSectionContent(document: ParsedSpecDocument, expectedTitle: string): string | undefined {
+  const expected = normalizeHeading(expectedTitle);
+  return document.sections.find((section) => normalizeHeading(section.title) === expected)?.content.trim();
+}
+
+function isPlaceholderSectionContent(content: string | undefined): boolean {
+  if (!content) {
+    return false;
+  }
+
+  const meaningfulLines = content
+    .split("\n")
+    .map((line) => line.replace(/^[-*]\s*/, "").trim())
+    .filter((line) => line.length > 0);
+
+  return meaningfulLines.length > 0 && meaningfulLines.every((line) => line === "TODO" || line.startsWith("TODO:"));
+}
+
+export function validateSpecDocument(document: ParsedSpecDocument): ValidationIssue[] {
+  const issues: ValidationIssue[] = [];
+  const { frontmatter } = document;
+
+  if (!document.id) {
+    issues.push(issue(document.path, "Spec frontmatter must declare a non-empty `id`."));
+  }
+
+  if (!document.status || !SPEC_STATUSES.includes(document.status as (typeof SPEC_STATUSES)[number])) {
+    issues.push(issue(document.path, "Spec frontmatter must declare `status` as draft, active, or deprecated."));
+  }
+
+  if (typeof frontmatter.kind === "string" && frontmatter.kind !== "benchmark" && !isRegisteredExtensionKind(frontmatter.kind)) {
+    issues.push(issue(document.path, `Spec frontmatter declares unknown extension kind \`${frontmatter.kind}\`.`));
+  }
+
+  if (frontmatter.kind === "benchmark" && document.kind !== "benchmark") {
+    issues.push(issue(document.path, "Benchmark sidecars must use the `.bench.` sidecar naming convention."));
+  }
+
+  if (document.area && document.id && !document.id.startsWith(`${document.area}.`)) {
+    issues.push(issue(document.path, `Spec id \`${document.id}\` must be prefixed with \`${document.area}.\`.`));
+  }
+
+  if (document.kind === "base") {
+    const codePaths = asStringArray(frontmatter.code_paths);
+
+    if (!isStringArray(frontmatter.code_paths)) {
+      issues.push(issue(document.path, "Base specs must declare `code_paths` as an array of strings."));
+    }
+
+    if (codePaths.length === 0) {
+      issues.push(issue(document.path, "Base specs must declare at least one `code_paths` entry."));
+    }
+
+    if (hasPlaceholderValue(codePaths)) {
+      issues.push(issue(document.path, "Replace placeholder values in `code_paths` before using this spec."));
+    }
+
+    const testPaths = asStringArray(frontmatter.test_paths);
+    const testCommands = asStringArray(frontmatter.test_commands);
+
+    if (frontmatter.test_paths !== undefined && !isStringArray(frontmatter.test_paths)) {
+      issues.push(issue(document.path, "`test_paths` must be an array of strings when provided."));
+    }
+
+    if (frontmatter.test_commands !== undefined && !isStringArray(frontmatter.test_commands)) {
+      issues.push(issue(document.path, "`test_commands` must be an array of strings when provided."));
+    }
+
+    if (!hasNonEmptyStringArray(frontmatter.test_paths) && !hasNonEmptyStringArray(frontmatter.test_commands)) {
+      issues.push(issue(document.path, "Base specs must declare `test_paths` or `test_commands`."));
+    }
+
+    if (hasPlaceholderValue(testPaths)) {
+      issues.push(issue(document.path, "Replace placeholder values in `test_paths` before using this spec."));
+    }
+
+    if (hasPlaceholderValue(testCommands)) {
+      issues.push(issue(document.path, "Replace placeholder values in `test_commands` before using this spec."));
+    }
+
+    if (frontmatter.owners !== undefined && !isStringArray(frontmatter.owners)) {
+      issues.push(issue(document.path, "`owners` must be an array of strings when provided."));
+    }
+
+    if (frontmatter.depends_on !== undefined && !isStringArray(frontmatter.depends_on)) {
+      issues.push(issue(document.path, "`depends_on` entries must be strings when provided."));
+    }
+
+    for (const sectionTitle of ["Summary", "Use Cases", "Invariants", "Failure Modes", "Acceptance Criteria", "Out of Scope"]) {
+      if (isPlaceholderSectionContent(getSectionContent(document, sectionTitle))) {
+        issues.push(issue(document.path, `Replace placeholder content in \`${sectionTitle}\` before using this spec.`));
+      }
+    }
+
+    if (document.status === "active") {
+      for (const requiredSection of ["Summary", "Use Cases", "Invariants", "Failure Modes", "Acceptance Criteria", "Out of Scope"]) {
+        if (!hasSection(document, requiredSection)) {
+          issues.push(issue(document.path, `Active base specs must include a non-empty \`${requiredSection}\` section.`));
+        }
+      }
+    }
+  }
+
+  if (document.extensionKind === "benchmark") {
+    if (frontmatter.kind !== "benchmark") {
+      issues.push(issue(document.path, "Benchmark sidecars must declare `kind: benchmark` in frontmatter."));
+    }
+
+    if (typeof frontmatter.extends !== "string" || frontmatter.extends.trim().length === 0) {
+      issues.push(issue(document.path, "Benchmark sidecars must declare a non-empty `extends` spec id."));
+    }
+
+    if (typeof frontmatter.environment !== "string" || frontmatter.environment.trim().length === 0) {
+      issues.push(issue(document.path, "Benchmark sidecars must declare a non-empty `environment`."));
+    }
+
+    if (typeof frontmatter.command !== "string" || frontmatter.command.trim().length === 0) {
+      issues.push(issue(document.path, "Benchmark sidecars must declare a non-empty `command`."));
+    }
+
+    if (frontmatter.required !== undefined && typeof frontmatter.required !== "boolean") {
+      issues.push(issue(document.path, "Benchmark sidecars must declare `required` as a boolean when provided."));
+    }
+
+    if (frontmatter.noise_tolerance_pct !== undefined && typeof frontmatter.noise_tolerance_pct !== "number") {
+      issues.push(issue(document.path, "Benchmark sidecars must declare `noise_tolerance_pct` as a number when provided."));
+    }
+
+    if (frontmatter.metrics !== undefined && !isStringRecord(frontmatter.metrics)) {
+      issues.push(issue(document.path, "Benchmark sidecars must declare `metrics` as an object with string threshold values."));
+    }
+
+    if (Object.keys(asStringRecord(frontmatter.metrics)).length === 0) {
+      issues.push(issue(document.path, "Benchmark sidecars must declare at least one string-valued metric threshold under `metrics`."));
+    }
+  }
+
+  return issues;
+}
+
+export function validateSpecConfig(config: ParsedSpecConfig): ValidationIssue[] {
+  const issues: ValidationIssue[] = [];
+
+  if (!isRecord(config.raw.default_stage_profiles)) {
+    issues.push(issue(config.path, "Spec config must declare `default_stage_profiles` as an object."));
+  }
+
+  if (!isRecord(config.raw.profiles) || Object.keys(config.profiles).length === 0) {
+    issues.push(issue(config.path, "Spec config must declare at least one stage profile under `profiles`."));
+  }
+
+  if (!isRecord(config.raw.harnesses) || Object.keys(config.harnesses).length === 0) {
+    issues.push(issue(config.path, "Spec config must declare at least one harness under `harnesses`."));
+  }
+
+  for (const stage of SPEC_STAGES) {
+    if (!config.defaultStageProfiles[stage]) {
+      issues.push(issue(config.path, `Spec config must map the \`${stage}\` stage under \`default_stage_profiles\`.`));
+    }
+  }
+
+  for (const [profileName, profile] of Object.entries(config.profiles)) {
+    if (!profile.harness) {
+      issues.push(issue(config.path, `Profile \`${profileName}\` must declare a non-empty \`harness\`.`));
+    }
+
+    if (profile.allowEdits === undefined) {
+      issues.push(issue(config.path, `Profile \`${profileName}\` must declare boolean \`allow_edits\`.`));
+    }
+
+    if (profile.model === undefined && profile.raw.model !== undefined && typeof profile.raw.model !== "string") {
+      issues.push(issue(config.path, `Profile \`${profileName}\` must declare \`model\` as a string when provided.`));
+    }
+
+    if (profile.harness && !config.harnesses[resolveConfiguredHarnessId(config, profile.harness)]) {
+      issues.push(issue(config.path, `Profile \`${profileName}\` references unknown harness \`${profile.harness}\`.`));
+    }
+  }
+
+  for (const [stage, profileName] of Object.entries(config.defaultStageProfiles)) {
+    if (!config.profiles[profileName]) {
+      issues.push(issue(config.path, `Stage \`${stage}\` references unknown profile \`${profileName}\`.`));
+    }
+  }
+
+  for (const [harnessName, harness] of Object.entries(config.harnesses)) {
+    if (!harness.command) {
+      issues.push(issue(config.path, `Harness \`${harnessName}\` must declare a non-empty \`command\`.`));
+    }
+
+    if (harness.raw.args !== undefined && !Array.isArray(harness.raw.args)) {
+      issues.push(issue(config.path, `Harness \`${harnessName}\` must declare \`args\` as an array when provided.`));
+    }
+
+    if (Array.isArray(harness.raw.args) && !isStringArray(harness.raw.args)) {
+      issues.push(issue(config.path, `Harness \`${harnessName}\` must declare \`args\` as an array of strings.`));
+    }
+  }
+
+  if (config.raw.agent_guidance !== undefined && !isRecord(config.raw.agent_guidance)) {
+    issues.push(issue(config.path, "Spec config `agent_guidance` must be an object when provided."));
+  }
+
+  if (config.agentGuidance) {
+    if (config.agentGuidance.raw.enabled !== undefined && typeof config.agentGuidance.raw.enabled !== "boolean") {
+      issues.push(issue(config.path, "Spec config `agent_guidance.enabled` must be a boolean when provided."));
+    }
+
+    if (config.agentGuidance.raw.target !== undefined && typeof config.agentGuidance.raw.target !== "string") {
+      issues.push(issue(config.path, "Spec config `agent_guidance.target` must be a string when provided."));
+    }
+
+    if (config.agentGuidance.raw.targets !== undefined && (!Array.isArray(config.agentGuidance.raw.targets) || !config.agentGuidance.raw.targets.every((entry: unknown) => typeof entry === "string"))) {
+      issues.push(issue(config.path, "Spec config `agent_guidance.targets` must be a list of strings when provided."));
+    }
+
+    if (config.agentGuidance.target !== undefined && config.agentGuidance.target.trim().length === 0) {
+      issues.push(issue(config.path, "Spec config `agent_guidance.target` must not be empty when provided."));
+    }
+
+    const allTargets = [
+      ...(config.agentGuidance.target ? [config.agentGuidance.target] : []),
+      ...(config.agentGuidance.targets ?? []),
+    ];
+
+    for (const target of allTargets) {
+      if (!isSafeRepoRelativePath(target)) {
+        issues.push(issue(config.path, `Spec config agent_guidance target \`${target}\` must stay inside the repository using a safe relative path.`));
+      }
+
+      if (isReservedLisaPath(target)) {
+        issues.push(issue(config.path, `Spec config agent_guidance target \`${target}\` must not point inside \`.specs/\` or \`.lisa/\`.`));
+      }
+    }
+
+    const guidanceEnabled = config.agentGuidance.enabled ?? Boolean(config.agentGuidance.target) ?? (config.agentGuidance.targets && config.agentGuidance.targets.length > 0);
+    if (guidanceEnabled && allTargets.length === 0) {
+      issues.push(issue(config.path, "Spec config `agent_guidance` must declare `target` or `targets` when enabled."));
+    }
+  }
+
+  return issues;
+}
+
+export function validateEnvironmentConfig(environment: ParsedEnvironmentConfig): ValidationIssue[] {
+  const issues: ValidationIssue[] = [];
+
+  if (!environment.name) {
+    issues.push(issue(environment.path, "Environment config must declare a non-empty `name`."));
+  }
+
+  if (environment.raw.runtime !== undefined && !isRecord(environment.raw.runtime)) {
+    issues.push(issue(environment.path, "Environment config `runtime` must be an object when provided."));
+  }
+
+  if (environment.raw.resources !== undefined && !isRecord(environment.raw.resources)) {
+    issues.push(issue(environment.path, "Environment config `resources` must be an object when provided."));
+  }
+
+  if (environment.raw.notes !== undefined && typeof environment.raw.notes !== "string") {
+    issues.push(issue(environment.path, "Environment config `notes` must be a string when provided."));
+  }
+
+  const expectedName = basename(environment.path).replace(/\.(yaml|yml)$/i, "");
+  if (environment.name && environment.name !== expectedName) {
+    issues.push(issue(environment.path, `Environment file name should match declared name \`${environment.name}\`.`, "warning"));
+  }
+
+  return issues;
+}
+
+export function validateLoadedSpecWorkspace(
+  workspace: Pick<LoadedSpecWorkspace, "documents" | "environments" | "config"> & { workspacePath?: string },
+): ValidationIssue[] {
+  const issues: ValidationIssue[] = [];
+  const seenSpecIds = new Map<string, string>();
+  const baseSpecIds = new Set<string>();
+  const seenEnvironmentNames = new Map<string, string>();
+
+  for (const document of workspace.documents) {
+    if (document.id) {
+      const previousPath = seenSpecIds.get(document.id);
+      if (previousPath) {
+        issues.push(issue(document.path, `Spec id \`${document.id}\` is duplicated in ${previousPath}.`));
+      } else {
+        seenSpecIds.set(document.id, document.path);
+      }
+    }
+
+    if (document.kind === "base" && document.id) {
+      baseSpecIds.add(document.id);
+    }
+  }
+
+  for (const environment of workspace.environments) {
+    if (!environment.name) {
+      continue;
+    }
+
+    const previousPath = seenEnvironmentNames.get(environment.name);
+    if (previousPath) {
+      issues.push(issue(environment.path, `Environment \`${environment.name}\` is duplicated in ${previousPath}.`));
+    } else {
+      seenEnvironmentNames.set(environment.name, environment.path);
+    }
+  }
+
+  for (const document of workspace.documents) {
+    if (document.extensionKind !== "benchmark") {
+      continue;
+    }
+
+    const extendsId = typeof document.frontmatter.extends === "string" ? document.frontmatter.extends : undefined;
+    const environmentName = typeof document.frontmatter.environment === "string" ? document.frontmatter.environment : undefined;
+
+    if (extendsId && !baseSpecIds.has(extendsId)) {
+      issues.push(issue(document.path, `Benchmark sidecar extends unknown base spec \`${extendsId}\`.`));
+    }
+
+    if (environmentName && !seenEnvironmentNames.has(environmentName)) {
+      issues.push(issue(document.path, `Benchmark sidecar references unknown environment \`${environmentName}\`.`));
+    }
+  }
+
+  if (workspace.config) {
+    for (const stage of SPEC_STAGES) {
+      const profileName = workspace.config.defaultStageProfiles[stage];
+        if (profileName && !workspace.config.profiles[profileName]) {
+          issues.push(issue(workspace.config.path, `Stage \`${stage}\` references unknown profile \`${profileName}\`.`));
+        }
+      }
+    }
+
+  if (workspace.workspacePath) {
+    issues.push(...validateGrammarCrossReferences(workspace.documents, { repoRoot: workspace.workspacePath }).issues);
+  }
+
+  return issues;
+}