agent-contracts 0.11.5 → 0.11.7

package/README.md

@@ -309,6 +309,7 @@ Design regressions become testable.
 * **YAML safety linting** for reserved word collision detection across YAML 1.1/1.2
 * **`x-extensions` declarations** for documenting project-specific custom extension fields
 * **`resolve --expand-defaults`** to materialize all Zod schema defaults in output
+* **DSL completeness scoring** with 7 dimensions, text/JSON output, and `--threshold` CI gate
 * **JSON Schema for editor support and external tooling**
 * **CI-friendly workflow checks**
 
@@ -843,6 +844,7 @@ npx agent-contracts
 | `agent-contracts validate [path]` | Validate schema and references                         |
 | `agent-contracts lint [path]`     | Run semantic lint                                      |
 | `agent-contracts render`          | Render outputs from config                             |
+| `agent-contracts score [path]`    | Calculate DSL completeness score                       |
 | `agent-contracts generate guardrails` | Generate guardrail artifacts from bindings       |
 | `agent-contracts check`           | Run resolve → validate → lint → render --check         |
 
@@ -857,6 +859,26 @@ If `-c` / `--config` is specified, the DSL path from the config file is used.
 | `--expand-defaults` | Expand all Zod default values in output. Fields like `required_validations: []`, `tags: []`, and `can_read_artifacts: []` are written explicitly instead of being silently applied by schema defaults. |
 | `-c, --config <path>` | Path to `agent-contracts.config.yaml` |
 
+#### `score` options
+
+| Option | Description |
+|--------|-------------|
+| `--format <text\|json>` | Output format (default: `text`) |
+| `--threshold <number>` | Minimum score; exit 1 if below (for CI gates) |
+| `-c, --config <path>` | Path to `agent-contracts.config.yaml` |
+
+The score command evaluates 7 dimensions:
+
+| Dimension | What it measures | Weight |
+|-----------|-----------------|--------|
+| Artifact validation coverage | % of artifacts with non-empty `required_validations` | High |
+| Task validation coverage | % of tasks with at least one entry in `validations` | High |
+| Guardrail policy coverage | % of guardrails referenced by at least one policy rule | Medium |
+| Workflow validation integration | % of blocking validations referenced in workflow steps or tasks | High |
+| Schema completeness | % of optional fields filled (description, rationale, trigger, etc.) | Low |
+| Cross-reference bidirectionality | % of agent↔artifact, agent↔tool refs that are reciprocated | Medium |
+| Guardrail scope resolution | % of guardrail scope entries that resolve to existing entities | Medium |
+
 ### Common usage
 
 ````bash
@@ -864,6 +886,9 @@ agent-contracts resolve
 agent-contracts resolve --expand-defaults --format json
 agent-contracts validate
 agent-contracts lint --strict
+agent-contracts score
+agent-contracts score -c agent-contracts.config.yaml --threshold 70
+agent-contracts score --format json
 agent-contracts render -c agent-contracts.config.yaml
 agent-contracts render -c agent-contracts.config.yaml --check
 agent-contracts check -c agent-contracts.config.yaml --strict
@@ -1234,7 +1259,9 @@ Checks:
 Checks:
 
 * bidirectional consistency
-* validation coverage
+* validation coverage — warns when artifacts lack validations or have empty `required_validations` (fails under `--strict`)
+* artifact-required-validation wiring — verifies every entry in `artifact.required_validations` exists, targets the correct artifact, and is referenced in a workflow step or task
+* task-output-validation completeness — checks that tasks producing artifacts (via `execution_steps.produces_artifact` or agent `can_write_artifacts`) cover those artifacts' `required_validations`
 * workflow graph completeness
 * merge integrity
 * read-only write violations
@@ -1244,6 +1271,20 @@ Checks:
 * YAML safety — warns when YAML 1.1 reserved words (`on`, `yes`, `no`, `true`, `false`, etc.) are used in positions where they may be misinterpreted by non-1.2 parsers
 * naming/style issues through Spectral rules
 
+#### `--strict` mode
+
+When `--strict` is passed to `lint` or `check`, warnings are treated as failures (exit code 1). This is particularly relevant for artifact-centric validation rules — empty `required_validations`, orphaned validation wiring, and incomplete task coverage are all warnings that become blocking under `--strict`.
+
+### Completeness scoring
+
+`agent-contracts score` provides a quantitative assessment of the DSL's completeness. While `validate` checks structural correctness (pass/fail) and `lint` checks semantic quality (warnings/errors), `score` produces a **numeric metric** (0–100) covering validation coverage, schema completeness, cross-reference consistency, and more.
+
+Use `--threshold` in CI to enforce a minimum quality bar:
+
+````bash
+agent-contracts score -c config.yaml --threshold 70
+````
+
 ---
 
 ## Best used with runtime frameworks

package/dist/cli.js

@@ -50428,7 +50428,7 @@ var require_dist13 = __commonJS({
 import { readFileSync } from "fs";
 import { fileURLToPath } from "url";
 import { resolve as resolve7, dirname as dirname7 } from "path";
-import { Command as Command7 } from "commander";
+import { Command as Command8 } from "commander";
 
 // src/cli/commands/resolve.ts
 import { Command } from "commander";
@@ -52288,6 +52288,7 @@ var validationCoverageRule = {
       }
       validationsByArtifact.get(val.target_artifact).add(val.kind);
     }
+    const strictArtifactTypes = ["code", "config", "schema"];
     for (const [artId, art] of Object.entries(dsl.artifacts)) {
       const kinds = validationsByArtifact.get(artId);
       if (!kinds || kinds.size === 0) {
@@ -52299,8 +52300,7 @@ var validationCoverageRule = {
         });
         continue;
       }
-      const mechanicalTypes = ["code", "config", "schema"];
-      if (mechanicalTypes.includes(art.type) && !kinds.has("mechanical") && !kinds.has("schema")) {
+      if (strictArtifactTypes.includes(art.type) && !kinds.has("mechanical") && !kinds.has("schema")) {
         diagnostics.push({
           ruleId: "validation-coverage",
           severity: "warning",
@@ -52309,6 +52309,16 @@ var validationCoverageRule = {
         });
       }
     }
+    for (const [artId, art] of Object.entries(dsl.artifacts)) {
+      if (art.required_validations.length === 0) {
+        diagnostics.push({
+          ruleId: "validation-coverage",
+          severity: "warning",
+          path: `artifacts.${artId}`,
+          message: `Artifact "${artId}" has empty required_validations`
+        });
+      }
+    }
     const referencedValidations = /* @__PURE__ */ new Set();
     for (const wf of Object.values(dsl.workflow)) {
       for (const step of wf.steps) {
@@ -52695,6 +52705,101 @@ var yamlReservedKeySafetyRule = {
   }
 };
 
+// src/linter/rules/artifact-required-validation-wiring.ts
+var artifactRequiredValidationWiringRule = {
+  id: "artifact-required-validation-wiring",
+  description: "Every validation listed in artifact.required_validations must exist, target the artifact, and be referenced in a workflow step or task.",
+  run(dsl) {
+    const diagnostics = [];
+    const referencedValidations = /* @__PURE__ */ new Set();
+    for (const wf of Object.values(dsl.workflow)) {
+      for (const step of wf.steps) {
+        if (step.type === "validation") {
+          referencedValidations.add(step.validation);
+        }
+      }
+    }
+    for (const task of Object.values(dsl.tasks)) {
+      for (const valId of task.validations) {
+        referencedValidations.add(valId);
+      }
+    }
+    for (const [artId, art] of Object.entries(dsl.artifacts)) {
+      for (const reqValId of art.required_validations) {
+        const val = dsl.validations[reqValId];
+        if (!val) {
+          diagnostics.push({
+            ruleId: "artifact-required-validation-wiring",
+            severity: "error",
+            path: `artifacts.${artId}`,
+            message: `required_validation "${reqValId}" does not exist in validations`
+          });
+          continue;
+        }
+        if (val.target_artifact !== artId) {
+          diagnostics.push({
+            ruleId: "artifact-required-validation-wiring",
+            severity: "error",
+            path: `artifacts.${artId}`,
+            message: `required_validation "${reqValId}" has target_artifact "${val.target_artifact}" instead of "${artId}"`
+          });
+        }
+        if (!referencedValidations.has(reqValId)) {
+          diagnostics.push({
+            ruleId: "artifact-required-validation-wiring",
+            severity: "warning",
+            path: `artifacts.${artId}`,
+            message: `required_validation "${reqValId}" is defined but not referenced in any workflow step or task`
+          });
+        }
+      }
+    }
+    return diagnostics;
+  }
+};
+
+// src/linter/rules/task-output-validation-completeness.ts
+var taskOutputValidationCompletenessRule = {
+  id: "task-output-validation-completeness",
+  description: "Tasks producing artifacts should cover those artifacts' required_validations in their validations list.",
+  run(dsl) {
+    const diagnostics = [];
+    for (const [taskId, task] of Object.entries(dsl.tasks)) {
+      const producedArtifacts = /* @__PURE__ */ new Set();
+      if (task.execution_steps) {
+        for (const step of task.execution_steps) {
+          if (step.produces_artifact) {
+            producedArtifacts.add(step.produces_artifact);
+          }
+        }
+      }
+      const agent = dsl.agents[task.target_agent];
+      if (agent) {
+        for (const artId of agent.can_write_artifacts) {
+          producedArtifacts.add(artId);
+        }
+      }
+      const taskValidations = new Set(task.validations);
+      for (const artId of producedArtifacts) {
+        const art = dsl.artifacts[artId];
+        if (!art || art.required_validations.length === 0) continue;
+        const missing = art.required_validations.filter(
+          (v2) => !taskValidations.has(v2)
+        );
+        if (missing.length > 0) {
+          diagnostics.push({
+            ruleId: "task-output-validation-completeness",
+            severity: "warning",
+            path: `tasks.${taskId}`,
+            message: `Task produces artifact "${artId}" which requires validations [${missing.join(", ")}] but task.validations does not include them`
+          });
+        }
+      }
+    }
+    return diagnostics;
+  }
+};
+
 // src/linter/linter.ts
 var builtinRules = [
   validationCoverageRule,
@@ -52704,7 +52809,9 @@ var builtinRules = [
   artifactOwnershipRule,
   toolCommandsRule,
   guardrailPolicyCoverageRule,
-  yamlReservedKeySafetyRule
+  yamlReservedKeySafetyRule,
+  artifactRequiredValidationWiringRule,
+  taskOutputValidationCompletenessRule
 ];
 function lint(dsl, rules = builtinRules) {
   const diagnostics = [];
@@ -55076,10 +55183,347 @@ var generateGuardrailsCommand = new Command6("generate").description("Generate g
   }
 );
 
+// src/cli/commands/score.ts
+import { Command as Command7 } from "commander";
+
+// src/scorer/scorer.ts
+function pct(n3, d2) {
+  return d2 === 0 ? 100 : Math.round(n3 / d2 * 100);
+}
+function artifactValidationCoverage(dsl) {
+  const entries = Object.entries(dsl.artifacts);
+  const total = entries.length;
+  const covered = entries.filter(
+    ([, a3]) => a3.required_validations.length > 0
+  ).length;
+  const missing = entries.filter(([, a3]) => a3.required_validations.length === 0).map(([id]) => id);
+  return {
+    id: "artifact-validation-coverage",
+    label: "Artifact validation coverage",
+    score: covered,
+    total,
+    percent: pct(covered, total),
+    weight: 3,
+    recommendations: missing.length > 0 ? [`${missing.length} artifacts missing required_validations: ${missing.join(", ")}`] : []
+  };
+}
+function taskValidationCoverage(dsl) {
+  const entries = Object.entries(dsl.tasks);
+  const total = entries.length;
+  const covered = entries.filter(([, t3]) => t3.validations.length > 0).length;
+  const missing = entries.filter(([, t3]) => t3.validations.length === 0).map(([id]) => id);
+  return {
+    id: "task-validation-coverage",
+    label: "Task validation coverage",
+    score: covered,
+    total,
+    percent: pct(covered, total),
+    weight: 3,
+    recommendations: missing.length > 0 ? [`${missing.length} tasks missing validations: ${missing.join(", ")}`] : []
+  };
+}
+function guardrailPolicyCoverage(dsl) {
+  const guardrailIds = Object.keys(dsl.guardrails);
+  const total = guardrailIds.length;
+  const referenced = /* @__PURE__ */ new Set();
+  for (const policy of Object.values(dsl.guardrail_policies)) {
+    for (const rule of policy.rules) {
+      referenced.add(rule.guardrail);
+    }
+  }
+  const covered = guardrailIds.filter((id) => referenced.has(id)).length;
+  const missing = guardrailIds.filter((id) => !referenced.has(id));
+  return {
+    id: "guardrail-policy-coverage",
+    label: "Guardrail policy coverage",
+    score: covered,
+    total,
+    percent: pct(covered, total),
+    weight: 2,
+    recommendations: missing.length > 0 ? [`${missing.length} guardrails not in any policy: ${missing.join(", ")}`] : []
+  };
+}
+function workflowValidationIntegration(dsl) {
+  const blockingValidations = Object.entries(dsl.validations).filter(
+    ([, v2]) => v2.blocking
+  );
+  const total = blockingValidations.length;
+  const referenced = /* @__PURE__ */ new Set();
+  for (const wf of Object.values(dsl.workflow)) {
+    for (const step of wf.steps) {
+      if (step.type === "validation") {
+        referenced.add(step.validation);
+      }
+    }
+  }
+  for (const task of Object.values(dsl.tasks)) {
+    for (const valId of task.validations) {
+      referenced.add(valId);
+    }
+  }
+  const covered = blockingValidations.filter(
+    ([id]) => referenced.has(id)
+  ).length;
+  const missing = blockingValidations.filter(([id]) => !referenced.has(id)).map(([id]) => id);
+  return {
+    id: "workflow-validation-integration",
+    label: "Workflow validation integration",
+    score: covered,
+    total,
+    percent: pct(covered, total),
+    weight: 3,
+    recommendations: missing.length > 0 ? [`${missing.length} blocking validations not wired: ${missing.join(", ")}`] : []
+  };
+}
+var OPTIONAL_ENTITY_FIELDS = {
+  agents: {
+    section: "agents",
+    fields: ["responsibilities", "constraints", "rules"]
+  },
+  tasks: {
+    section: "tasks",
+    fields: ["description", "responsibilities", "execution_steps", "completion_criteria"]
+  },
+  artifacts: {
+    section: "artifacts",
+    fields: ["description", "visibility"]
+  },
+  tools: {
+    section: "tools",
+    fields: ["description", "commands"]
+  },
+  workflow: {
+    section: "workflow",
+    fields: ["description", "trigger", "entry_conditions"]
+  },
+  guardrails: {
+    section: "guardrails",
+    fields: ["rationale", "tags"]
+  }
+};
+function hasNonEmpty(obj, field) {
+  const val = obj[field];
+  if (val === void 0 || val === null) return false;
+  if (typeof val === "string") return val.length > 0;
+  if (Array.isArray(val)) return val.length > 0;
+  return true;
+}
+function schemaCompleteness(dsl) {
+  let totalSlots = 0;
+  let filledSlots = 0;
+  const lowSections = [];
+  for (const [sectionKey, meta] of Object.entries(OPTIONAL_ENTITY_FIELDS)) {
+    const entities = dsl[meta.section];
+    if (!entities) continue;
+    let sectionTotal = 0;
+    let sectionFilled = 0;
+    for (const entity of Object.values(entities)) {
+      for (const field of meta.fields) {
+        sectionTotal++;
+        totalSlots++;
+        if (hasNonEmpty(entity, field)) {
+          sectionFilled++;
+          filledSlots++;
+        }
+      }
+    }
+    if (sectionTotal > 0 && pct(sectionFilled, sectionTotal) < 50) {
+      lowSections.push(sectionKey);
+    }
+  }
+  return {
+    id: "schema-completeness",
+    label: "Schema completeness",
+    score: filledSlots,
+    total: totalSlots,
+    percent: pct(filledSlots, totalSlots),
+    weight: 1,
+    recommendations: lowSections.length > 0 ? [`Low optional field coverage in: ${lowSections.join(", ")}`] : []
+  };
+}
+function crossReferenceBidirectionality(dsl) {
+  let totalChecks = 0;
+  let passedChecks = 0;
+  const issues = [];
+  for (const [agentId, agent] of Object.entries(dsl.agents)) {
+    for (const toolId of agent.can_execute_tools) {
+      totalChecks++;
+      const tool = dsl.tools[toolId];
+      if (tool && tool.invokable_by.includes(agentId)) {
+        passedChecks++;
+      } else {
+        issues.push(`agent ${agentId} \u2192 tool ${toolId}`);
+      }
+    }
+    for (const artId of agent.can_write_artifacts) {
+      totalChecks++;
+      const art = dsl.artifacts[artId];
+      if (art && (art.producers.includes(agentId) || art.editors.includes(agentId))) {
+        passedChecks++;
+      } else {
+        issues.push(`agent ${agentId} \u2192 artifact ${artId} (write)`);
+      }
+    }
+    for (const artId of agent.can_read_artifacts) {
+      totalChecks++;
+      const art = dsl.artifacts[artId];
+      if (art && (art.consumers.includes(agentId) || art.producers.includes(agentId) || art.editors.includes(agentId))) {
+        passedChecks++;
+      } else {
+        issues.push(`agent ${agentId} \u2192 artifact ${artId} (read)`);
+      }
+    }
+  }
+  for (const [toolId, tool] of Object.entries(dsl.tools)) {
+    for (const agentId of tool.invokable_by) {
+      totalChecks++;
+      const agent = dsl.agents[agentId];
+      if (agent && agent.can_execute_tools.includes(toolId)) {
+        passedChecks++;
+      } else {
+        issues.push(`tool ${toolId} \u2192 agent ${agentId}`);
+      }
+    }
+  }
+  return {
+    id: "cross-reference-bidirectionality",
+    label: "Cross-reference bidirectionality",
+    score: passedChecks,
+    total: totalChecks,
+    percent: pct(passedChecks, totalChecks),
+    weight: 2,
+    recommendations: issues.length > 0 ? [`${issues.length} unreciprocated cross-references: ${issues.slice(0, 5).join(", ")}${issues.length > 5 ? `, ... (${issues.length - 5} more)` : ""}`] : []
+  };
+}
+function guardrailScopeResolution(dsl) {
+  let totalRefs = 0;
+  let resolvedRefs = 0;
+  const unresolved = [];
+  const sectionMap = {
+    agents: dsl.agents,
+    tasks: dsl.tasks,
+    tools: dsl.tools,
+    artifacts: dsl.artifacts,
+    workflows: dsl.workflow
+  };
+  for (const [guardrailId, guardrail] of Object.entries(dsl.guardrails)) {
+    const scope = guardrail.scope;
+    for (const [scopeKey, entityIds] of Object.entries(scope)) {
+      if (!Array.isArray(entityIds)) continue;
+      const section = sectionMap[scopeKey];
+      if (!section) continue;
+      for (const entityId of entityIds) {
+        totalRefs++;
+        if (section[entityId]) {
+          resolvedRefs++;
+        } else {
+          unresolved.push(`${guardrailId}.scope.${scopeKey}: ${entityId}`);
+        }
+      }
+    }
+  }
+  return {
+    id: "guardrail-scope-resolution",
+    label: "Guardrail scope resolution",
+    score: resolvedRefs,
+    total: totalRefs,
+    percent: pct(resolvedRefs, totalRefs),
+    weight: 2,
+    recommendations: unresolved.length > 0 ? [`${unresolved.length} unresolved scope refs: ${unresolved.join(", ")}`] : []
+  };
+}
+function score(dsl) {
+  const dimensions = [
+    artifactValidationCoverage(dsl),
+    taskValidationCoverage(dsl),
+    guardrailPolicyCoverage(dsl),
+    workflowValidationIntegration(dsl),
+    schemaCompleteness(dsl),
+    crossReferenceBidirectionality(dsl),
+    guardrailScopeResolution(dsl)
+  ];
+  const totalWeight = dimensions.reduce((s2, d2) => s2 + d2.weight, 0);
+  const weightedSum = dimensions.reduce(
+    (s2, d2) => s2 + d2.percent * d2.weight,
+    0
+  );
+  const overall = totalWeight === 0 ? 100 : Math.round(weightedSum / totalWeight);
+  return { overall, dimensions };
+}
+
+// src/cli/commands/score.ts
+var DIR_DEFAULT4 = "agent-contracts.yaml";
+function formatText(result) {
+  const lines = [];
+  lines.push(`DSL Completeness Score: ${result.overall}/100`);
+  lines.push("");
+  for (const d2 of result.dimensions) {
+    const detail = d2.total > 0 ? ` (${d2.score}/${d2.total} ${d2.id.split("-")[0]})` : "";
+    lines.push(
+      `  ${d2.label.padEnd(40)} ${String(d2.percent).padStart(3)}%${detail}`
+    );
+  }
+  const allRecs = result.dimensions.flatMap((d2) => d2.recommendations);
+  if (allRecs.length > 0) {
+    lines.push("");
+    lines.push("Recommendations:");
+    for (const rec of allRecs) {
+      lines.push(`  - ${rec}`);
+    }
+  }
+  return lines.join("\n");
+}
+var scoreCommand = new Command7("score").description("Calculate DSL completeness score").argument("[dir]", "Path to agent-contracts.yaml", DIR_DEFAULT4).option("-c, --config <path>", "Path to agent-contracts.config.yaml").option("--format <format>", "Output format (text|json)", "text").option("--threshold <number>", "Minimum score (exit 1 if below)", void 0).action(
+  async (dir, opts) => {
+    try {
+      const config = await loadConfig(opts.config);
+      const dslPath = resolveDslPath(dir, DIR_DEFAULT4, config);
+      const resolved = await resolve4(dslPath);
+      const data = config?.vars ? substituteVars(resolved.data, config.vars) : resolved.data;
+      const schemaResult = validateSchema(data);
+      if (!schemaResult.success) {
+        const issues = schemaResult.diagnostics.map((d2) => `  ${d2.path}: ${d2.message}`).join("\n");
+        process.stderr.write(`Schema validation failed:
+${issues}
+`);
+        process.exit(1);
+      }
+      const result = score(schemaResult.data);
+      if (opts.format === "json") {
+        process.stdout.write(JSON.stringify(result, null, 2) + "\n");
+      } else {
+        process.stdout.write(formatText(result) + "\n");
+      }
+      if (opts.threshold !== void 0) {
+        const threshold = parseInt(opts.threshold, 10);
+        if (isNaN(threshold)) {
+          process.stderr.write(
+            `Error: --threshold must be a number, got "${opts.threshold}"
+`
+          );
+          process.exit(1);
+        }
+        if (result.overall < threshold) {
+          process.stderr.write(
+            `Score ${result.overall} is below threshold ${threshold}
+`
+          );
+          process.exit(1);
+        }
+      }
+    } catch (err) {
+      const msg = err instanceof Error ? err.message : String(err);
+      process.stderr.write(`Error: ${msg}
+`);
+      process.exit(1);
+    }
+  }
+);
+
 // src/cli/cli.ts
 var __dirname = dirname7(fileURLToPath(import.meta.url));
 var pkg = JSON.parse(readFileSync(resolve7(__dirname, "../package.json"), "utf8"));
-var program = new Command7();
+var program = new Command8();
 program.name("agent-contracts").description("Agent contracts tooling \u2014 validate, lint, render DSL files").version(pkg.version);
 program.addCommand(resolveCommand);
 program.addCommand(validateCommand);
@@ -55087,6 +55531,7 @@ program.addCommand(lintCommand);
 program.addCommand(renderCommand);
 program.addCommand(checkCommand);
 program.addCommand(generateGuardrailsCommand);
+program.addCommand(scoreCommand);
 program.parse();
 /*! Bundled license information: