@intentsolutions/audit-harness 1.1.5 → 1.1.7

package/schemas/audit-profile/v1.schema.json

@@ -0,0 +1,294 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://evals.intentsolutions.io/audit-profile/v1.schema.json",
+  "title": "Audit Profile (v1)",
+  "description": "Closed, versioned, hash-bearing value describing the audit profile of a repository: the UNION of detected classifications, the resolved gate set (each with applicability + enforcement + dimension), and the explicit unresolved residue. Produced by `audit-harness classify` as read-only stdout JSON; the classifier NEVER mutates the repo. A profile is a deterministic function of (repo contents at commit_sha, the canonical dimension-to-gate registry pinned by registry_hash, and any engineer .audit-harness.yml overrides). Mirrors the gate-result/v1 closed-value conventions (additionalProperties:false, version-in-$id) so the two predicates compose in one Evidence Bundle.",
+  "type": "object",
+  "required": [
+    "schema_version",
+    "subject",
+    "classifier",
+    "registry_hash",
+    "timestamp",
+    "classifications",
+    "gates",
+    "unresolved"
+  ],
+  "additionalProperties": false,
+  "properties": {
+    "schema_version": {
+      "description": "Self-describing schema identity. MUST equal the version segment of $id.",
+      "const": "audit-profile/v1"
+    },
+    "subject": {
+      "description": "Identity of the repository (or monorepo package) the profile was computed for.",
+      "type": "object",
+      "required": [
+        "name",
+        "commit_sha"
+      ],
+      "additionalProperties": false,
+      "properties": {
+        "name": {
+          "type": "string",
+          "description": "Repository or package name (e.g. 'audit-harness', '@j-rig/core').",
+          "minLength": 1
+        },
+        "commit_sha": {
+          "type": "string",
+          "description": "Git commit SHA the classification was computed against. Full 40-hex or short 7-hex.",
+          "pattern": "^[a-f0-9]{7,40}$"
+        },
+        "root": {
+          "type": "string",
+          "description": "Repo-relative path of the classified root. '.' for the repo root; a package path (e.g. 'packages/core') for a monorepo member classified independently.",
+          "default": "."
+        }
+      }
+    },
+    "classifier": {
+      "type": "string",
+      "description": "Tool + semver identifier that produced the profile. Same shape as gate-result/v1 'runner'.",
+      "pattern": "^[a-z0-9][a-z0-9-]*@[0-9]+\\.[0-9]+\\.[0-9]+(-[A-Za-z0-9.-]+)?(\\+[A-Za-z0-9.-]+)?$",
+      "examples": [
+        "audit-harness@1.2.0"
+      ]
+    },
+    "registry_hash": {
+      "type": "string",
+      "description": "SHA-256 of the canonical dimension-to-gate registry datum the profile was resolved against, prefixed sha256:. The registry is the single canonical source of 'which gates apply to repo-type X'; layer-applicability.md and TESTING.md are projections of it. Recording its hash makes a profile reproducible against the exact registry version.",
+      "pattern": "^sha256:[a-f0-9]{64}$"
+    },
+    "timestamp": {
+      "type": "string",
+      "format": "date-time",
+      "description": "RFC 3339 UTC timestamp. Moment the classifier emitted the profile."
+    },
+    "classifications": {
+      "type": "array",
+      "description": "The UNION of all repo-type classifications detected (NOT a single winner). A repo that is a TS monorepo AND ships SKILL.md AND an MCP server carries all three entries; picking one silently drops the others' gates (a false-negative, worse than a false-positive).",
+      "minItems": 1,
+      "items": {
+        "$ref": "#/$defs/classification"
+      }
+    },
+    "dimensions": {
+      "type": "array",
+      "description": "Convenience projection: the distinct audit dimensions covered by the resolved gates. Derivable from gates[].dimension; consumers MUST treat gates[] as authoritative.",
+      "items": {
+        "$ref": "#/$defs/dimension"
+      },
+      "uniqueItems": true
+    },
+    "gates": {
+      "type": "array",
+      "description": "The resolved gate set for this repo: every gate the registry maps to the detected classifications, each tagged with applicability (from the registry) and enforcement (advisory by default; blocking only when engineer-pinned in TESTING.md and FP-rate-gated).",
+      "items": {
+        "$ref": "#/$defs/gate"
+      }
+    },
+    "unresolved": {
+      "type": "array",
+      "description": "The explicit residue the DETERMINISTIC classifier could not resolve. This is the ONLY surface a Claude inspector (/audit-tests) may refine — it never co-authors the deterministic classification value, so CI stays reproducible. An empty array means a fully deterministic profile.",
+      "items": {
+        "$ref": "#/$defs/unresolved_item"
+      }
+    },
+    "overrides": {
+      "type": "object",
+      "description": "Engineer-owned .audit-harness.yml directives that influenced this profile (classification pins, per-gate advisory/disable toggles, kill-switch). Informative provenance; the effect is already baked into classifications[] and gates[].",
+      "additionalProperties": false,
+      "properties": {
+        "source": {
+          "type": "string",
+          "description": "Path of the override file applied, repo-relative.",
+          "examples": [
+            ".audit-harness.yml"
+          ]
+        },
+        "override_hash": {
+          "type": "string",
+          "description": "SHA-256 of the applied override file, prefixed sha256:.",
+          "pattern": "^sha256:[a-f0-9]{64}$"
+        },
+        "kill_switch": {
+          "type": "boolean",
+          "description": "True when AUDIT_HARNESS_DISABLE=1 or the override file disabled all gates. A killed profile still emits classifications for visibility but marks every gate enforcement=disabled.",
+          "default": false
+        }
+      }
+    }
+  },
+  "$defs": {
+    "dimension": {
+      "type": "string",
+      "description": "The comprehensive audit dimensions. testing-depth = the 7-layer pyramid; conformance = artifact/schema/protocol validation; currency = upstream-drift (advisory only); security = supply-chain; hygiene = repo docs/links/changelog; skill-quality = behavioral verdicts consumed from j-rig.",
+      "enum": [
+        "testing-depth",
+        "conformance",
+        "currency",
+        "security",
+        "hygiene",
+        "skill-quality"
+      ]
+    },
+    "classification": {
+      "type": "object",
+      "required": [
+        "kind",
+        "confidence"
+      ],
+      "additionalProperties": false,
+      "properties": {
+        "kind": {
+          "type": "string",
+          "description": "Repo-type or artifact-kind classification. The first block mirrors layer-applicability.md repo types; the second covers Claude-ecosystem artifact kinds; 'regulated' is an overlay that escalates gate severity.",
+          "enum": [
+            "service",
+            "api",
+            "frontend",
+            "cli",
+            "library",
+            "embedded",
+            "monorepo",
+            "regulated",
+            "skill",
+            "agent",
+            "hook",
+            "mcp",
+            "plugin",
+            "marketplace",
+            "action",
+            "unknown"
+          ]
+        },
+        "confidence": {
+          "type": "string",
+          "description": "How the classification was reached. 'declared' = an explicit engineer override/marker; 'detected' = an unambiguous deterministic signal; 'inferred' = a weaker heuristic; 'unresolved' = the classifier could not decide (a matching entry MUST appear in unresolved[]).",
+          "enum": [
+            "declared",
+            "detected",
+            "inferred",
+            "unresolved"
+          ]
+        },
+        "signals": {
+          "type": "array",
+          "description": "Deterministic evidence for the classification (e.g. 'package.json:react', 'SKILL.md present', 'pnpm-workspace.yaml').",
+          "items": {
+            "type": "string"
+          }
+        }
+      }
+    },
+    "gate": {
+      "type": "object",
+      "required": [
+        "gate_id",
+        "dimension",
+        "applicability",
+        "enforcement"
+      ],
+      "additionalProperties": false,
+      "properties": {
+        "gate_id": {
+          "type": "string",
+          "description": "Pipeline-hop-qualified gate identifier, same regex as gate-result/v1 gate_id (tool:side:gate-id). The gate-result row a run later emits for this gate MUST carry the same gate_id.",
+          "pattern": "^[a-z0-9][a-z0-9-]*:(client|server|ci|sandbox|local):[a-zA-Z0-9][a-zA-Z0-9.-]*$",
+          "examples": [
+            "audit-harness:ci:crap-score",
+            "audit-harness:local:conform-skillmd",
+            "audit-harness:ci:secrets-gitleaks"
+          ]
+        },
+        "dimension": {
+          "$ref": "#/$defs/dimension"
+        },
+        "applicability": {
+          "type": "string",
+          "description": "From the registry, given the detected classifications. required = P0 (✅); recommended = P1 advisory (⭕); conditional = fires P0/P1 only if a sibling signal is present (⚠); waived = not run for this repo type (❌).",
+          "enum": [
+            "required",
+            "recommended",
+            "conditional",
+            "waived"
+          ]
+        },
+        "enforcement": {
+          "type": "string",
+          "description": "What a FAIL does. New gates ship 'advisory' (exit 0, finding logged). 'blocking' is earned: an engineer pins it in the hash-pinned TESTING.md after the gate shows a measured false-positive rate below the stated bar on the IEP corpus. 'disabled' = waived or kill-switched.",
+          "enum": [
+            "blocking",
+            "advisory",
+            "disabled"
+          ]
+        },
+        "result_class_default": {
+          "type": "string",
+          "description": "The result class a successful no-finding run would emit (informative). INDETERMINATE is reserved for infra failures (≠ policy failure): pure-local policy gates fail closed (FAIL); network-touching checks fail open/advisory (INDETERMINATE).",
+          "enum": [
+            "PASS",
+            "FAIL",
+            "ADVISORY",
+            "NOT_APPLICABLE",
+            "INDETERMINATE"
+          ]
+        },
+        "tool": {
+          "type": "string",
+          "description": "External tool this gate shells out to, if any (e.g. 'gitleaks', 'osv-scanner', 'spectral', 'ajv', 'markdownlint', 'lychee'). Absent for native/policy gates. A missing tool at run-time yields an 'unmeasured' gate-result, not a FAIL.",
+          "examples": [
+            "gitleaks",
+            "osv-scanner",
+            "spectral"
+          ]
+        }
+      },
+      "allOf": [
+        {
+          "if": {
+            "properties": {
+              "applicability": {
+                "const": "waived"
+              }
+            }
+          },
+          "then": {
+            "properties": {
+              "enforcement": {
+                "const": "disabled"
+              }
+            }
+          }
+        }
+      ]
+    },
+    "unresolved_item": {
+      "type": "object",
+      "required": [
+        "kind",
+        "reason"
+      ],
+      "additionalProperties": false,
+      "properties": {
+        "kind": {
+          "type": "string",
+          "description": "What could not be resolved (e.g. 'repo-type', 'monorepo-package-boundary', 'is-published-library').",
+          "minLength": 1
+        },
+        "reason": {
+          "type": "string",
+          "description": "Why the deterministic classifier abstained — the question a human (or /audit-tests) must answer.",
+          "minLength": 1
+        },
+        "candidates": {
+          "type": "array",
+          "description": "The candidate classifications the classifier was torn between, if any.",
+          "items": {
+            "type": "string"
+          }
+        }
+      }
+    }
+  }
+}

package/schemas/conform/v1/agent-frontmatter.schema.json

@@ -0,0 +1,24 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://evals.intentsolutions.io/conform/v1/agent-frontmatter.schema.json",
+  "title": "Subagent definition frontmatter — deterministic structural floor (conform/v1)",
+  "description": "The DETERMINISTIC STRUCTURAL FLOOR for a Claude subagent definition (.md with YAML frontmatter): it parses and carries name + description. tools/model are type-checked when present. Trigger-precision + tool-allowlist-tightness judgment stays in /validate-agent. additionalProperties:true for forward-compat.",
+  "type": "object",
+  "required": ["name", "description"],
+  "additionalProperties": true,
+  "properties": {
+    "name": {
+      "type": "string",
+      "minLength": 1,
+      "pattern": "^[a-zA-Z0-9][a-zA-Z0-9_-]*$"
+    },
+    "description": { "type": "string", "minLength": 1 },
+    "tools": {
+      "anyOf": [
+        { "type": "string" },
+        { "type": "array", "items": { "type": "string" } }
+      ]
+    },
+    "model": { "type": "string" }
+  }
+}

package/schemas/conform/v1/mcp-config.schema.json

@@ -0,0 +1,31 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://evals.intentsolutions.io/conform/v1/mcp-config.schema.json",
+  "title": ".mcp.json MCP server config — deterministic structural floor (conform/v1)",
+  "description": "The DETERMINISTIC STRUCTURAL FLOOR for a Claude .mcp.json: it parses, declares an mcpServers map, and each declared server is launchable (carries either a `command` for a stdio server or a `url` for a remote sse/http server). Server entries keep additionalProperties:true for forward-compat with newer transport fields; the floor only asserts the launch contract + types of known keys.",
+  "type": "object",
+  "required": ["mcpServers"],
+  "additionalProperties": true,
+  "properties": {
+    "mcpServers": {
+      "type": "object",
+      "additionalProperties": {
+        "type": "object",
+        "additionalProperties": true,
+        "anyOf": [
+          { "required": ["command"] },
+          { "required": ["url"] }
+        ],
+        "properties": {
+          "command": { "type": "string", "minLength": 1 },
+          "args": { "type": "array", "items": { "type": "string" } },
+          "env": { "type": "object" },
+          "url": { "type": "string", "minLength": 1 },
+          "type": { "type": "string", "enum": ["stdio", "sse", "http"] },
+          "headers": { "type": "object" },
+          "cwd": { "type": "string" }
+        }
+      }
+    }
+  }
+}

package/schemas/conform/v1/plugin-manifest.schema.json

@@ -0,0 +1,26 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://evals.intentsolutions.io/conform/v1/plugin-manifest.schema.json",
+  "title": ".claude-plugin/plugin.json manifest — deterministic structural floor (conform/v1)",
+  "description": "The DETERMINISTIC STRUCTURAL FLOOR for a Claude plugin manifest: it parses and carries a name. Version/description/author are type-checked when present. The richer catalog-invariant + cross-artifact checks (INV-PLUGIN-SKILL etc.) live in /validate-plugin + the SAK kernel — conform asserts only the manifest floor. additionalProperties:true for forward-compat with the evolving plugin manifest surface.",
+  "type": "object",
+  "required": ["name"],
+  "additionalProperties": true,
+  "properties": {
+    "name": {
+      "type": "string",
+      "minLength": 1,
+      "pattern": "^[a-zA-Z0-9][a-zA-Z0-9_-]*$"
+    },
+    "version": { "type": "string" },
+    "description": { "type": "string" },
+    "author": {
+      "anyOf": [
+        { "type": "string" },
+        { "type": "object" }
+      ]
+    },
+    "license": { "type": "string" },
+    "homepage": { "type": "string" }
+  }
+}

package/schemas/conform/v1/skillmd-frontmatter.schema.json

@@ -0,0 +1,40 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://evals.intentsolutions.io/conform/v1/skillmd-frontmatter.schema.json",
+  "title": "SKILL.md frontmatter — deterministic structural floor (conform/v1)",
+  "description": "The DETERMINISTIC STRUCTURAL FLOOR for a SKILL.md frontmatter block: it parses, carries the Anthropic-spec required pair (name + description), and the known optional fields are the right type. This is NOT the IS 100-point marketplace rubric or the 8-field enterprise required set — that grading is judgment-bearing and stays in /validate-skillmd (Tier 1) + the SAK authoring kernel. conform validates the floor; Claude adds judgment on top. additionalProperties is true because SKILL.md frontmatter is intentionally extensible (model, effort, context, hooks, metadata, user-invocable, etc.).",
+  "type": "object",
+  "required": ["name", "description"],
+  "additionalProperties": true,
+  "properties": {
+    "name": {
+      "type": "string",
+      "minLength": 1,
+      "pattern": "^[a-zA-Z0-9][a-zA-Z0-9_-]*$"
+    },
+    "description": {
+      "type": "string",
+      "minLength": 1
+    },
+    "allowed-tools": {
+      "anyOf": [
+        { "type": "string" },
+        { "type": "array", "items": { "type": "string" } }
+      ]
+    },
+    "disallowed-tools": {
+      "anyOf": [
+        { "type": "string" },
+        { "type": "array", "items": { "type": "string" } }
+      ]
+    },
+    "version": { "type": "string" },
+    "author": { "type": "string" },
+    "license": { "type": "string" },
+    "compatibility": { "type": "string", "maxLength": 500 },
+    "tags": { "type": "array", "items": { "type": "string" } },
+    "user-invocable": { "type": "boolean" },
+    "disable-model-invocation": { "type": "boolean" },
+    "argument-hint": { "type": "string" }
+  }
+}

package/schemas/currency/pins.v1.json

@@ -0,0 +1,55 @@
+{
+  "pins_version": "currency-pins/v1",
+  "description": "Per-upstream-identity pin relation. Each upstream the harness/skills depend on carries ITS OWN pinned version + the date it was last verified against upstream (checked_at) + a staleness window. The `currency` advisory report reads this datum and flags pins whose checked_at is older than their window — i.e. it makes the PIN'S OWN STALENESS detectable, without ever live-fetching. Currency is advisory-only: it reports + (in /sync-testing-harness) opens PRs; it has no exit-code authority and never auto-fixes. Updating a pin (after a human re-verifies against upstream) is an engineer edit to this file + a fresh checked_at.",
+  "default_staleness_window_days": 90,
+  "pins": [
+    {
+      "identity": "mcp-spec",
+      "pinned_version": "2025-06-18",
+      "source": "https://spec.modelcontextprotocol.io/ (protocol revision)",
+      "checked_at": "2026-06-06",
+      "staleness_window_days": 90,
+      "notes": "MCP protocol spec revision the .mcp.json conform schema targets."
+    },
+    {
+      "identity": "skill-md-schema",
+      "pinned_version": "3.7.0",
+      "source": "claude-code-plugins 000-docs/SCHEMA_CHANGELOG.md",
+      "checked_at": "2026-06-06",
+      "staleness_window_days": 90,
+      "notes": "IS SKILL.md schema the conform skillmd-frontmatter floor tracks (full rubric stays in /validate-skillmd)."
+    },
+    {
+      "identity": "claude-code",
+      "pinned_version": "2.1.152",
+      "source": "https://code.claude.com/docs/en/changelog",
+      "checked_at": "2026-06-06",
+      "staleness_window_days": 60,
+      "notes": "Claude Code release (added disallowed-tools frontmatter at 2.1.152)."
+    },
+    {
+      "identity": "gate-result-predicate",
+      "pinned_version": "v1",
+      "source": "@intentsolutions/core gate-result/v1 (https://evals.intentsolutions.io/gate-result/v1)",
+      "checked_at": "2026-06-06",
+      "staleness_window_days": 180,
+      "notes": "Evidence Bundle predicate every gate emits against. Immutable URI; bump = new version, never overwrite."
+    },
+    {
+      "identity": "anthropic-sdk",
+      "pinned_version": "unverified",
+      "source": "https://github.com/anthropics/anthropic-sdk-python (+ -typescript)",
+      "checked_at": "2026-06-06",
+      "staleness_window_days": 90,
+      "notes": "Anthropic SDK surface referenced by downstream skills; pinned_version=unverified until first deliberate verification."
+    },
+    {
+      "identity": "agentskills-spec",
+      "pinned_version": "1.0.0",
+      "source": "https://agentskills.io/specification",
+      "checked_at": "2026-06-06",
+      "staleness_window_days": 90,
+      "notes": "Open SKILL.md standard (compatibility/metadata/license fields)."
+    }
+  ]
+}