@awak-app/simy-cli 0.5.0 → 0.5.2

package/src/sqm/checkers.js

@@ -2,6 +2,8 @@ import { createHash } from "node:crypto";
 import { lstat, readFile } from "node:fs/promises";
 import path from "node:path";
 
+import { lintKnowledgeModule } from "./reliability.js";
+
 const MAX_FILE_BYTES = 1024 * 1024;
 const MAX_FILES_PER_PATTERN = 5_000;
 
@@ -12,10 +14,25 @@ export async function executeKnowledgeBundle(bundle, repository) {
 export async function evaluateKnowledgeBundle(bundle, repository) {
   const findings = [];
   const modules = [];
+  const matcherDiagnostics = [];
   for (const module of bundle.modules) {
-    if (["candidate", "retired"].includes(module.lifecycle.status)) continue;
-    if (!moduleApplies(module, repository.repository)) continue;
-    if (!(await matchModule(module, repository))) continue;
+    if (["candidate", "retired"].includes(module.lifecycle.status)) {
+      matcherDiagnostics.push(moduleDiagnostic(module, "inactive_lifecycle"));
+      continue;
+    }
+    if (!moduleApplies(module, repository.repository)) {
+      matcherDiagnostics.push(moduleDiagnostic(module, "repository_scope_mismatch"));
+      continue;
+    }
+    const matcherTrace = await matchModule(module, repository);
+    matcherDiagnostics.push(moduleDiagnostic(
+      module,
+      matcherTrace.matched ? "matched" : "matcher_not_satisfied",
+      matcherTrace.matchers,
+    ));
+    if (!matcherTrace.matched) continue;
+    const reliability = lintKnowledgeModule(module);
+    const enforcement = reliability.eligible_for_enforcement ? "blocking" : "advisory";
     const exceptions = activeExceptionPatterns(module);
     const ruleResults = [];
     for (const rule of module.detection.checks) {
@@ -32,6 +49,8 @@ export async function evaluateKnowledgeBundle(bundle, repository) {
           module_version: module.version,
           rule_id: rule.id,
           lifecycle: module.lifecycle.status,
+          enforcement,
+          reliability_issues: reliability.issues.map((issue) => issue.code),
           severity: module.lifecycle.severity,
           message: rule.message,
           invariant: module.invariant.statement,
@@ -49,15 +68,31 @@ export async function evaluateKnowledgeBundle(bundle, repository) {
           stressor_ids: assurance.stressor_ids,
           strength_ids: assurance.strength_ids,
           scenario_ids: assurance.scenario_ids,
+          matcher_trace: matcherTrace.matchers,
         });
       }
     }
-    modules.push(evaluateAssurance(module, ruleResults));
+    modules.push(evaluateAssurance(module, ruleResults, {
+      enforcement,
+      reliability,
+      matcherTrace: matcherTrace.matchers,
+    }));
   }
-  return { findings, modules };
+  return { findings, modules, matcherDiagnostics };
+}
+
+function moduleDiagnostic(module, reason, matchers = []) {
+  return {
+    module_id: module.id,
+    module_version: module.version,
+    lifecycle: module.lifecycle.status,
+    applicable: reason === "matched",
+    reason,
+    matchers,
+  };
 }
 
-function evaluateAssurance(module, ruleResults) {
+function evaluateAssurance(module, ruleResults, { enforcement, reliability, matcherTrace }) {
   const failed = new Set(ruleResults.filter((result) => result.status === "failed").map((result) => result.rule_id));
   const assurance = module.assurance;
   if (!assurance) {
@@ -66,6 +101,9 @@ function evaluateAssurance(module, ruleResults) {
       module_version: module.version,
       lifecycle: module.lifecycle.status,
       severity: module.lifecycle.severity,
+      enforcement,
+      reliability,
+      matcher_trace: matcherTrace,
       rule_results: ruleResults,
       state_models: [],
       stressors: [],
@@ -85,11 +123,14 @@ function evaluateAssurance(module, ruleResults) {
   }));
   const strengthStatus = new Map(strengths.map((strength) => [strength.id, strength.status]));
   const scenarios = assurance.scenarios.map((scenario) => {
-    let status = scenario.execution_requirement;
+    const checkFailed = scenario.check_refs.some((ref) => failed.has(ref));
+    const strengthFailed = scenario.expected_strength_refs.some((ref) => strengthStatus.get(ref) === "failed");
+    const activated = checkFailed || strengthFailed;
+    let status = "passed";
     if (scenario.execution_requirement === "local") {
-      const checkFailed = scenario.check_refs.some((ref) => failed.has(ref));
-      const strengthFailed = scenario.expected_strength_refs.some((ref) => strengthStatus.get(ref) === "failed");
       status = checkFailed || strengthFailed ? "failed" : "passed";
+    } else if (activated && enforcement === "blocking") {
+      status = scenario.execution_requirement;
     }
     return {
       id: scenario.id,
@@ -99,6 +140,7 @@ function evaluateAssurance(module, ruleResults) {
       stressor_refs: [...scenario.stressor_refs],
       expected_strength_refs: [...scenario.expected_strength_refs],
       check_refs: [...scenario.check_refs],
+      activated,
     };
   });
   const scenarioStatus = new Map(scenarios.map((scenario) => [scenario.id, scenario.status]));
@@ -120,6 +162,9 @@ function evaluateAssurance(module, ruleResults) {
     module_version: module.version,
     lifecycle: module.lifecycle.status,
     severity: module.lifecycle.severity,
+    enforcement,
+    reliability,
+    matcher_trace: matcherTrace,
     rule_results: ruleResults,
     state_models: stateModels,
     stressors,
@@ -146,24 +191,39 @@ function moduleApplies(module, repository) {
 }
 
 async function matchModule(module, repository) {
-  for (const matcher of module.detection.matchers) {
-    if (!(await match(matcher, repository))) return false;
+  const traces = [];
+  for (const [index, matcher] of module.detection.matchers.entries()) {
+    const result = await match(matcher, repository);
+    traces.push({
+      index,
+      type: matcher.type,
+      paths: [...matcher.paths],
+      matched: result.matched,
+      matched_files: result.matchedFiles,
+    });
   }
-  return true;
+  return { matched: traces.every((trace) => trace.matched), matchers: traces };
 }
 
 async function match(matcher, repository) {
   const changed = repository.diff.filter((file) => matchesAny(matcher.paths, file.path));
-  if (matcher.type === "changed_path") return changed.length > 0;
+  if (matcher.type === "changed_path") return matchResult(changed.length > 0, changed.map((file) => file.path));
   const regex = safeRegex(matcher.pattern);
   if (matcher.type === "diff_regex" || matcher.type === "reference") {
-    return changed.some((file) => file.lines.some((line) => sideMatches(matcher.side, line.side) && test(regex, line.text)));
+    const matched = changed.filter((file) => file.lines.some((line) => sideMatches(matcher.side, line.side) && test(regex, line.text)));
+    return matchResult(matched.length > 0, matched.map((file) => file.path));
   }
   if (matcher.type === "content_regex") {
-    const files = selectPaths(repository.paths, matcher.paths);
-    for (const file of files) if (test(regex, await readText(repository.root, file))) return true;
+    const files = changed.map((file) => file.path);
+    const matched = [];
+    for (const file of files) if (test(regex, await readText(repository.root, file))) matched.push(file);
+    return matchResult(matched.length > 0, matched);
   }
-  return false;
+  return matchResult(false, []);
+}
+
+function matchResult(matched, matchedFiles) {
+  return { matched, matchedFiles: [...new Set(matchedFiles)].slice(0, 50) };
 }
 
 async function executeRule(rule, repository) {

package/src/sqm/command.js

@@ -5,7 +5,9 @@ import path from "node:path";
 import { isSessionValid, readSession } from "../session-store.js";
 import { resolveWebOrigin } from "../web-origin.js";
 import { uploadRunEvidence } from "./evidence-client.js";
+import { evaluateKnowledgeBundle } from "./checkers.js";
 import { runSqmCheck, runSqmMinCheck } from "./index.js";
+import { lintKnowledgeModule, summarizeReplay } from "./reliability.js";
 import { ensureSqmSession } from "./session.js";
 
 export async function runSqmCommand(argv, { cliVersion }) {
@@ -13,7 +15,9 @@ export async function runSqmCommand(argv, { cliVersion }) {
     printHelp();
     return 0;
   }
-  if (!["min-check", "check"].includes(argv[0])) throw new Error("Usage: simy sqm <min-check|check> [options]");
+  if (argv[0] === "lint") return await runLintCommand(argv);
+  if (argv[0] === "replay") return await runReplayCommand(argv);
+  if (!["min-check", "check"].includes(argv[0])) throw new Error("Usage: simy sqm <min-check|check|lint|replay> [options]");
   const webOrigin = resolveWebOrigin(option(argv, "--host"));
   let session = await readSession(webOrigin);
   let validSession = isSessionValid(session, Date.now(), webOrigin) ? session : null;
@@ -61,14 +65,27 @@ export async function runSqmCommand(argv, { cliVersion }) {
   const result = await runSqmCheck({ ...common, minCheck, deviceId, signingKeyPath: option(argv, "--signing-key") || undefined });
   const proofPath = path.resolve(option(argv, "--proof") || "proof.json");
   const evidencePath = path.resolve(option(argv, "--signed-evidence") || `${proofPath}.signed.json`);
-  await Promise.all([atomicJson(proofPath, result.proof), atomicJson(evidencePath, result.signed_evidence)]);
+  const diagnosticsPath = path.resolve(option(argv, "--diagnostics") || `${proofPath}.diagnostics.json`);
+  await Promise.all([
+    atomicJson(proofPath, result.proof),
+    atomicJson(evidencePath, result.signed_evidence),
+    atomicJson(diagnosticsPath, {
+      policy_version: "sqm-reliability.v1",
+      matcher_diagnostics: result.matcher_diagnostics,
+    }),
+  ]);
   let upload = { uploaded: false, reason: "disabled" };
   if (!argv.includes("--no-upload")) {
     const evidenceEndpoint = process.env.SIMY_SQM_EVIDENCE_URL || (validSession?.api_base_url ? new URL("sqm/run-evidence", validSession.api_base_url).href : undefined);
     upload = await uploadRunEvidence({ endpoint: evidenceEndpoint, token, proof: result.proof, signedEvidence: result.signed_evidence });
   }
   result.upload = upload;
-  result.artifacts = { min_check: minPath, proof: proofPath, signed_evidence: evidencePath };
+  result.artifacts = {
+    min_check: minPath,
+    proof: proofPath,
+    signed_evidence: evidencePath,
+    diagnostics: diagnosticsPath,
+  };
   if (argv.includes("--json")) console.log(JSON.stringify(result, null, 2));
   else printFull(result);
   return result.proof.outcome.status === "failed" ? 1 : 0;
@@ -78,12 +95,16 @@ function printHelp() {
   console.log(`Usage:
   simy sqm min-check [options]
   simy sqm check [options]
+  simy sqm lint --bundle <path>
+  simy sqm replay --bundle <path> --dataset <path>
 
 Two-stage SQM flow:
   min-check  Fast deterministic diff preflight and full-check execution plan.
   check      Full state/invariant/stress/strength/scenario verification. It
              verifies the pinned min-check identity, writes proof.json, and
              produces independent per-run signed evidence.
+  lint       Report broad matchers and missing evaluation gates before publish.
+  replay     Evaluate a bundle against labelled historical repository snapshots.
 
 Options:
   --base <branch>             Compare against this base branch (default: dev)
@@ -95,12 +116,75 @@ Options:
   --min-result <path>         full check input (default: the last local min-check)
   --proof <path>              proof.json output (default: ./proof.json)
   --signed-evidence <path>    per-run signature output
+  --diagnostics <path>        local matcher trace output
   --no-upload                 keep proof local instead of uploading hashes/results
   --json                      print machine-readable output
 
 Neither phase executes Cloud-provided code or calls a local LLM.`);
 }
 
+async function runLintCommand(argv) {
+  const bundlePath = requiredOption(argv, "--bundle");
+  const input = await readJson(bundlePath);
+  const modules = Array.isArray(input.modules) ? input.modules : [input];
+  const report = {
+    policy_version: "sqm-reliability.v1",
+    modules: modules.map((module) => ({
+      module_id: module?.id ?? null,
+      lifecycle: module?.lifecycle?.status ?? null,
+      ...lintKnowledgeModule(module),
+    })),
+  };
+  report.issue_count = report.modules.reduce((total, module) => total + module.issues.length, 0);
+  if (argv.includes("--json")) console.log(JSON.stringify(report, null, 2));
+  else {
+    console.log(`SQM lint: ${report.issue_count} issue(s) across ${report.modules.length} module(s)`);
+    for (const module of report.modules) {
+      for (const issue of module.issues) console.log(`- ${module.module_id}: ${issue.code} · ${issue.message}`);
+    }
+  }
+  return report.modules.some((module) => module.lifecycle === "blocking" && !module.eligible_for_enforcement) ? 1 : 0;
+}
+
+async function runReplayCommand(argv) {
+  const bundlePath = requiredOption(argv, "--bundle");
+  const datasetPath = requiredOption(argv, "--dataset");
+  const [bundle, dataset] = await Promise.all([readJson(bundlePath), readJson(datasetPath)]);
+  if (!Array.isArray(bundle?.modules)) throw new Error("Replay bundle must contain a modules array.");
+  if (!Array.isArray(dataset?.cases) || !dataset.cases.length) throw new Error("Replay dataset must contain at least one labelled case.");
+  const results = [];
+  for (const [index, item] of dataset.cases.entries()) {
+    if (!["trigger", "clean"].includes(item?.expected)) throw new Error(`Replay case ${index} expected must be trigger or clean.`);
+    const repository = item.repository;
+    if (!repository?.repository || !Array.isArray(repository.paths) || !Array.isArray(repository.diff)) {
+      throw new Error(`Replay case ${index} must contain a repository snapshot.`);
+    }
+    const evaluation = await evaluateKnowledgeBundle(bundle, { root: repository.root || process.cwd(), ...repository });
+    results.push({
+      id: item.id || `case-${index + 1}`,
+      expected: item.expected,
+      triggered: evaluation.findings.length > 0,
+      finding_count: evaluation.findings.length,
+      module_ids: [...new Set(evaluation.findings.map((finding) => finding.module_id))],
+    });
+  }
+  const metrics = summarizeReplay(results);
+  const report = { policy_version: "sqm-reliability.v1", dataset: dataset.name || path.basename(datasetPath), metrics, results };
+  if (argv.includes("--json")) console.log(JSON.stringify(report, null, 2));
+  else console.log(`SQM replay: ${metrics.sample_count} case(s) · precision ${(metrics.precision * 100).toFixed(1)}% · recall ${(metrics.recall * 100).toFixed(1)}% · ${metrics.false_positives} false positive(s)`);
+  return metrics.false_positives || metrics.false_negatives ? 1 : 0;
+}
+
+async function readJson(location) {
+  return JSON.parse(await readFile(path.resolve(location), "utf8"));
+}
+
+function requiredOption(argv, name) {
+  const value = option(argv, name);
+  if (!value) throw new Error(`${name} is required.`);
+  return value;
+}
+
 function explicitSqmEnvironment(argv) {
   const organization = process.env.SIMY_SQM_ORGANIZATION_ID?.trim();
   const account = process.env.SIMY_SQM_ACCOUNT_ID?.trim();
@@ -123,7 +207,7 @@ function printFull(result) {
   console.log(`SQM full check: ${result.proof.outcome.status} · ${result.findings.length} finding(s)`);
   console.log(`Proof ${result.proof.proof_id} (${result.proof.integrity.digest})`);
   console.log(`Per-run signature: valid · ${result.signature_verification.key_id}`);
-  console.log(`Artifacts: ${result.artifacts.proof} · ${result.artifacts.signed_evidence}`);
+  console.log(`Artifacts: ${result.artifacts.proof} · ${result.artifacts.signed_evidence} · ${result.artifacts.diagnostics}`);
   if (result.upload.uploaded) console.log(`Cloud evidence: stored · ${result.upload.evidence_id || result.signed_evidence.evidence_id}`);
   else console.log(`Cloud evidence: local only (${result.upload.reason})`);
 }

package/src/sqm/index.js

@@ -99,7 +99,7 @@ async function buildMinCheck({ repository, bundle, evaluation, organizationId, c
     knowledge_bundle: bundleIdentity(bundle),
     executed_at: new Date().toISOString(),
     cli_version: cliVersion,
-    status: evaluation.findings.length ? "preliminary_failed" : "passed",
+    status: evaluation.findings.some((finding) => finding.enforcement === "blocking") ? "preliminary_failed" : "passed",
     preliminary_findings: evaluation.findings,
     matched_modules: evaluation.modules.map((module) => ({ module_id: module.module_id, module_version: module.module_version, state_model_ids: module.state_models.map((model) => model.id), transition_ids: module.state_models.flatMap((model) => model.transitions.map((transition) => transition.id)), stressor_ids: module.stressors.map((stressor) => stressor.id), scenario_ids: module.scenarios.map((scenario) => scenario.id) })),
     full_check_plan: plan,
@@ -119,6 +119,7 @@ async function buildFullCheck({ repository, bundle, minCheck, evaluation, organi
     min_check: minCheck,
     findings: evaluation.findings,
     executions: evaluation.modules,
+    matcher_diagnostics: evaluation.matcherDiagnostics || [],
     proof,
     signed_evidence: signedEvidence,
     signature_verification: verification,

package/src/sqm/proof.js

@@ -38,13 +38,15 @@ export function verifyMinCheck(value) {
 }
 
 export function buildProof({ organizationId, repository, bundle, minCheck, evaluation, findings, cliVersion, executedAt = new Date().toISOString() }) {
-  const statuses = evaluation.modules.flatMap((module) => module.scenarios.map((scenario) => scenario.status));
-  const failedModules = evaluation.modules.filter((module) => module.scenarios.some((scenario) => scenario.status === "failed"));
-  const hasFailure = findings.length > 0 || failedModules.length > 0;
-  const shadowOnly = hasFailure && findings.every((finding) => finding.lifecycle === "shadow") && failedModules.every((module) => module.lifecycle === "shadow");
+  const enforcingModules = evaluation.modules.filter((module) => module.enforcement === "blocking");
+  const enforcingFindings = findings.filter((finding) => finding.enforcement === "blocking");
+  const statuses = enforcingModules.flatMap((module) => module.scenarios.map((scenario) => scenario.status));
+  const failedModules = enforcingModules.filter((module) => module.scenarios.some((scenario) => scenario.status === "failed"));
+  const hasFailure = enforcingFindings.length > 0 || failedModules.length > 0;
+  const advisoryFindingCount = findings.length - enforcingFindings.length;
   const requiresHuman = statuses.includes("requires_human");
   const requiresPostDeploy = statuses.includes("requires_post_deploy");
-  const status = shadowOnly ? "shadow" : hasFailure ? "failed" : requiresHuman ? "requires_human" : requiresPostDeploy ? "requires_post_deploy" : evaluation.modules.length === 0 ? "skipped" : "passed";
+  const status = hasFailure ? "failed" : requiresHuman ? "requires_human" : requiresPostDeploy ? "requires_post_deploy" : "passed";
   const proof = {
     schema_version: "1.0.0",
     proof_id: `SQM-PROOF-${randomUUID()}`,
@@ -67,9 +69,9 @@ export function buildProof({ organizationId, repository, bundle, minCheck, evalu
     outcome: {
       status,
       reasons: [
-        ...(hasFailure && !shadowOnly ? ["one_or_more_declarative_checks_failed"] : []),
-        ...(shadowOnly ? ["declarative_checks_failed_in_shadow_mode"] : []),
-        ...(evaluation.modules.length === 0 ? ["no_applicable_knowledge_module"] : []),
+        ...(hasFailure ? ["one_or_more_blocking_checks_failed"] : []),
+        ...(advisoryFindingCount > 0 ? ["advisory_findings_observed"] : []),
+        ...(evaluation.modules.length === 0 ? ["no_applicable_rules"] : []),
         ...(requiresHuman ? ["human_evidence_required"] : []),
         ...(requiresPostDeploy ? ["post_deploy_evidence_required"] : []),
       ],

package/src/sqm/reliability.js

@@ -0,0 +1,94 @@
+const BROAD_PATHS = new Set(["**", "**/*", "src/**", "src/**/*"]);
+const BROAD_EXTENSION_PATH = /^\*\*\/\*\.[a-z0-9{}*,]+$/i;
+
+export const RELIABILITY_POLICY_VERSION = "sqm-reliability.v1";
+
+export function lintKnowledgeModule(module) {
+  const issues = [];
+  const repositories = module?.scope?.repositories || [];
+  const matchers = module?.detection?.matchers || [];
+  const lifecycle = module?.lifecycle?.status;
+  const evaluation = module?.evaluation || {};
+
+  if (!repositories.length) {
+    issues.push(issue("repository_scope_required", "Module must declare at least one repository."));
+  }
+
+  const broadMatchers = matchers.filter((matcher) =>
+    (matcher.paths || []).some(isBroadPath)
+  );
+  if (broadMatchers.length) {
+    issues.push(issue(
+      "broad_matcher",
+      "Matchers using repository-wide or top-level source globs are observation-only.",
+      { matcher_types: [...new Set(broadMatchers.map((matcher) => matcher.type))] },
+    ));
+  }
+
+  const hasDiffSignal = matchers.some((matcher) =>
+    ["changed_path", "diff_regex", "reference"].includes(matcher.type) &&
+    (matcher.paths || []).some((path) => !isBroadPath(path))
+  );
+  if (matchers.some((matcher) => matcher.type === "content_regex") && !hasDiffSignal) {
+    issues.push(issue(
+      "content_matcher_requires_diff_scope",
+      "A content matcher must be combined with a narrow changed-path or diff signal.",
+    ));
+  }
+
+  if (["warn", "blocking"].includes(lifecycle)) {
+    const total = number(evaluation.true_positives) + number(evaluation.false_positives);
+    const precision = total > 0 ? number(evaluation.true_positives) / total : 0;
+    const minimumSamples = lifecycle === "blocking" ? 50 : 20;
+    const minimumPrecision = lifecycle === "blocking" ? 0.99 : 0.95;
+    if (evaluation.status !== "historical_replay" && evaluation.status !== "prospective") {
+      issues.push(issue("replay_required", "Warn and blocking modules require replay or prospective evaluation."));
+    }
+    if (total < minimumSamples) {
+      issues.push(issue("insufficient_positive_samples", `Lifecycle ${lifecycle} requires at least ${minimumSamples} adjudicated positive samples.`, { actual: total, required: minimumSamples }));
+    }
+    if (precision < minimumPrecision) {
+      issues.push(issue("precision_below_threshold", `Lifecycle ${lifecycle} requires precision of at least ${minimumPrecision}.`, { actual: precision, required: minimumPrecision }));
+    }
+    if (lifecycle === "blocking" && number(evaluation.false_negatives) > 0) {
+      issues.push(issue("blocking_false_negative", "Blocking modules cannot have a known false negative."));
+    }
+  }
+
+  return {
+    policy_version: RELIABILITY_POLICY_VERSION,
+    eligible_for_enforcement: lifecycle === "blocking" && issues.length === 0,
+    issues,
+  };
+}
+
+export function summarizeReplay(results) {
+  const counts = { true_positives: 0, false_positives: 0, false_negatives: 0, true_negatives: 0 };
+  for (const result of results) {
+    if (result.expected === "trigger" && result.triggered) counts.true_positives += 1;
+    else if (result.expected === "clean" && result.triggered) counts.false_positives += 1;
+    else if (result.expected === "trigger") counts.false_negatives += 1;
+    else counts.true_negatives += 1;
+  }
+  const predictedPositive = counts.true_positives + counts.false_positives;
+  const actualPositive = counts.true_positives + counts.false_negatives;
+  return {
+    ...counts,
+    sample_count: results.length,
+    precision: predictedPositive ? counts.true_positives / predictedPositive : 0,
+    recall: actualPositive ? counts.true_positives / actualPositive : 0,
+  };
+}
+
+function isBroadPath(path) {
+  const value = String(path || "").replace(/^\.\//, "");
+  return BROAD_PATHS.has(value) || BROAD_EXTENSION_PATH.test(value);
+}
+
+function number(value) {
+  return Number.isFinite(Number(value)) ? Number(value) : 0;
+}
+
+function issue(code, message, details = {}) {
+  return { code, message, ...details };
+}