@dzhechkov/harness-core 0.3.99 → 0.3.101

package/src/claim-check.ts

@@ -0,0 +1,211 @@
+// SPDX-License-Identifier: MIT
+//
+// Ported from rUv / ruview (`@ruvnet/ruview@0.2.0`, `src/guardrails.js`), (c) rUv / ruview,
+// MIT. Adapted for the dz harness monorepo. The hard-won detection semantics (label-vs-metric
+// disambiguation, short-token `.map`/`map-reduce` guards, the `\b`-free `PERFECT_PCT_RE`, the
+// code-span-scrub asymmetry, and the four-branch check order) are ported VERBATIM with their
+// explanatory comments intact; ONLY the domain vocabulary lists (metric terms, reproducer hints,
+// honest tags) and the human-facing reason/suggestion strings are re-worded from WiFi-sensing
+// (ruview) to a skills/harness monorepo (dz).
+//
+// The dz Integrity Rule (CLAUDE.md) declares "NO shortcuts, fake data, or false claims; ALWAYS
+// verify before claiming success." Today that rule is prose. This module is the static enforcement
+// of it: every quantitative accuracy/coverage/count claim must be tagged MEASURED (with a
+// reproducer) or CLAIMED/SYNTHETIC/ESTIMATED, and the retracted "100% / perfect" framing must
+// never reappear untagged. It is a pure, never-throw, no-I/O module — all file reading lives in the
+// `dz claim-check` CLI adapter and the `dz publish` pre-publish gate, mirroring how `usage.ts`
+// keeps computation pure while the CLI does the reading.
+
+/** A single untagged/overstated-claim finding. The pure engine sees only text — no `file` field. */
+export interface ClaimFinding {
+  readonly severity: 'high' | 'medium';
+  readonly line: number; // 1-based, within the scanned text
+  readonly excerpt: string; // clip()'d to 120 chars
+  readonly reason: string;
+  readonly suggestion: string;
+}
+
+/** Result of a claim-check pass over one block of text. */
+export interface ClaimCheckResult {
+  readonly ok: boolean;
+  readonly findings: ClaimFinding[];
+}
+
+/** Phrases that signal a quantitative accuracy claim (safe as substrings). */
+const METRIC_TERMS = [
+  // Generic accuracy vocabulary kept from ruview.
+  'accuracy', 'precision', 'recall',
+  'error rate', 'detection rate', 'true positive',
+  // dz claim vocabulary — a harness monorepo claims coverage, test counts, catalogue sizes,
+  // and performance numbers, not WiFi-sensing PCK/MPJPE.
+  'coverage', 'tests passed', 'test count', 'skills', 'commands',
+  'packages', 'presets', 'downloads', 'benchmark', 'speedup', 'latency',
+];
+
+// Short/ambiguous metric tokens (ADR-263 F11): 'map' is usually the English
+// word or a file extension, 'f1'/'o1' collide with finding/option labels.
+// They only count as metric mentions when word-bounded, not a `.map` file
+// reference, and the line (after scrubbing) carries a number — "mAP 62.3" is
+// a claim, "F-numbers map to findings" is not.
+// 'map' additionally must not be a `.map` file suffix or a hyphenated
+// compound ("map-free", "map-reduce") — mAP the metric never appears as either.
+// `\d+ tests` is dz's single most-repeated headline claim ("2136 tests") and was slipping through:
+// the substring term is 'tests passed', so a bare count never fired. Anchor it to a PRECEDING number
+// rather than adding a bare 'test' term — otherwise every `usage.test.ts:42` path reference would
+// register as a metric mention.
+const METRIC_TERMS_SHORT = [
+  /(?<![.\w])map\b(?!-)/, /\bf1\b/, /\bauc\b/, /\biou\b/,
+  /\b\d[\d,._]*\s+tests?\b/,
+];
+// Finding/option labels (F1, O2, …) count as labels unless the token sits in a
+// metric context: an immediately following score/=/%/digit or colon ("F1: 0.91"),
+// or a number later in the same clause ("F1 reaches 0.91" — an F1-score claim).
+// Bare option refs ("F7 fixes", "O1–O9", "ADR-263 O2") carry no clause number of
+// their own and stay labels. (A surviving 'f1' still only fires as a metric when
+// its scrubbed line actually carries a number — see mentionsMetricTerm.)
+const LABEL_TOKEN_RE = /\b[fo]\d+\b(?!\s*(?:score|=|\d|%|:))(?![^\n.;]*\d)/g;
+const CODE_SPAN_RE = /`[^`]*`/g; // backticked identifiers are code, not claims
+
+// Markdown link/image TARGETS are machinery, not prose claims. A shields.io badge URL
+// (`![Skills](https://img.shields.io/badge/skills-167-brightgreen)`) embeds the very words and
+// numbers this checker hunts for, so leaving URLs in produced ~600 findings on this repo — noise
+// that buries the real ones. The link TEXT is kept, so a claim written in prose still fires; the
+// same counts always appear in prose next to the badges. Consequence, accepted knowingly: a number
+// that exists ONLY inside a URL is not checked.
+const MD_URL_RE = /\]\([^)\s]*(?:\s[^)]*)?\)/g;
+const AUTOLINK_RE = /<https?:\/\/[^>]*>|(?<![(<])\bhttps?:\/\/\S+/g;
+
+/** Strip markdown link/image targets and bare URLs, keeping the surrounding prose. */
+function stripUrls(s: string): string {
+  return s.replace(MD_URL_RE, '] ').replace(AUTOLINK_RE, ' ');
+}
+const HAS_NUMBER_RE = /\d/;
+
+/** Line with code spans and finding/option labels removed. */
+function scrubLine(lower: string): string {
+  return lower.replace(CODE_SPAN_RE, ' ').replace(LABEL_TOKEN_RE, ' ');
+}
+
+function mentionsMetricTerm(lower: string, scrubbed: string): boolean {
+  if (METRIC_TERMS.some((t) => lower.includes(t))) return true;
+  if (!HAS_NUMBER_RE.test(scrubbed)) return false;
+  return METRIC_TERMS_SHORT.some((re) => re.test(scrubbed));
+}
+
+/** Tags that make a claim honest (case-insensitive). */
+// `estimated` agrees with the `estimated: true` honest-uncertainty marker `dz usage` already
+// emits — the two honesty systems must not contradict each other.
+const HONEST_TAGS = ['measured', 'claimed', 'synthetic', 'unvalidated', 'baseline', 'estimated'];
+
+/** Reproducer references that count as evidence backing a MEASURED claim. */
+const REPRODUCER_HINTS = [
+  // Generic evidence hints kept from ruview.
+  'baseline', 'reproduce', 'sha256', 'tarball', 'cargo test',
+  // Packaging-claim reproducers (npm reviews): the tarball itself.
+  'npm pack', 'npm view', 'npm i ', 'npm install',
+  // dz reproducers — the actual commands/artifacts that back a dz claim.
+  'npm test', 'vitest', 'npm run', 'git rev', 'commit',
+  'test output', 'coverage report', 'measured on',
+];
+
+const PERCENT_RE = /\b(\d{1,3}(?:\.\d+)?)\s?%/g;
+// "perfect" / "100%" framing is the specific retracted claim — always high severity.
+// NOTE: no trailing \b after "%": "%"→" " is non-word→non-word, so a trailing \b
+// never matches and would silently miss "100%". Bare 100% is only damning next to a
+// metric term (see claimCheck); the word phrases are inherently accuracy claims.
+const PERFECT_PCT_RE = /\b100(?:\.0+)?\s?%/;
+const PERFECT_WORD_RE = /perfect accuracy|flawless|never (?:wrong|fails)/i;
+
+/**
+ * Lint a block of text for untagged or overstated accuracy claims.
+ * Pure and never-throws: a non-string or empty input returns `{ ok: true, findings: [] }`.
+ */
+export function claimCheck(text: string): ClaimCheckResult {
+  const findings: ClaimFinding[] = [];
+  if (typeof text !== 'string' || text.length === 0) {
+    return { ok: true, findings };
+  }
+  const lines = text.split(/\r?\n/);
+
+  lines.forEach((raw, i) => {
+    const original = raw.trim();
+    if (!original) return;
+    // Analyse the URL-stripped line, but report the ORIGINAL so the excerpt stays recognisable.
+    const line = stripUrls(original);
+    if (!line.trim()) return;
+    const lower = line.toLowerCase();
+
+    const hasPercent = PERCENT_RE.test(line);
+    PERCENT_RE.lastIndex = 0; // reset stateful global regex
+    const scrubbed = scrubLine(lower);
+    // DELIBERATE DIVERGENCE from ruview, which matched METRIC_TERMS against the UNSCRUBBED line.
+    // dz's vocabulary contains path-like words ('packages', 'commands', 'skills'), so a code span
+    // such as `packages/x/usage.test.ts:42` registered as a metric mention and produced a false
+    // positive on prose that merely cites a file. Matching the code-span-scrubbed line fixes it:
+    // "accuracy reached `0.95`" still fires because 'accuracy' sits OUTSIDE the span, while the
+    // number check below deliberately keeps reading the un-code-scrubbed line so `0.95` counts.
+    const mentionsMetric = mentionsMetricTerm(scrubbed, scrubbed);
+    if (!hasPercent && !mentionsMetric) return;
+
+    const tagged = HONEST_TAGS.some((t) => lower.includes(t));
+    const hasReproducer = REPRODUCER_HINTS.some((h) => lower.includes(h));
+    const perfect = PERFECT_WORD_RE.test(line) || (mentionsMetric && PERFECT_PCT_RE.test(line));
+
+    if (perfect && !lower.includes('retract')) {
+      findings.push({
+        severity: 'high',
+        line: i + 1,
+        excerpt: clip(original),
+        reason: 'States perfect/100% accuracy — this is the exact framing the Integrity Rule forbids.',
+        suggestion: 'Replace with a measured number vs a baseline, tagged MEASURED (name the reproducer: npm test, coverage report), or mark the old claim "retracted".',
+      });
+      return;
+    }
+
+    // A quantitative claim needs a number. Digits hidden in a code span still
+    // count — "accuracy reached `0.95`" is a claim — so test the line with only
+    // finding/option labels stripped, NOT the code-span-scrubbed copy: scrubbing
+    // dropped `0.95` and wrongly short-circuited both the untagged and the
+    // MEASURED-without-reproducer checks below. A bare metric word in prose
+    // ("precision matters here", "every accuracy number must be MEASURED") has no
+    // number and is not a taggable claim (ADR-263 F11).
+    if (!hasPercent && !HAS_NUMBER_RE.test(lower.replace(LABEL_TOKEN_RE, ' '))) return;
+
+    // A metric/percent with no honesty tag at all.
+    if (!tagged) {
+      findings.push({
+        severity: 'medium',
+        line: i + 1,
+        excerpt: clip(original),
+        reason: 'Accuracy claim is not tagged MEASURED / CLAIMED / SYNTHETIC / ESTIMATED.',
+        suggestion: 'Tag it. If MEASURED, name the reproducer (npm test, coverage report, git rev, npm view).',
+      });
+      return;
+    }
+
+    // Tagged MEASURED but cites no reproducer — still a gap (reached now even
+    // when the only number is inside a code span, e.g. "accuracy `0.97` (MEASURED)").
+    if (lower.includes('measured') && !hasReproducer) {
+      findings.push({
+        severity: 'medium',
+        line: i + 1,
+        excerpt: clip(original),
+        reason: 'Tagged MEASURED but cites no reproducer/evidence.',
+        suggestion: 'Add the evidence path: npm test output, a coverage report, npm view, or a git rev/commit.',
+      });
+    }
+  });
+
+  return { ok: findings.length === 0, findings };
+}
+
+function clip(s: string, n = 120): string {
+  return s.length > n ? `${s.slice(0, n - 1)}…` : s;
+}
+
+/** Convenience: a one-line human summary for CLI output. */
+export function summarize(result: ClaimCheckResult): string {
+  if (result.ok) return 'claim-check: PASS — no untagged or overstated accuracy claims.';
+  const high = result.findings.filter((f) => f.severity === 'high').length;
+  return `claim-check: ${result.findings.length} finding(s) (${high} high) — accuracy claims need MEASURED/CLAIMED tags + a reproducer.`;
+}

package/src/index.ts

@@ -119,6 +119,8 @@ export type { PluginManifest } from './plugin.js';
 export type { SetupOptions, SetupResult, SetupStep } from './setup.js';
 export type { PretrainResult, DetectedTech } from './pretrain.js';
 export type { RecommendationReport, SkillRecommendation } from './recommend.js';
+export { claimCheck, summarize } from './claim-check.js';
+export type { ClaimFinding, ClaimCheckResult } from './claim-check.js';
 export { discoverPackages, publishPackages, bumpPatch, compareVersions, findUnpackagedSkills, orderByDependencies, syncReadmeVersion } from './publish.js';
 export { fetchAllDownloads } from './downloads.js';
 export type { PackageDownloads, DownloadsReport } from './downloads.js';
@@ -198,5 +200,25 @@ export {
   CODE_LANDED_BARRIER_SLEEPS_SECONDS,
 } from './feature-adr-routing.js';
 export type { StageOpts, RoutingEnv, UsageSignal, UsageAction } from './feature-adr-routing.js';
-export { computeUsage, readUsageLimits } from './usage.js';
-export type { UsageEstimate, UsageLimits } from './usage.js';
+export {
+  CLAUDE_USAGE_MODELS,
+  computeUsage,
+  deriveUsageCalibration,
+  fixedBlockWindowFor,
+  normalizeClaudeUsageModel,
+  normalizeClaudeUsageModelKey,
+  parseWeeklyResetAnchor,
+  readUsageLimits,
+  weeklyWindowFor,
+} from './usage.js';
+export type {
+  ClaudeUsageModel,
+  UsageCalibrationChange,
+  UsageCalibrationInput,
+  UsageCalibrationPlan,
+  UsageEstimate,
+  UsageLimits,
+  UsageModelEstimate,
+  UsageWindow,
+  WeeklyResetAnchor,
+} from './usage.js';

package/src/publish.ts

@@ -9,6 +9,8 @@ import { existsSync, readFileSync, writeFileSync, readdirSync } from 'node:fs';
 import { join } from 'node:path';
 import { execSync } from 'node:child_process';
 
+import { claimCheck } from './claim-check.js';
+
 /** Result for a single package publish attempt. */
 export interface PublishResult {
   readonly name: string;
@@ -16,6 +18,12 @@ export interface PublishResult {
   readonly newVersion: string;
   readonly status: 'published' | 'skipped' | 'error';
   readonly error?: string | undefined;
+  /**
+   * Pre-publish claim-check summary for this package's README, present only when the
+   * opt-in `claimCheck` gate ran (`'warn'`/`'block'`). Additive: absent by default so an
+   * unmodified `publishPackages` call is byte-compatible with pre-gate behavior.
+   */
+  readonly claimCheck?: { readonly findings: number; readonly high: number } | undefined;
 }
 
 /** Full publish report. */
@@ -203,6 +211,14 @@ export function publishPackages(
     dryRun?: boolean | undefined;
     filter?: string[] | undefined;
     bumpOnly?: boolean | undefined;
+    /**
+     * Pre-publish claim-check gate over each package's README (ADR-001). Default `'warn'`:
+     * records the finding count on the result but NEVER changes publish status — additive, so the
+     * existing publish path and its tests are unaffected. `'error'` flips ONLY a package with a
+     * `high` finding to `status: 'error'`, leaving the rest of the batch unaffected. `'off'`
+     * disables the gate entirely (no `claimCheck` field is emitted).
+     */
+    claimGate?: 'off' | 'warn' | 'error' | undefined;
   } = {},
 ): PublishReport {
   const packages = discoverPackages(monorepoRoot);
@@ -238,8 +254,41 @@ export function publishPackages(
       continue;
     }
 
+    // Pre-publish claim-check gate. Default `'warn'` per ADR-001: publishing SURFACES a
+    // README's untagged claims by default, but `'warn'` NEVER changes publish status, so the
+    // existing publish path is unaffected. `'error'` fails only THIS package when it carries a
+    // high-severity claim; `'off'` disables the gate entirely (no `claimCheck` field emitted).
+    // Runs BEFORE the dry-run short-circuit so `--dry-run` previews surface what a live publish
+    // would. Reading the README never blocks the gate itself — unreadable ⇒ "no findings".
+    const claimGate = opts.claimGate ?? 'warn';
+    let claimCheckSummary: { findings: number; high: number } | undefined;
+    if (claimGate !== 'off') {
+      const readmePath = join(pkg.dir, 'README.md');
+      if (existsSync(readmePath)) {
+        try {
+          const text = readFileSync(readmePath, 'utf-8');
+          const result = claimCheck(text);
+          const high = result.findings.filter((f) => f.severity === 'high').length;
+          claimCheckSummary = { findings: result.findings.length, high };
+          if (claimGate === 'error' && high > 0) {
+            results.push({
+              name: pkg.name,
+              oldVersion,
+              newVersion,
+              status: 'error',
+              error: `claim-check: ${high} high-severity claim(s) in README.md — tag MEASURED with a reproducer or CLAIMED/SYNTHETIC before publishing.`,
+              claimCheck: claimCheckSummary,
+            });
+            continue;
+          }
+        } catch {
+          /* unreadable README never blocks the gate itself */
+        }
+      }
+    }
+
     if (opts.dryRun) {
-      results.push({ name: pkg.name, oldVersion, newVersion, status: 'skipped' });
+      results.push({ name: pkg.name, oldVersion, newVersion, status: 'skipped', claimCheck: claimCheckSummary });
       continue;
     }
 
@@ -254,7 +303,7 @@ export function publishPackages(
       originalReadme = syncReadmeVersion(pkg.dir, oldVersion, newVersion);
 
       if (opts.bumpOnly) {
-        results.push({ name: pkg.name, oldVersion, newVersion, status: 'published' });
+        results.push({ name: pkg.name, oldVersion, newVersion, status: 'published', claimCheck: claimCheckSummary });
         continue;
       }
 
@@ -272,7 +321,7 @@ export function publishPackages(
         env: { ...process.env },
       });
 
-      results.push({ name: pkg.name, oldVersion, newVersion, status: 'published' });
+      results.push({ name: pkg.name, oldVersion, newVersion, status: 'published', claimCheck: claimCheckSummary });
     } catch (err) {
       // The version was written BEFORE build+publish; on any failure restore the
       // original package.json (and README, if we rewrote its version) so a failed
@@ -288,6 +337,7 @@ export function publishPackages(
         newVersion,
         status: 'error',
         error: err instanceof Error ? err.message : String(err),
+        claimCheck: claimCheckSummary,
       });
     }
   }