@criterionx/testing 0.3.1

package/README.md

@@ -0,0 +1,189 @@
+# @criterionx/testing
+
+Testing utilities for Criterion decisions. Provides property-based testing, fuzzing, and coverage analysis.
+
+## Installation
+
+```bash
+npm install @criterionx/testing
+# or
+pnpm add @criterionx/testing
+```
+
+## Features
+
+- **testDecision** - Test decisions with expected outcomes
+- **fuzz** - Fuzz testing with random inputs
+- **checkProperty** - Property-based testing with fast-check
+- **coverage** - Rule coverage analysis
+- **detectDeadRules** - Dead code detection
+
+## Usage
+
+### Testing Decisions
+
+```typescript
+import { testDecision } from "@criterionx/testing";
+import { riskDecision } from "./decisions";
+
+const result = testDecision(riskDecision, {
+  profile: defaultProfile,
+  cases: [
+    {
+      name: "high risk transaction",
+      input: { amount: 50000, country: "US" },
+      expected: { status: "OK", ruleId: "high-amount" },
+    },
+    {
+      name: "blocked country",
+      input: { amount: 100, country: "NK" },
+      expected: { status: "OK", ruleId: "blocked-country" },
+    },
+  ],
+  expect: {
+    noUnreachableRules: true,
+  },
+});
+
+console.log(result.passed); // true or false
+console.log(result.failures); // Array of test failures
+console.log(result.rulesCovered); // Rules that were exercised
+```
+
+### Fuzz Testing
+
+```typescript
+import { fuzz, fc } from "@criterionx/testing";
+
+const result = fuzz(riskDecision, {
+  profile: defaultProfile,
+  iterations: 1000,
+  inputArbitrary: fc.record({
+    amount: fc.integer({ min: 0, max: 100000 }),
+    country: fc.constantFrom("US", "UK", "NK", "DE"),
+  }),
+});
+
+console.log(result.totalRuns); // 1000
+console.log(result.failed); // Number of failures
+console.log(result.ruleDistribution); // Hits per rule
+```
+
+### Property-Based Testing
+
+```typescript
+import { checkProperty, fc } from "@criterionx/testing";
+
+const result = checkProperty(riskDecision, {
+  profile: defaultProfile,
+  inputArbitrary: fc.record({
+    amount: fc.integer({ min: 0, max: 100000 }),
+    country: fc.constantFrom("US", "UK"),
+  }),
+  property: (input, result) => {
+    // Property: valid inputs should never return INVALID_OUTPUT
+    return result.status !== "INVALID_OUTPUT";
+  },
+  numRuns: 100,
+});
+
+if (!result.passed) {
+  console.log("Property violated with:", result.counterExample);
+}
+```
+
+### Coverage Analysis
+
+```typescript
+import { coverage, formatCoverageReport, meetsCoverageThreshold } from "@criterionx/testing";
+
+const report = coverage(riskDecision, {
+  profile: defaultProfile,
+  testCases: [
+    { input: { amount: 100, country: "NK" } },
+    { input: { amount: 50000, country: "US" } },
+    { input: { amount: 100, country: "US" } },
+  ],
+});
+
+console.log(formatCoverageReport(report));
+// === Rule Coverage Report ===
+// Coverage: 3/3 rules (100.0%)
+// Covered rules:
+//   ✓ blocked-country (1 hits)
+//   ✓ high-amount (1 hits)
+//   ✓ default (1 hits)
+
+if (!meetsCoverageThreshold(report, 80)) {
+  throw new Error("Coverage below 80%");
+}
+```
+
+### Dead Code Detection
+
+```typescript
+import { detectDeadRules } from "@criterionx/testing";
+
+const deadRules = detectDeadRules(myDecision);
+if (deadRules.length > 0) {
+  console.warn("Dead rules detected:", deadRules);
+}
+```
+
+## API Reference
+
+### `testDecision(decision, options)`
+
+Test a decision with provided test cases.
+
+**Options:**
+- `profile` - Profile to use for evaluation
+- `cases` - Array of test cases with input, profile, and expected outcomes
+- `expect.noUnreachableRules` - Fail if any rules aren't covered
+
+**Returns:** `TestDecisionResult` with `passed`, `failures`, `rulesCovered`, `rulesUncovered`
+
+### `fuzz(decision, options)`
+
+Run fuzz tests on a decision.
+
+**Options:**
+- `profile` - Profile to use
+- `iterations` - Number of random inputs (default: 100)
+- `inputArbitrary` - fast-check arbitrary for generating inputs
+- `seed` - Seed for reproducibility
+
+**Returns:** `FuzzResult` with `totalRuns`, `passed`, `failed`, `errors`, `ruleDistribution`
+
+### `checkProperty(decision, options)`
+
+Run property-based tests.
+
+**Options:**
+- `profile` - Profile to use
+- `inputArbitrary` - fast-check arbitrary for inputs
+- `property` - Function that returns true if property holds
+- `numRuns` - Number of test runs (default: 100)
+- `seed` - Seed for reproducibility
+
+**Returns:** `{ passed, counterExample?, error? }`
+
+### `coverage(decision, options)`
+
+Analyze rule coverage.
+
+**Options:**
+- `profile` - Profile to use
+- `testCases` - Array of inputs to test
+
+**Returns:** `CoverageReport` with coverage percentage and rule hit counts
+
+### `detectDeadRules(decision)`
+
+Detect potentially unreachable rules.
+
+**Returns:** Array of rule IDs that appear after catch-all rules
+
+## License
+
+MIT

package/dist/index.d.ts

@@ -0,0 +1,157 @@
+import { Decision, Engine } from '@criterionx/core';
+import * as fc from 'fast-check';
+import { Arbitrary } from 'fast-check';
+export { fc };
+
+/**
+ * Test case for a decision
+ */
+interface TestCase<TInput, TOutput, TProfile> {
+    name?: string;
+    input: TInput;
+    profile: TProfile;
+    expected?: {
+        status?: "OK" | "NO_MATCH" | "INVALID_INPUT" | "INVALID_OUTPUT";
+        ruleId?: string;
+        output?: Partial<TOutput>;
+    };
+}
+/**
+ * Options for testDecision
+ */
+interface TestDecisionOptions<TInput, TOutput, TProfile> {
+    profile: TProfile;
+    cases?: TestCase<TInput, TOutput, TProfile>[];
+    expect?: {
+        /** All rules should be reachable (have at least one matching case) */
+        noUnreachableRules?: boolean;
+        /** No rules should always fail */
+        noDeadCode?: boolean;
+    };
+}
+/**
+ * Result of testDecision
+ */
+interface TestDecisionResult {
+    passed: boolean;
+    failures: TestFailure[];
+    rulesCovered: string[];
+    rulesUncovered: string[];
+}
+/**
+ * A test failure
+ */
+interface TestFailure {
+    type: "case_failed" | "unreachable_rule" | "dead_code";
+    message: string;
+    details?: {
+        testCase?: string;
+        ruleId?: string;
+        expected?: unknown;
+        actual?: unknown;
+    };
+}
+/**
+ * Options for fuzz testing
+ */
+interface FuzzOptions<TInput, TProfile> {
+    /** Number of iterations */
+    iterations?: number;
+    /** Profile to use */
+    profile: TProfile;
+    /** Custom input arbitrary (for property-based testing) */
+    inputArbitrary?: Arbitrary<TInput>;
+    /** Seed for reproducibility */
+    seed?: number;
+}
+/**
+ * Result of fuzz testing
+ */
+interface FuzzResult<TInput> {
+    totalRuns: number;
+    passed: number;
+    failed: number;
+    errors: FuzzError<TInput>[];
+    ruleDistribution: Record<string, number>;
+}
+/**
+ * A fuzz error
+ */
+interface FuzzError<TInput> {
+    input: TInput;
+    error: string;
+    iteration: number;
+}
+/**
+ * Options for coverage analysis
+ */
+interface CoverageOptions<TInput, TProfile> {
+    profile: TProfile;
+    testCases?: Array<{
+        input: TInput;
+    }>;
+}
+/**
+ * Coverage report
+ */
+interface CoverageReport {
+    totalRules: number;
+    coveredRules: number;
+    coveragePercentage: number;
+    rulesCovered: string[];
+    rulesUncovered: string[];
+    ruleHits: Record<string, number>;
+}
+
+/**
+ * Test a decision with provided test cases and assertions
+ */
+declare function testDecision<TInput, TOutput, TProfile>(decision: Decision<TInput, TOutput, TProfile>, options: TestDecisionOptions<TInput, TOutput, TProfile>): TestDecisionResult;
+
+/**
+ * Fuzz test a decision with random inputs
+ *
+ * Uses fast-check for property-based testing to discover edge cases
+ */
+declare function fuzz<TInput, TOutput, TProfile>(decision: Decision<TInput, TOutput, TProfile>, options: FuzzOptions<TInput, TProfile>): FuzzResult<TInput>;
+/**
+ * Run property-based tests on a decision
+ *
+ * Allows defining custom properties that should hold for all inputs
+ */
+declare function checkProperty<TInput, TOutput, TProfile>(decision: Decision<TInput, TOutput, TProfile>, options: {
+    profile: TProfile;
+    inputArbitrary: fc.Arbitrary<TInput>;
+    property: (input: TInput, result: ReturnType<Engine["run"]>) => boolean;
+    numRuns?: number;
+    seed?: number;
+}): {
+    passed: boolean;
+    counterExample?: TInput;
+    error?: string;
+};
+
+/**
+ * Analyze rule coverage for a decision
+ *
+ * Reports which rules have been exercised by the provided test cases
+ */
+declare function coverage<TInput, TOutput, TProfile>(decision: Decision<TInput, TOutput, TProfile>, options: CoverageOptions<TInput, TProfile>): CoverageReport;
+/**
+ * Generate a coverage report as a formatted string
+ */
+declare function formatCoverageReport(report: CoverageReport): string;
+/**
+ * Check if coverage meets a threshold
+ */
+declare function meetsCoverageThreshold(report: CoverageReport, threshold: number): boolean;
+/**
+ * Analyze which rules are potentially dead code
+ *
+ * A rule is considered potentially dead if:
+ * 1. It was never matched in fuzzing
+ * 2. It comes after a rule that always matches (like a catch-all)
+ */
+declare function detectDeadRules<TInput, TOutput, TProfile>(decision: Decision<TInput, TOutput, TProfile>): string[];
+
+export { type CoverageOptions, type CoverageReport, type FuzzError, type FuzzOptions, type FuzzResult, type TestCase, type TestDecisionOptions, type TestDecisionResult, type TestFailure, checkProperty, coverage, detectDeadRules, formatCoverageReport, fuzz, meetsCoverageThreshold, testDecision };

package/dist/index.js

@@ -0,0 +1,305 @@
+// src/test-decision.ts
+import { Engine } from "@criterionx/core";
+var engine = new Engine();
+function testDecision(decision, options) {
+  const failures = [];
+  const ruleHits = /* @__PURE__ */ new Map();
+  for (const rule of decision.rules) {
+    ruleHits.set(rule.id, 0);
+  }
+  if (options.cases) {
+    for (const testCase of options.cases) {
+      const profile = testCase.profile ?? options.profile;
+      const result = engine.run(decision, testCase.input, { profile });
+      if (result.status === "OK" && result.meta.matchedRule) {
+        const current = ruleHits.get(result.meta.matchedRule) ?? 0;
+        ruleHits.set(result.meta.matchedRule, current + 1);
+      }
+      if (testCase.expected) {
+        const caseFailures = validateExpectations(
+          testCase,
+          result,
+          testCase.name
+        );
+        failures.push(...caseFailures);
+      }
+    }
+  }
+  if (options.expect?.noUnreachableRules) {
+    for (const [ruleId, hits] of ruleHits) {
+      if (hits === 0) {
+        failures.push({
+          type: "unreachable_rule",
+          message: `Rule "${ruleId}" was never matched by any test case`,
+          details: { ruleId }
+        });
+      }
+    }
+  }
+  const rulesCovered = [...ruleHits.entries()].filter(([, hits]) => hits > 0).map(([id]) => id);
+  const rulesUncovered = [...ruleHits.entries()].filter(([, hits]) => hits === 0).map(([id]) => id);
+  return {
+    passed: failures.length === 0,
+    failures,
+    rulesCovered,
+    rulesUncovered
+  };
+}
+function validateExpectations(testCase, result, caseName) {
+  const failures = [];
+  const { expected } = testCase;
+  const caseLabel = caseName ?? JSON.stringify(testCase.input);
+  if (!expected) return failures;
+  if (expected.status && result.status !== expected.status) {
+    failures.push({
+      type: "case_failed",
+      message: `Test case "${caseLabel}" expected status "${expected.status}" but got "${result.status}"`,
+      details: {
+        testCase: caseLabel,
+        expected: expected.status,
+        actual: result.status
+      }
+    });
+  }
+  if (expected.ruleId && result.meta.matchedRule !== expected.ruleId) {
+    failures.push({
+      type: "case_failed",
+      message: `Test case "${caseLabel}" expected rule "${expected.ruleId}" but got "${result.meta.matchedRule ?? "none"}"`,
+      details: {
+        testCase: caseLabel,
+        expected: expected.ruleId,
+        actual: result.meta.matchedRule
+      }
+    });
+  }
+  if (expected.output && result.data) {
+    for (const [key, value] of Object.entries(expected.output)) {
+      const actualValue = result.data[key];
+      if (!deepEqual(actualValue, value)) {
+        failures.push({
+          type: "case_failed",
+          message: `Test case "${caseLabel}" expected output.${key} to be ${JSON.stringify(value)} but got ${JSON.stringify(actualValue)}`,
+          details: {
+            testCase: caseLabel,
+            expected: { [key]: value },
+            actual: { [key]: actualValue }
+          }
+        });
+      }
+    }
+  }
+  return failures;
+}
+function deepEqual(a, b) {
+  if (a === b) return true;
+  if (typeof a !== typeof b) return false;
+  if (typeof a !== "object" || a === null || b === null) return false;
+  const keysA = Object.keys(a);
+  const keysB = Object.keys(b);
+  if (keysA.length !== keysB.length) return false;
+  for (const key of keysA) {
+    if (!keysB.includes(key)) return false;
+    if (!deepEqual(
+      a[key],
+      b[key]
+    ))
+      return false;
+  }
+  return true;
+}
+
+// src/fuzz.ts
+import { Engine as Engine2 } from "@criterionx/core";
+import * as fc from "fast-check";
+var engine2 = new Engine2();
+function fuzz(decision, options) {
+  const iterations = options.iterations ?? 100;
+  const errors = [];
+  const ruleDistribution = {};
+  let passed = 0;
+  let failed = 0;
+  for (const rule of decision.rules) {
+    ruleDistribution[rule.id] = 0;
+  }
+  ruleDistribution["NO_MATCH"] = 0;
+  ruleDistribution["INVALID_INPUT"] = 0;
+  const inputArbitrary = options.inputArbitrary ?? createDefaultArbitrary();
+  const inputs = [];
+  const seed = options.seed ?? Date.now();
+  fc.assert(
+    fc.property(inputArbitrary, (input) => {
+      inputs.push(input);
+      return true;
+    }),
+    {
+      numRuns: iterations,
+      seed
+    }
+  );
+  for (let i = 0; i < inputs.length; i++) {
+    const input = inputs[i];
+    try {
+      const result = engine2.run(decision, input, { profile: options.profile });
+      if (result.status === "OK" && result.meta.matchedRule) {
+        ruleDistribution[result.meta.matchedRule]++;
+        passed++;
+      } else if (result.status === "NO_MATCH") {
+        ruleDistribution["NO_MATCH"]++;
+        passed++;
+      } else if (result.status === "INVALID_INPUT") {
+        ruleDistribution["INVALID_INPUT"]++;
+        passed++;
+      } else {
+        failed++;
+        errors.push({
+          input,
+          error: result.meta.explanation,
+          iteration: i
+        });
+      }
+    } catch (error) {
+      failed++;
+      errors.push({
+        input,
+        error: String(error),
+        iteration: i
+      });
+    }
+  }
+  return {
+    totalRuns: inputs.length,
+    passed,
+    failed,
+    errors,
+    ruleDistribution
+  };
+}
+function checkProperty(decision, options) {
+  try {
+    fc.assert(
+      fc.property(options.inputArbitrary, (input) => {
+        const result = engine2.run(decision, input, {
+          profile: options.profile
+        });
+        return options.property(input, result);
+      }),
+      {
+        numRuns: options.numRuns ?? 100,
+        seed: options.seed
+      }
+    );
+    return { passed: true };
+  } catch (error) {
+    if (error instanceof Error) {
+      const fcError = error;
+      return {
+        passed: false,
+        counterExample: fcError.counterexample?.[0],
+        error: fcError.message
+      };
+    }
+    return {
+      passed: false,
+      error: String(error)
+    };
+  }
+}
+function createDefaultArbitrary() {
+  return fc.dictionary(
+    fc.string({ minLength: 1, maxLength: 20 }),
+    fc.oneof(
+      fc.string(),
+      fc.integer(),
+      fc.double({ noNaN: true }),
+      fc.boolean(),
+      fc.constant(null)
+    )
+  );
+}
+
+// src/coverage.ts
+import { Engine as Engine3 } from "@criterionx/core";
+var engine3 = new Engine3();
+function coverage(decision, options) {
+  const ruleHits = {};
+  const testCases = options.testCases ?? [];
+  for (const rule of decision.rules) {
+    ruleHits[rule.id] = 0;
+  }
+  for (const testCase of testCases) {
+    const result = engine3.run(decision, testCase.input, {
+      profile: options.profile
+    });
+    if (result.status === "OK" && result.meta.matchedRule) {
+      ruleHits[result.meta.matchedRule]++;
+    }
+  }
+  const totalRules = decision.rules.length;
+  const rulesCovered = Object.entries(ruleHits).filter(([, hits]) => hits > 0).map(([id]) => id);
+  const rulesUncovered = Object.entries(ruleHits).filter(([, hits]) => hits === 0).map(([id]) => id);
+  const coveredRules = rulesCovered.length;
+  const coveragePercentage = totalRules > 0 ? coveredRules / totalRules * 100 : 100;
+  return {
+    totalRules,
+    coveredRules,
+    coveragePercentage,
+    rulesCovered,
+    rulesUncovered,
+    ruleHits
+  };
+}
+function formatCoverageReport(report) {
+  const lines = [];
+  lines.push("=== Rule Coverage Report ===");
+  lines.push("");
+  lines.push(
+    `Coverage: ${report.coveredRules}/${report.totalRules} rules (${report.coveragePercentage.toFixed(1)}%)`
+  );
+  lines.push("");
+  if (report.rulesCovered.length > 0) {
+    lines.push("Covered rules:");
+    for (const ruleId of report.rulesCovered) {
+      const hits = report.ruleHits[ruleId];
+      lines.push(`  \u2713 ${ruleId} (${hits} hits)`);
+    }
+  }
+  if (report.rulesUncovered.length > 0) {
+    lines.push("");
+    lines.push("Uncovered rules:");
+    for (const ruleId of report.rulesUncovered) {
+      lines.push(`  \u2717 ${ruleId}`);
+    }
+  }
+  return lines.join("\n");
+}
+function meetsCoverageThreshold(report, threshold) {
+  return report.coveragePercentage >= threshold;
+}
+function detectDeadRules(decision) {
+  const deadRules = [];
+  let foundCatchAll = false;
+  for (const rule of decision.rules) {
+    if (foundCatchAll) {
+      deadRules.push(rule.id);
+      continue;
+    }
+    const whenStr = rule.when.toString();
+    if (whenStr.includes("=> true") || whenStr.includes("return true") || whenStr.includes("()=>true")) {
+      foundCatchAll = true;
+    }
+  }
+  return deadRules;
+}
+
+// src/index.ts
+import * as fc2 from "fast-check";
+export {
+  checkProperty,
+  coverage,
+  detectDeadRules,
+  fc2 as fc,
+  formatCoverageReport,
+  fuzz,
+  meetsCoverageThreshold,
+  testDecision
+};

package/package.json

@@ -0,0 +1,59 @@
+{
+  "name": "@criterionx/testing",
+  "version": "0.3.1",
+  "description": "Testing utilities for Criterion decisions",
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "files": [
+    "dist",
+    "LICENSE",
+    "README.md"
+  ],
+  "scripts": {
+    "build": "tsup src/index.ts --format esm --dts --clean",
+    "dev": "tsup src/index.ts --format esm --dts --watch",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "test:coverage": "vitest run --coverage",
+    "typecheck": "tsc --noEmit"
+  },
+  "keywords": [
+    "criterion",
+    "testing",
+    "decision-engine",
+    "fuzzing",
+    "property-based-testing"
+  ],
+  "author": {
+    "name": "Tomas Maritano",
+    "url": "https://github.com/tomymaritano"
+  },
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/tomymaritano/criterionx.git",
+    "directory": "packages/testing"
+  },
+  "bugs": {
+    "url": "https://github.com/tomymaritano/criterionx/issues"
+  },
+  "homepage": "https://github.com/tomymaritano/criterionx#readme",
+  "dependencies": {
+    "@criterionx/core": "workspace:*",
+    "fast-check": "^3.23.0"
+  },
+  "devDependencies": {
+    "@types/node": "^20.0.0",
+    "tsup": "^8.0.0",
+    "typescript": "^5.3.0",
+    "vitest": "^4.0.0",
+    "zod": "^3.23.0"
+  },
+  "peerDependencies": {
+    "zod": "^3.0.0"
+  },
+  "engines": {
+    "node": ">=18"
+  }
+}