@futux/whitebox-dedup 0.0.0-stage → 0.1.1

package/README.md

@@ -1,3 +1,103 @@
-# Temporary Holding Version
+# @futux/whitebox-dedup (Interpretable Entity Deduplication in TypeScript)
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+[![npm version](https://img.shields.io/npm/v/@futux/whitebox-dedup.svg?color=0B3541&logo=npm&logoColor=white)](https://www.npmjs.com/package/@futux/whitebox-dedup)
+[![npm downloads](https://img.shields.io/npm/dm/@futux/whitebox-dedup.svg)](https://www.npmjs.com/package/@futux/whitebox-dedup)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![Live Web Demo](https://img.shields.io/badge/Interactive_Playground-Live_Online-0FBFF1?style=flat&logo=google-chrome&logoColor=white)](https://futuxagriconsult.github.io/whitebox-dedup-ml/)
+
+> **Auditable, transparent entity deduplication and record linkage with Multivariate Adaptive Regression Splines (MARS) and Inductive Conformal Prediction (ICP).** High-performance, zero-native-dependency TypeScript library for Node.js, Next.js, and browser Web Workers.
+
+---
+
+### 🌐 Try the Interactive Web Playground
+
+Test `@futux/whitebox-dedup` live directly in your browser:  
+👉 **[Launch Live Web Demo (https://futuxagriconsult.github.io/whitebox-dedup-ml/)](https://futuxagriconsult.github.io/whitebox-dedup-ml/)**
+
+---
+
+## 🚀 Key Features
+
+- **Auditable MARS Scoring:** No black-box neural networks. Outputs clear hinge-function math explaining precisely why two entities matched.
+- **Three Conformal Statistical Tiers:**
+  - **Tier 1 (Auto-Merge, $\ge 0.85$):** Conformal $p$-value $\ge 0.95$; statistically safe for automated merging.
+  - **Tier 2 (Circuit Breaker / Review, $0.60 - 0.84$):** Ambiguous boundary cases routed to human adjudicators with exact contributing features.
+  - **Tier 3 (Auto-Reject, $< 0.60$):** Distinct records; kept separate.
+- **Client & Server Compatible:** Runs in Node.js, Bun, Deno, Next.js API routes, and client-side Web Workers.
+- **Zero Heavy Native Dependencies:** Fast cold-start, pure TypeScript, instant browser compilation.
+
+---
+
+## 📦 Installation
+
+```bash
+# npm
+npm install @futux/whitebox-dedup
+
+# yarn
+yarn add @futux/whitebox-dedup
+
+# pnpm
+pnpm add @futux/whitebox-dedup
+```
+
+---
+
+## 💡 Quick Start
+
+```typescript
+import { DeduplicationPipeline } from "@futux/whitebox-dedup";
+
+const records = [
+  { id: "1", name: "Ibrahim Musa", phone: "+254712345678" },
+  { id: "2", name: "Ibraheem Mussa", phone: "+254712345678" },
+  { id: "3", name: "Sarah Cherono", phone: "+254787654321" }
+];
+
+const pipeline = new DeduplicationPipeline({
+  model: "text_phone_v1",
+  confidence: 0.95
+});
+
+const results = pipeline.run(records);
+
+results.matches.forEach(match => {
+  console.log(`Pair: ${match.recordA.name} <-> ${match.recordB.name}`);
+  console.log(`Score: ${(match.score * 100).toFixed(1)}% | Tier: ${match.tier}`);
+  console.log(`Auditable Basis:`, match.explanation);
+});
+```
+
+---
+
+## 🔬 How the Math Works
+
+The score is calculated via transparent piecewise linear splines:
+
+```text
+Score = 1.0 / (1.0 + exp(-(
+    -3.8421
+    + 5.1204 * max(0, jaro_winkler - 0.7200)
+    - 2.8910 * max(0, 0.7200 - jaro_winkler)
+    + 1.8432 * max(0, token_sort_ratio - 0.6500)
+    + 0.9420 * exact_phone_match
+)))
+```
+
+Every prediction can be audited, reviewed, and justified in regulatory or administrative hearings.
+
+---
+
+## 🐍 Also Available in Python
+
+For Python data pipelines and pandas integration, install the companion package:
+```bash
+pip install whitebox-dedup
+```
+See [GitHub Repository](https://github.com/futuxagriconsult/whitebox-dedup-ml) for CLI and calibration tools.
+
+---
+
+## 📄 License
+
+MIT. Developed and maintained by **FutuX Agri-consult Limited** ([info@futuxconsult.com](mailto:info@futuxconsult.com)).

package/dist/conformal.d.ts

@@ -0,0 +1,12 @@
+export interface ICPCalibrationData {
+    model_name: string;
+    alpha: number;
+    threshold_duplicate: number;
+    threshold_distinct: number;
+    description?: string;
+}
+export declare class ICPPredictor {
+    cal: ICPCalibrationData;
+    constructor(cal: ICPCalibrationData);
+    predictSet(probability: number): Set<number>;
+}

package/dist/conformal.js

@@ -0,0 +1,27 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ICPPredictor = void 0;
+class ICPPredictor {
+    cal;
+    constructor(cal) {
+        this.cal = cal;
+    }
+    predictSet(probability) {
+        const nonconformityDup = 1.0 - probability;
+        const nonconformityDist = probability;
+        const includeDup = nonconformityDup <= this.cal.threshold_duplicate;
+        const includeDist = nonconformityDist <= this.cal.threshold_distinct;
+        const set = new Set();
+        if (includeDist)
+            set.add(0);
+        if (includeDup)
+            set.add(1);
+        if (set.size === 0) {
+            // Empty set safeguard -> ambiguous
+            set.add(0);
+            set.add(1);
+        }
+        return set;
+    }
+}
+exports.ICPPredictor = ICPPredictor;

package/dist/features.d.ts

@@ -0,0 +1,3 @@
+export declare function jaroWinkler(s1: string, s2: string): number;
+export declare function haversineM(lat1: number, lon1: number, lat2: number, lon2: number): number;
+export declare function extractPairFeatures(a: Record<string, string | number | null | undefined>, b: Record<string, string | number | null | undefined>): Record<string, number | null>;

package/dist/features.js

@@ -0,0 +1,100 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.jaroWinkler = jaroWinkler;
+exports.haversineM = haversineM;
+exports.extractPairFeatures = extractPairFeatures;
+function jaroWinkler(s1, s2) {
+    if (s1 === s2)
+        return 1.0;
+    const len1 = s1.length;
+    const len2 = s2.length;
+    if (len1 === 0 || len2 === 0)
+        return 0.0;
+    const matchDistance = Math.floor(Math.max(len1, len2) / 2) - 1;
+    const s1Matches = new Array(len1).fill(false);
+    const s2Matches = new Array(len2).fill(false);
+    let matches = 0;
+    for (let i = 0; i < len1; i++) {
+        const start = Math.max(0, i - matchDistance);
+        const end = Math.min(i + matchDistance + 1, len2);
+        for (let j = start; j < end; j++) {
+            if (s2Matches[j] || s1[i] !== s2[j])
+                continue;
+            s1Matches[i] = true;
+            s2Matches[j] = true;
+            matches++;
+            break;
+        }
+    }
+    if (matches === 0)
+        return 0.0;
+    let transpositions = 0;
+    let k = 0;
+    for (let i = 0; i < len1; i++) {
+        if (!s1Matches[i])
+            continue;
+        while (!s2Matches[k])
+            k++;
+        if (s1[i] !== s2[k])
+            transpositions++;
+        k++;
+    }
+    const jaro = (matches / len1 + matches / len2 + (matches - transpositions / 2) / matches) / 3;
+    let prefix = 0;
+    for (let i = 0; i < Math.min(4, Math.min(len1, len2)); i++) {
+        if (s1[i] === s2[i])
+            prefix++;
+        else
+            break;
+    }
+    return jaro + prefix * 0.1 * (1 - jaro);
+}
+function haversineM(lat1, lon1, lat2, lon2) {
+    const R = 6371000.0; // Earth radius in meters
+    const toRad = (deg) => (deg * Math.PI) / 180.0;
+    const dLat = toRad(lat2 - lat1);
+    const dLon = toRad(lon2 - lon1);
+    const a = Math.sin(dLat / 2) * Math.sin(dLat / 2) +
+        Math.cos(toRad(lat1)) * Math.cos(toRad(lat2)) * Math.sin(dLon / 2) * Math.sin(dLon / 2);
+    const c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1 - a));
+    return R * c;
+}
+function extractPairFeatures(a, b) {
+    const feats = {};
+    // Name similarities
+    const nameA = String(a["full_name"] ?? a["name"] ?? "").trim().toLowerCase();
+    const nameB = String(b["full_name"] ?? b["name"] ?? "").trim().toLowerCase();
+    if (nameA && nameB) {
+        feats["jw_name_full"] = jaroWinkler(nameA, nameB);
+        const tokensA = nameA.split(/\s+/);
+        const tokensB = nameB.split(/\s+/);
+        feats["jw_name_first"] = jaroWinkler(tokensA[0] ?? "", tokensB[0] ?? "");
+        feats["jw_name_last"] = jaroWinkler(tokensA[tokensA.length - 1] ?? "", tokensB[tokensB.length - 1] ?? "");
+    }
+    else {
+        feats["jw_name_full"] = null;
+        feats["jw_name_first"] = null;
+        feats["jw_name_last"] = null;
+    }
+    // Exact Phone Match
+    const phoneA = String(a["phone"] ?? a["phone_number"] ?? "").replace(/\D/g, "");
+    const phoneB = String(b["phone"] ?? b["phone_number"] ?? "").replace(/\D/g, "");
+    if (phoneA.length >= 7 && phoneB.length >= 7) {
+        feats["exact_phone"] = phoneA === phoneB ? 1.0 : 0.0;
+    }
+    else {
+        feats["exact_phone"] = null;
+    }
+    // Haversine GPS distance
+    const latA = Number(a["latitude"] ?? a["lat"]);
+    const lonA = Number(a["longitude"] ?? a["lon"]);
+    const latB = Number(b["latitude"] ?? b["lat"]);
+    const lonB = Number(b["longitude"] ?? b["lon"]);
+    if (!isNaN(latA) && !isNaN(lonA) && !isNaN(latB) && !isNaN(lonB)) {
+        feats["haversine_m"] = haversineM(latA, lonA, latB, lonB);
+    }
+    else {
+        feats["haversine_m"] = null;
+    }
+    return feats;
+}

package/dist/index.d.ts

@@ -0,0 +1,9 @@
+export { MARSScorer } from "./mars_scorer.js";
+export * from "./mars_scorer.js";
+export { ICPPredictor } from "./conformal.js";
+export * from "./conformal.js";
+export { DecisionRouter, Tier } from "./router.js";
+export * from "./router.js";
+export { DeduplicationPipeline } from "./pipeline.js";
+export * from "./pipeline.js";
+export { jaroWinkler, haversineM, extractPairFeatures } from "./features.js";

package/dist/index.js

@@ -0,0 +1,34 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.extractPairFeatures = exports.haversineM = exports.jaroWinkler = exports.DeduplicationPipeline = exports.Tier = exports.DecisionRouter = exports.ICPPredictor = exports.MARSScorer = void 0;
+var mars_scorer_js_1 = require("./mars_scorer.js");
+Object.defineProperty(exports, "MARSScorer", { enumerable: true, get: function () { return mars_scorer_js_1.MARSScorer; } });
+__exportStar(require("./mars_scorer.js"), exports);
+var conformal_js_1 = require("./conformal.js");
+Object.defineProperty(exports, "ICPPredictor", { enumerable: true, get: function () { return conformal_js_1.ICPPredictor; } });
+__exportStar(require("./conformal.js"), exports);
+var router_js_1 = require("./router.js");
+Object.defineProperty(exports, "DecisionRouter", { enumerable: true, get: function () { return router_js_1.DecisionRouter; } });
+Object.defineProperty(exports, "Tier", { enumerable: true, get: function () { return router_js_1.Tier; } });
+__exportStar(require("./router.js"), exports);
+var pipeline_js_1 = require("./pipeline.js");
+Object.defineProperty(exports, "DeduplicationPipeline", { enumerable: true, get: function () { return pipeline_js_1.DeduplicationPipeline; } });
+__exportStar(require("./pipeline.js"), exports);
+var features_js_1 = require("./features.js");
+Object.defineProperty(exports, "jaroWinkler", { enumerable: true, get: function () { return features_js_1.jaroWinkler; } });
+Object.defineProperty(exports, "haversineM", { enumerable: true, get: function () { return features_js_1.haversineM; } });
+Object.defineProperty(exports, "extractPairFeatures", { enumerable: true, get: function () { return features_js_1.extractPairFeatures; } });

package/dist/mars_scorer.d.ts

@@ -0,0 +1,45 @@
+export interface HingeDefinition {
+    feature: string;
+    coefficient: number;
+    knot: number;
+    reflected?: boolean;
+}
+export interface MARSModelData {
+    model_name: string;
+    description?: string;
+    required_features: string[];
+    intercept: number;
+    hinges: HingeDefinition[];
+}
+export interface HingeContribution {
+    feature: string;
+    inputValue: number | null;
+    knot: number;
+    reflected: boolean;
+    coefficient: number;
+    hingeOutput: number;
+}
+export interface MARSTraceResult {
+    probability: number;
+    logit: number;
+    intercept: number;
+    hingeContributions: HingeContribution[];
+}
+export declare class HingeFunction {
+    feature: string;
+    coefficient: number;
+    knot: number;
+    reflected: boolean;
+    constructor(feature: string, coefficient: number, knot: number, reflected?: boolean);
+    evaluate(x: number): number;
+}
+export declare class MARSScorer {
+    modelName: string;
+    description: string;
+    intercept: number;
+    hinges: HingeFunction[];
+    requiredFeatures: string[];
+    constructor(data: MARSModelData);
+    score(features: Record<string, number | null | undefined>): number;
+    scoreWithTrace(features: Record<string, number | null | undefined>): MARSTraceResult;
+}

package/dist/mars_scorer.js

@@ -0,0 +1,81 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MARSScorer = exports.HingeFunction = void 0;
+class HingeFunction {
+    feature;
+    coefficient;
+    knot;
+    reflected;
+    constructor(feature, coefficient, knot, reflected = false) {
+        this.feature = feature;
+        this.coefficient = coefficient;
+        this.knot = knot;
+        this.reflected = reflected;
+    }
+    evaluate(x) {
+        if (this.reflected) {
+            return this.coefficient * Math.max(0, this.knot - x);
+        }
+        return this.coefficient * Math.max(0, x - this.knot);
+    }
+}
+exports.HingeFunction = HingeFunction;
+class MARSScorer {
+    modelName;
+    description;
+    intercept;
+    hinges;
+    requiredFeatures;
+    constructor(data) {
+        this.modelName = data.model_name;
+        this.description = data.description ?? "";
+        this.intercept = data.intercept;
+        this.requiredFeatures = data.required_features;
+        this.hinges = data.hinges.map((h) => new HingeFunction(h.feature, h.coefficient, h.knot, Boolean(h.reflected)));
+    }
+    score(features) {
+        let logit = this.intercept;
+        for (const hinge of this.hinges) {
+            const val = features[hinge.feature];
+            if (val !== null && val !== undefined) {
+                logit += hinge.evaluate(val);
+            }
+        }
+        if (logit >= 50.0)
+            return 1.0;
+        if (logit <= -50.0)
+            return 0.0;
+        return 1.0 / (1.0 + Math.exp(-logit));
+    }
+    scoreWithTrace(features) {
+        let logit = this.intercept;
+        const contributions = [];
+        for (const hinge of this.hinges) {
+            const val = features[hinge.feature] ?? null;
+            const contribution = val !== null ? hinge.evaluate(val) : 0.0;
+            logit += contribution;
+            contributions.push({
+                feature: hinge.feature,
+                inputValue: val,
+                knot: hinge.knot,
+                reflected: hinge.reflected,
+                coefficient: hinge.coefficient,
+                hingeOutput: contribution,
+            });
+        }
+        let prob = 0.0;
+        if (logit >= 50.0)
+            prob = 1.0;
+        else if (logit <= -50.0)
+            prob = 0.0;
+        else
+            prob = 1.0 / (1.0 + Math.exp(-logit));
+        return {
+            probability: Number(prob.toFixed(6)),
+            logit: Number(logit.toFixed(6)),
+            intercept: this.intercept,
+            hingeContributions: contributions,
+        };
+    }
+}
+exports.MARSScorer = MARSScorer;

package/dist/pipeline.d.ts

@@ -0,0 +1,24 @@
+import { MARSModelData } from "./mars_scorer.js";
+import { ICPCalibrationData } from "./conformal.js";
+import { DecisionRouter, RoutingDecision } from "./router.js";
+export interface DeduplicationSummary {
+    totalRecords: number;
+    candidatePairsEvaluated: number;
+    tier1AutoMerge: number;
+    tier2HumanReview: number;
+    tier3AutoDistinct: number;
+}
+export declare class DeduplicationPipeline {
+    records: Record<string, string>[];
+    router: DecisionRouter;
+    constructor(records: Record<string, string>[], router: DecisionRouter);
+    static fromCSV(csvContent: string, modelData: MARSModelData, calibrationData?: ICPCalibrationData): DeduplicationPipeline;
+    scorePair(recordA: Record<string, string | number | null | undefined>, recordB: Record<string, string | number | null | undefined>): RoutingDecision;
+    run(): {
+        summary: DeduplicationSummary;
+        results: RoutingDecision[];
+        autoMerge: RoutingDecision[];
+        humanReview: RoutingDecision[];
+        autoDistinct: RoutingDecision[];
+    };
+}

package/dist/pipeline.js

@@ -0,0 +1,66 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DeduplicationPipeline = void 0;
+const mars_scorer_js_1 = require("./mars_scorer.js");
+const conformal_js_1 = require("./conformal.js");
+const router_js_1 = require("./router.js");
+const features_js_1 = require("./features.js");
+const papaparse_1 = __importDefault(require("papaparse"));
+class DeduplicationPipeline {
+    records;
+    router;
+    constructor(records, router) {
+        this.records = records;
+        this.router = router;
+    }
+    static fromCSV(csvContent, modelData, calibrationData) {
+        const parsed = papaparse_1.default.parse(csvContent, {
+            header: true,
+            skipEmptyLines: true,
+        });
+        const scorer = new mars_scorer_js_1.MARSScorer(modelData);
+        const icp = calibrationData ? new conformal_js_1.ICPPredictor(calibrationData) : null;
+        const router = new router_js_1.DecisionRouter(scorer, icp);
+        return new DeduplicationPipeline(parsed.data, router);
+    }
+    scorePair(recordA, recordB) {
+        const feats = (0, features_js_1.extractPairFeatures)(recordA, recordB);
+        return this.router.route(feats);
+    }
+    run() {
+        const results = [];
+        const autoMerge = [];
+        const humanReview = [];
+        const autoDistinct = [];
+        const n = this.records.length;
+        for (let i = 0; i < n; i++) {
+            for (let j = i + 1; j < n; j++) {
+                const dec = this.scorePair(this.records[i], this.records[j]);
+                results.push(dec);
+                if (dec.tier === router_js_1.Tier.AUTO_MERGE)
+                    autoMerge.push(dec);
+                else if (dec.tier === router_js_1.Tier.HUMAN_REVIEW)
+                    humanReview.push(dec);
+                else
+                    autoDistinct.push(dec);
+            }
+        }
+        return {
+            summary: {
+                totalRecords: n,
+                candidatePairsEvaluated: results.length,
+                tier1AutoMerge: autoMerge.length,
+                tier2HumanReview: humanReview.length,
+                tier3AutoDistinct: autoDistinct.length,
+            },
+            results,
+            autoMerge,
+            humanReview,
+            autoDistinct,
+        };
+    }
+}
+exports.DeduplicationPipeline = DeduplicationPipeline;

package/dist/router.d.ts

@@ -0,0 +1,26 @@
+import { MARSScorer, MARSTraceResult } from "./mars_scorer.js";
+import { ICPPredictor } from "./conformal.js";
+export declare enum Tier {
+    AUTO_MERGE = "auto_merge",
+    HUMAN_REVIEW = "human_review",
+    AUTO_DISTINCT = "auto_distinct"
+}
+export interface RoutingDecision {
+    tier: Tier;
+    probability: number;
+    predictionSet: number[];
+    reason: string;
+    features: Record<string, number | null>;
+    marsTrace?: MARSTraceResult;
+}
+export declare class DecisionRouter {
+    scorer: MARSScorer;
+    icp: ICPPredictor | null;
+    dupThreshold: number;
+    distThreshold: number;
+    constructor(scorer: MARSScorer, icp?: ICPPredictor | null, dupThreshold?: number, distThreshold?: number);
+    route(features: Record<string, number | null>, hardSignals?: {
+        isExactIdMatch?: boolean;
+        isExactIdMismatch?: boolean;
+    }): RoutingDecision;
+}

package/dist/router.js

@@ -0,0 +1,81 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DecisionRouter = exports.Tier = void 0;
+var Tier;
+(function (Tier) {
+    Tier["AUTO_MERGE"] = "auto_merge";
+    Tier["HUMAN_REVIEW"] = "human_review";
+    Tier["AUTO_DISTINCT"] = "auto_distinct";
+})(Tier || (exports.Tier = Tier = {}));
+class DecisionRouter {
+    scorer;
+    icp;
+    dupThreshold;
+    distThreshold;
+    constructor(scorer, icp = null, dupThreshold = 0.898, distThreshold = 0.491) {
+        this.scorer = scorer;
+        this.icp = icp;
+        this.dupThreshold = dupThreshold;
+        this.distThreshold = distThreshold;
+    }
+    route(features, hardSignals = {}) {
+        // 1. Hard signals override
+        if (hardSignals.isExactIdMatch) {
+            return {
+                tier: Tier.AUTO_MERGE,
+                probability: 1.0,
+                predictionSet: [1],
+                reason: "hard_signal:exact_id_match",
+                features,
+            };
+        }
+        if (hardSignals.isExactIdMismatch) {
+            return {
+                tier: Tier.AUTO_DISTINCT,
+                probability: 0.0,
+                predictionSet: [0],
+                reason: "hard_signal:exact_id_mismatch",
+                features,
+            };
+        }
+        // 2. Probabilistic evaluation via MARS
+        const trace = this.scorer.scoreWithTrace(features);
+        const prob = trace.probability;
+        // 3. Conformal Prediction Set Check
+        let pset = [0, 1];
+        if (this.icp) {
+            const setObj = this.icp.predictSet(prob);
+            pset = Array.from(setObj);
+        }
+        if (pset.length === 1 && pset[0] === 1) {
+            return {
+                tier: Tier.AUTO_MERGE,
+                probability: prob,
+                predictionSet: pset,
+                reason: `mars:high_confidence_duplicate(p=${prob.toFixed(3)})`,
+                features,
+                marsTrace: trace,
+            };
+        }
+        if (pset.length === 1 && pset[0] === 0) {
+            return {
+                tier: Tier.AUTO_DISTINCT,
+                probability: prob,
+                predictionSet: pset,
+                reason: `mars:high_confidence_distinct(p=${prob.toFixed(3)})`,
+                features,
+                marsTrace: trace,
+            };
+        }
+        // Circuit Breaker triggered
+        return {
+            tier: Tier.HUMAN_REVIEW,
+            probability: prob,
+            predictionSet: pset,
+            reason: `circuit_breaker:ambiguous_region(p=${prob.toFixed(3)},set={0,1})`,
+            features,
+            marsTrace: trace,
+        };
+    }
+}
+exports.DecisionRouter = DecisionRouter;

package/package.json

@@ -1,6 +1,31 @@
 {
   "name": "@futux/whitebox-dedup",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.1.1",
+  "description": "Interpretable entity deduplication using auditable MARS equations and conformal prediction in TypeScript",
+  "main": "dist/index.js",
+  "module": "dist/index.mjs",
+  "types": "dist/index.d.ts",
+  "scripts": {
+    "build": "tsc",
+    "test": "vitest run"
+  },
+  "keywords": [
+    "deduplication",
+    "entity-resolution",
+    "mars",
+    "conformal-prediction",
+    "data-governance",
+    "dpi"
+  ],
+  "author": "Daniel Olagunju <d.olagunju@futuxconsult.com>, Temitope Olarewaju <t.olarewaju@futuxconsult.com>",
+  "license": "MIT",
+  "dependencies": {
+    "papaparse": "^5.4.1"
+  },
+  "devDependencies": {
+    "@types/node": "^22.0.0",
+    "@types/papaparse": "^5.3.14",
+    "typescript": "^5.5.0",
+    "vitest": "^2.0.0"
+  }
+}

package/src/conformal.ts

@@ -0,0 +1,31 @@
+export interface ICPCalibrationData {
+  model_name: string;
+  alpha: number;
+  threshold_duplicate: number;
+  threshold_distinct: number;
+  description?: string;
+}
+
+export class ICPPredictor {
+  constructor(public cal: ICPCalibrationData) {}
+
+  predictSet(probability: number): Set<number> {
+    const nonconformityDup = 1.0 - probability;
+    const nonconformityDist = probability;
+
+    const includeDup = nonconformityDup <= this.cal.threshold_duplicate;
+    const includeDist = nonconformityDist <= this.cal.threshold_distinct;
+
+    const set = new Set<number>();
+    if (includeDist) set.add(0);
+    if (includeDup) set.add(1);
+
+    if (set.size === 0) {
+      // Empty set safeguard -> ambiguous
+      set.add(0);
+      set.add(1);
+    }
+
+    return set;
+  }
+}

package/src/features.ts

@@ -0,0 +1,107 @@
+export function jaroWinkler(s1: string, s2: string): number {
+  if (s1 === s2) return 1.0;
+  const len1 = s1.length;
+  const len2 = s2.length;
+  if (len1 === 0 || len2 === 0) return 0.0;
+
+  const matchDistance = Math.floor(Math.max(len1, len2) / 2) - 1;
+  const s1Matches = new Array(len1).fill(false);
+  const s2Matches = new Array(len2).fill(false);
+
+  let matches = 0;
+  for (let i = 0; i < len1; i++) {
+    const start = Math.max(0, i - matchDistance);
+    const end = Math.min(i + matchDistance + 1, len2);
+    for (let j = start; j < end; j++) {
+      if (s2Matches[j] || s1[i] !== s2[j]) continue;
+      s1Matches[i] = true;
+      s2Matches[j] = true;
+      matches++;
+      break;
+    }
+  }
+
+  if (matches === 0) return 0.0;
+
+  let transpositions = 0;
+  let k = 0;
+  for (let i = 0; i < len1; i++) {
+    if (!s1Matches[i]) continue;
+    while (!s2Matches[k]) k++;
+    if (s1[i] !== s2[k]) transpositions++;
+    k++;
+  }
+
+  const jaro = (matches / len1 + matches / len2 + (matches - transpositions / 2) / matches) / 3;
+  let prefix = 0;
+  for (let i = 0; i < Math.min(4, Math.min(len1, len2)); i++) {
+    if (s1[i] === s2[i]) prefix++;
+    else break;
+  }
+
+  return jaro + prefix * 0.1 * (1 - jaro);
+}
+
+export function haversineM(lat1: number, lon1: number, lat2: number, lon2: number): number {
+  const R = 6371000.0; // Earth radius in meters
+  const toRad = (deg: number) => (deg * Math.PI) / 180.0;
+
+  const dLat = toRad(lat2 - lat1);
+  const dLon = toRad(lon2 - lon1);
+
+  const a =
+    Math.sin(dLat / 2) * Math.sin(dLat / 2) +
+    Math.cos(toRad(lat1)) * Math.cos(toRad(lat2)) * Math.sin(dLon / 2) * Math.sin(dLon / 2);
+  const c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1 - a));
+
+  return R * c;
+}
+
+export function extractPairFeatures(
+  a: Record<string, string | number | null | undefined>,
+  b: Record<string, string | number | null | undefined>
+): Record<string, number | null> {
+  const feats: Record<string, number | null> = {};
+
+  // Name similarities
+  const nameA = String(a["full_name"] ?? a["name"] ?? "").trim().toLowerCase();
+  const nameB = String(b["full_name"] ?? b["name"] ?? "").trim().toLowerCase();
+
+  if (nameA && nameB) {
+    feats["jw_name_full"] = jaroWinkler(nameA, nameB);
+    const tokensA = nameA.split(/\s+/);
+    const tokensB = nameB.split(/\s+/);
+    feats["jw_name_first"] = jaroWinkler(tokensA[0] ?? "", tokensB[0] ?? "");
+    feats["jw_name_last"] = jaroWinkler(
+      tokensA[tokensA.length - 1] ?? "",
+      tokensB[tokensB.length - 1] ?? ""
+    );
+  } else {
+    feats["jw_name_full"] = null;
+    feats["jw_name_first"] = null;
+    feats["jw_name_last"] = null;
+  }
+
+  // Exact Phone Match
+  const phoneA = String(a["phone"] ?? a["phone_number"] ?? "").replace(/\D/g, "");
+  const phoneB = String(b["phone"] ?? b["phone_number"] ?? "").replace(/\D/g, "");
+  if (phoneA.length >= 7 && phoneB.length >= 7) {
+    feats["exact_phone"] = phoneA === phoneB ? 1.0 : 0.0;
+  } else {
+    feats["exact_phone"] = null;
+  }
+
+  // Haversine GPS distance
+  const latA = Number(a["latitude"] ?? a["lat"]);
+  const lonA = Number(a["longitude"] ?? a["lon"]);
+  const latB = Number(b["latitude"] ?? b["lat"]);
+  const lonB = Number(b["longitude"] ?? b["lon"]);
+
+  if (!isNaN(latA) && !isNaN(lonA) && !isNaN(latB) && !isNaN(lonB)) {
+    feats["haversine_m"] = haversineM(latA, lonA, latB, lonB);
+  } else {
+    feats["haversine_m"] = null;
+  }
+
+  return feats;
+}

package/src/index.ts

@@ -0,0 +1,9 @@
+export { MARSScorer } from "./mars_scorer.js";
+export * from "./mars_scorer.js";
+export { ICPPredictor } from "./conformal.js";
+export * from "./conformal.js";
+export { DecisionRouter, Tier } from "./router.js";
+export * from "./router.js";
+export { DeduplicationPipeline } from "./pipeline.js";
+export * from "./pipeline.js";
+export { jaroWinkler, haversineM, extractPairFeatures } from "./features.js";

package/src/mars_scorer.ts

@@ -0,0 +1,108 @@
+export interface HingeDefinition {
+  feature: string;
+  coefficient: number;
+  knot: number;
+  reflected?: boolean;
+}
+
+export interface MARSModelData {
+  model_name: string;
+  description?: string;
+  required_features: string[];
+  intercept: number;
+  hinges: HingeDefinition[];
+}
+
+export interface HingeContribution {
+  feature: string;
+  inputValue: number | null;
+  knot: number;
+  reflected: boolean;
+  coefficient: number;
+  hingeOutput: number;
+}
+
+export interface MARSTraceResult {
+  probability: number;
+  logit: number;
+  intercept: number;
+  hingeContributions: HingeContribution[];
+}
+
+export class HingeFunction {
+  constructor(
+    public feature: string,
+    public coefficient: number,
+    public knot: number,
+    public reflected: boolean = false
+  ) {}
+
+  evaluate(x: number): number {
+    if (this.reflected) {
+      return this.coefficient * Math.max(0, this.knot - x);
+    }
+    return this.coefficient * Math.max(0, x - this.knot);
+  }
+}
+
+export class MARSScorer {
+  public modelName: string;
+  public description: string;
+  public intercept: number;
+  public hinges: HingeFunction[];
+  public requiredFeatures: string[];
+
+  constructor(data: MARSModelData) {
+    this.modelName = data.model_name;
+    this.description = data.description ?? "";
+    this.intercept = data.intercept;
+    this.requiredFeatures = data.required_features;
+    this.hinges = data.hinges.map(
+      (h) => new HingeFunction(h.feature, h.coefficient, h.knot, Boolean(h.reflected))
+    );
+  }
+
+  score(features: Record<string, number | null | undefined>): number {
+    let logit = this.intercept;
+    for (const hinge of this.hinges) {
+      const val = features[hinge.feature];
+      if (val !== null && val !== undefined) {
+        logit += hinge.evaluate(val);
+      }
+    }
+    if (logit >= 50.0) return 1.0;
+    if (logit <= -50.0) return 0.0;
+    return 1.0 / (1.0 + Math.exp(-logit));
+  }
+
+  scoreWithTrace(features: Record<string, number | null | undefined>): MARSTraceResult {
+    let logit = this.intercept;
+    const contributions: HingeContribution[] = [];
+
+    for (const hinge of this.hinges) {
+      const val = features[hinge.feature] ?? null;
+      const contribution = val !== null ? hinge.evaluate(val) : 0.0;
+      logit += contribution;
+      contributions.push({
+        feature: hinge.feature,
+        inputValue: val,
+        knot: hinge.knot,
+        reflected: hinge.reflected,
+        coefficient: hinge.coefficient,
+        hingeOutput: contribution,
+      });
+    }
+
+    let prob = 0.0;
+    if (logit >= 50.0) prob = 1.0;
+    else if (logit <= -50.0) prob = 0.0;
+    else prob = 1.0 / (1.0 + Math.exp(-logit));
+
+    return {
+      probability: Number(prob.toFixed(6)),
+      logit: Number(logit.toFixed(6)),
+      intercept: this.intercept,
+      hingeContributions: contributions,
+    };
+  }
+}

package/src/pipeline.ts

@@ -0,0 +1,81 @@
+import { MARSScorer, MARSModelData } from "./mars_scorer.js";
+import { ICPPredictor, ICPCalibrationData } from "./conformal.js";
+import { DecisionRouter, RoutingDecision, Tier } from "./router.js";
+import { extractPairFeatures } from "./features.js";
+import Papa from "papaparse";
+
+export interface DeduplicationSummary {
+  totalRecords: number;
+  candidatePairsEvaluated: number;
+  tier1AutoMerge: number;
+  tier2HumanReview: number;
+  tier3AutoDistinct: number;
+}
+
+export class DeduplicationPipeline {
+  constructor(
+    public records: Record<string, string>[],
+    public router: DecisionRouter
+  ) {}
+
+  static fromCSV(
+    csvContent: string,
+    modelData: MARSModelData,
+    calibrationData?: ICPCalibrationData
+  ): DeduplicationPipeline {
+    const parsed = Papa.parse<Record<string, string>>(csvContent, {
+      header: true,
+      skipEmptyLines: true,
+    });
+    const scorer = new MARSScorer(modelData);
+    const icp = calibrationData ? new ICPPredictor(calibrationData) : null;
+    const router = new DecisionRouter(scorer, icp);
+    return new DeduplicationPipeline(parsed.data, router);
+  }
+
+  scorePair(
+    recordA: Record<string, string | number | null | undefined>,
+    recordB: Record<string, string | number | null | undefined>
+  ): RoutingDecision {
+    const feats = extractPairFeatures(recordA, recordB);
+    return this.router.route(feats);
+  }
+
+  run(): {
+    summary: DeduplicationSummary;
+    results: RoutingDecision[];
+    autoMerge: RoutingDecision[];
+    humanReview: RoutingDecision[];
+    autoDistinct: RoutingDecision[];
+  } {
+    const results: RoutingDecision[] = [];
+    const autoMerge: RoutingDecision[] = [];
+    const humanReview: RoutingDecision[] = [];
+    const autoDistinct: RoutingDecision[] = [];
+
+    const n = this.records.length;
+    for (let i = 0; i < n; i++) {
+      for (let j = i + 1; j < n; j++) {
+        const dec = this.scorePair(this.records[i]!, this.records[j]!);
+        results.push(dec);
+        if (dec.tier === Tier.AUTO_MERGE) autoMerge.push(dec);
+        else if (dec.tier === Tier.HUMAN_REVIEW) humanReview.push(dec);
+        else autoDistinct.push(dec);
+      }
+    }
+
+    return {
+      summary: {
+        totalRecords: n,
+        candidatePairsEvaluated: results.length,
+        tier1AutoMerge: autoMerge.length,
+        tier2HumanReview: humanReview.length,
+        tier3AutoDistinct: autoDistinct.length,
+      },
+      results,
+      autoMerge,
+      humanReview,
+      autoDistinct,
+    };
+  }
+}

package/src/router.ts

@@ -0,0 +1,94 @@
+import { MARSScorer, MARSTraceResult } from "./mars_scorer.js";
+import { ICPPredictor } from "./conformal.js";
+
+export enum Tier {
+  AUTO_MERGE = "auto_merge",
+  HUMAN_REVIEW = "human_review",
+  AUTO_DISTINCT = "auto_distinct",
+}
+
+export interface RoutingDecision {
+  tier: Tier;
+  probability: number;
+  predictionSet: number[];
+  reason: string;
+  features: Record<string, number | null>;
+  marsTrace?: MARSTraceResult;
+}
+
+export class DecisionRouter {
+  constructor(
+    public scorer: MARSScorer,
+    public icp: ICPPredictor | null = null,
+    public dupThreshold: number = 0.898,
+    public distThreshold: number = 0.491
+  ) {}
+
+  route(
+    features: Record<string, number | null>,
+    hardSignals: { isExactIdMatch?: boolean; isExactIdMismatch?: boolean } = {}
+  ): RoutingDecision {
+    // 1. Hard signals override
+    if (hardSignals.isExactIdMatch) {
+      return {
+        tier: Tier.AUTO_MERGE,
+        probability: 1.0,
+        predictionSet: [1],
+        reason: "hard_signal:exact_id_match",
+        features,
+      };
+    }
+    if (hardSignals.isExactIdMismatch) {
+      return {
+        tier: Tier.AUTO_DISTINCT,
+        probability: 0.0,
+        predictionSet: [0],
+        reason: "hard_signal:exact_id_mismatch",
+        features,
+      };
+    }
+
+    // 2. Probabilistic evaluation via MARS
+    const trace = this.scorer.scoreWithTrace(features);
+    const prob = trace.probability;
+
+    // 3. Conformal Prediction Set Check
+    let pset = [0, 1];
+    if (this.icp) {
+      const setObj = this.icp.predictSet(prob);
+      pset = Array.from(setObj);
+    }
+
+    if (pset.length === 1 && pset[0] === 1) {
+      return {
+        tier: Tier.AUTO_MERGE,
+        probability: prob,
+        predictionSet: pset,
+        reason: `mars:high_confidence_duplicate(p=${prob.toFixed(3)})`,
+        features,
+        marsTrace: trace,
+      };
+    }
+
+    if (pset.length === 1 && pset[0] === 0) {
+      return {
+        tier: Tier.AUTO_DISTINCT,
+        probability: prob,
+        predictionSet: pset,
+        reason: `mars:high_confidence_distinct(p=${prob.toFixed(3)})`,
+        features,
+        marsTrace: trace,
+      };
+    }
+
+    // Circuit Breaker triggered
+    return {
+      tier: Tier.HUMAN_REVIEW,
+      probability: prob,
+      predictionSet: pset,
+      reason: `circuit_breaker:ambiguous_region(p=${prob.toFixed(3)},set={0,1})`,
+      features,
+      marsTrace: trace,
+    };
+  }
+}

package/tsconfig.json

@@ -0,0 +1,16 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
+    "declaration": true,
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "strict": true,
+    "noImplicitAny": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true
+  },
+  "include": ["src/**/*"]
+}