@futux/dqi 0.0.0-stage → 0.1.0

package/dist/dimensions/completeness.d.ts

@@ -0,0 +1,3 @@
+import { Schema } from "../schema.js";
+import { DimensionResult } from "../types.js";
+export declare function scoreCompleteness(rows: Record<string, string>[], schema: Schema): DimensionResult;

package/dist/dimensions/completeness.js

@@ -0,0 +1,31 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.scoreCompleteness = scoreCompleteness;
+function scoreCompleteness(rows, schema) {
+    const issues = [];
+    const requiredFields = Object.values(schema.fields)
+        .filter((f) => f.required)
+        .map((f) => f.name);
+    if (requiredFields.length === 0 || rows.length === 0) {
+        return { score: 100, issues: [] };
+    }
+    let totalRequiredSlots = rows.length * requiredFields.length;
+    let missingCount = 0;
+    rows.forEach((row, rowIndex) => {
+        for (const field of requiredFields) {
+            const val = row[field];
+            if (val === undefined || val === null || val.trim() === "") {
+                missingCount++;
+                issues.push({
+                    rowIndex,
+                    field,
+                    dimension: "completeness",
+                    reason: "required_field_empty",
+                    value: val ?? null,
+                });
+            }
+        }
+    });
+    const score = Math.max(0, Math.min(100, (1 - missingCount / totalRequiredSlots) * 100));
+    return { score: Number(score.toFixed(2)), issues };
+}

package/dist/dimensions/outliers.d.ts

@@ -0,0 +1,3 @@
+import { Schema } from "../schema.js";
+import { DimensionResult } from "../types.js";
+export declare function scoreOutliers(rows: Record<string, string>[], schema: Schema): DimensionResult;

package/dist/dimensions/outliers.js

@@ -0,0 +1,61 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.scoreOutliers = scoreOutliers;
+const types_js_1 = require("../types.js");
+function quantile(arr, q) {
+    const sorted = [...arr].sort((a, b) => a - b);
+    const pos = (sorted.length - 1) * q;
+    const base = Math.floor(pos);
+    const rest = pos - base;
+    if (sorted[base + 1] !== undefined) {
+        return sorted[base] + rest * (sorted[base + 1] - sorted[base]);
+    }
+    return sorted[base];
+}
+function scoreOutliers(rows, schema) {
+    const issues = [];
+    const flaggedRows = new Set();
+    const numericFields = Object.values(schema.fields)
+        .filter((f) => [types_js_1.FieldType.FLOAT, types_js_1.FieldType.INTEGER].includes(f.type))
+        .map((f) => f.name);
+    if (numericFields.length === 0 || rows.length < 4) {
+        return { score: 100, issues: [] };
+    }
+    for (const col of numericFields) {
+        const validEntries = [];
+        rows.forEach((row, rowIndex) => {
+            const v = row[col];
+            if (v !== undefined && v !== null && v.trim() !== "") {
+                const num = Number(v);
+                if (!isNaN(num)) {
+                    validEntries.push({ idx: rowIndex, val: num });
+                }
+            }
+        });
+        if (validEntries.length < 4)
+            continue;
+        const values = validEntries.map((e) => e.val);
+        const q1 = quantile(values, 0.25);
+        const q3 = quantile(values, 0.75);
+        const iqr = q3 - q1;
+        const lowerFence = q1 - 1.5 * iqr;
+        const upperFence = q3 + 1.5 * iqr;
+        for (const entry of validEntries) {
+            if (entry.val < lowerFence || entry.val > upperFence) {
+                flaggedRows.add(entry.idx);
+                issues.push({
+                    rowIndex: entry.idx,
+                    field: col,
+                    dimension: "outliers",
+                    reason: `iqr_outlier_[${lowerFence.toFixed(2)},${upperFence.toFixed(2)}]`,
+                    value: entry.val,
+                });
+            }
+        }
+    }
+    const score = rows.length === 0 ? 100 : Math.max(0, (1 - flaggedRows.size / rows.length) * 100);
+    return {
+        score: Number(score.toFixed(2)),
+        issues,
+    };
+}

package/dist/dimensions/uniqueness.d.ts

@@ -0,0 +1,4 @@
+import { Schema } from "../schema.js";
+import { DimensionResult } from "../types.js";
+export declare function jaroWinkler(s1: string, s2: string): number;
+export declare function scoreUniqueness(rows: Record<string, string>[], schema: Schema): DimensionResult;

package/dist/dimensions/uniqueness.js

@@ -0,0 +1,122 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.jaroWinkler = jaroWinkler;
+exports.scoreUniqueness = scoreUniqueness;
+function jaroWinkler(s1, s2) {
+    if (s1 === s2)
+        return 1.0;
+    const len1 = s1.length;
+    const len2 = s2.length;
+    if (len1 === 0 || len2 === 0)
+        return 0.0;
+    const matchDistance = Math.floor(Math.max(len1, len2) / 2) - 1;
+    const s1Matches = new Array(len1).fill(false);
+    const s2Matches = new Array(len2).fill(false);
+    let matches = 0;
+    for (let i = 0; i < len1; i++) {
+        const start = Math.max(0, i - matchDistance);
+        const end = Math.min(i + matchDistance + 1, len2);
+        for (let j = start; j < end; j++) {
+            if (s2Matches[j] || s1[i] !== s2[j])
+                continue;
+            s1Matches[i] = true;
+            s2Matches[j] = true;
+            matches++;
+            break;
+        }
+    }
+    if (matches === 0)
+        return 0.0;
+    let transpositions = 0;
+    let k = 0;
+    for (let i = 0; i < len1; i++) {
+        if (!s1Matches[i])
+            continue;
+        while (!s2Matches[k])
+            k++;
+        if (s1[i] !== s2[k])
+            transpositions++;
+        k++;
+    }
+    const jaro = (matches / len1 + matches / len2 + (matches - transpositions / 2) / matches) / 3;
+    let prefix = 0;
+    for (let i = 0; i < Math.min(4, Math.min(len1, len2)); i++) {
+        if (s1[i] === s2[i])
+            prefix++;
+        else
+            break;
+    }
+    return jaro + prefix * 0.1 * (1 - jaro);
+}
+function scoreUniqueness(rows, schema) {
+    const issues = [];
+    const flaggedRows = new Set();
+    let exactDuplicates = 0;
+    const uniqueKeyFields = Object.values(schema.fields)
+        .filter((f) => f.uniqueKey)
+        .map((f) => f.name);
+    // Exact duplicates
+    for (const col of uniqueKeyFields) {
+        const seen = new Map();
+        rows.forEach((row, rowIndex) => {
+            const val = row[col];
+            if (!val || val.trim() === "")
+                return;
+            const key = val.trim();
+            if (seen.has(key)) {
+                exactDuplicates++;
+                flaggedRows.add(rowIndex);
+                issues.push({
+                    rowIndex,
+                    field: col,
+                    dimension: "uniqueness",
+                    reason: "exact_duplicate_key",
+                    duplicateOf: seen.get(key),
+                });
+            }
+            else {
+                seen.set(key, rowIndex);
+            }
+        });
+    }
+    // Soft fuzzy candidate pairs
+    const fuzzyFields = Object.values(schema.fields)
+        .filter((f) => f.fuzzyKey)
+        .map((f) => f.name);
+    for (const col of fuzzyFields) {
+        const blocks = new Map();
+        rows.forEach((row, rowIndex) => {
+            const val = row[col];
+            if (!val || val.trim() === "")
+                return;
+            const clean = val.trim().toLowerCase();
+            const bucket = clean[0] ?? "_";
+            if (!blocks.has(bucket))
+                blocks.set(bucket, []);
+            blocks.get(bucket).push({ idx: rowIndex, val: clean });
+        });
+        for (const entries of blocks.values()) {
+            for (let i = 0; i < entries.length; i++) {
+                for (let j = i + 1; j < entries.length; j++) {
+                    const sim = jaroWinkler(entries[i].val, entries[j].val);
+                    if (sim >= 0.85) {
+                        flaggedRows.add(entries[j].idx);
+                        issues.push({
+                            rowIndex: entries[j].idx,
+                            field: col,
+                            dimension: "uniqueness",
+                            reason: "fuzzy_duplicate_candidate",
+                            duplicateOf: entries[i].idx,
+                        });
+                    }
+                }
+            }
+        }
+    }
+    const score = rows.length === 0 ? 100 : Math.max(0, (1 - flaggedRows.size / rows.length) * 100);
+    return {
+        score: Number(score.toFixed(2)),
+        exactDuplicateCount: exactDuplicates,
+        issues,
+    };
+}

package/dist/dimensions/validity.d.ts

@@ -0,0 +1,3 @@
+import { Schema } from "../schema.js";
+import { DimensionResult } from "../types.js";
+export declare function scoreValidity(rows: Record<string, string>[], schema: Schema): DimensionResult;

package/dist/dimensions/validity.js

@@ -0,0 +1,130 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.scoreValidity = scoreValidity;
+const types_js_1 = require("../types.js");
+const PHONE_REGEX = /^\+?[\d\s\-().]{7,15}$/;
+const EMAIL_REGEX = /^[^@\s]+@[^@\s]+\.[^@\s]+$/;
+function scoreValidity(rows, schema) {
+    const issues = [];
+    let checkedSlots = 0;
+    let invalidSlots = 0;
+    rows.forEach((row, rowIndex) => {
+        for (const [colName, fdef] of Object.entries(schema.fields)) {
+            const val = row[colName];
+            if (val === undefined || val === null || val.trim() === "") {
+                continue;
+            }
+            checkedSlots++;
+            const cleanVal = val.trim();
+            let isValid = true;
+            let reason = "";
+            switch (fdef.type) {
+                case types_js_1.FieldType.INTEGER: {
+                    const num = Number(cleanVal);
+                    if (isNaN(num) || !Number.isInteger(num)) {
+                        isValid = false;
+                        reason = "invalid_integer_format";
+                    }
+                    else {
+                        if (fdef.min !== undefined && num < Number(fdef.min)) {
+                            isValid = false;
+                            reason = `integer_below_min_${fdef.min}`;
+                        }
+                        if (fdef.max !== undefined && num > Number(fdef.max)) {
+                            isValid = false;
+                            reason = `integer_above_max_${fdef.max}`;
+                        }
+                    }
+                    break;
+                }
+                case types_js_1.FieldType.FLOAT: {
+                    const num = Number(cleanVal);
+                    if (isNaN(num)) {
+                        isValid = false;
+                        reason = "invalid_float_format";
+                    }
+                    else {
+                        if (fdef.min !== undefined && num < Number(fdef.min)) {
+                            isValid = false;
+                            reason = `float_below_min_${fdef.min}`;
+                        }
+                        if (fdef.max !== undefined && num > Number(fdef.max)) {
+                            isValid = false;
+                            reason = `float_above_max_${fdef.max}`;
+                        }
+                    }
+                    break;
+                }
+                case types_js_1.FieldType.LATITUDE: {
+                    const num = Number(cleanVal);
+                    const minLat = fdef.min !== undefined ? Number(fdef.min) : -90.0;
+                    const maxLat = fdef.max !== undefined ? Number(fdef.max) : 90.0;
+                    if (isNaN(num) || num < minLat || num > maxLat) {
+                        isValid = false;
+                        reason = `latitude_out_of_bounds_[${minLat},${maxLat}]`;
+                    }
+                    break;
+                }
+                case types_js_1.FieldType.LONGITUDE: {
+                    const num = Number(cleanVal);
+                    const minLon = fdef.min !== undefined ? Number(fdef.min) : -180.0;
+                    const maxLon = fdef.max !== undefined ? Number(fdef.max) : 180.0;
+                    if (isNaN(num) || num < minLon || num > maxLon) {
+                        isValid = false;
+                        reason = `longitude_out_of_bounds_[${minLon},${maxLon}]`;
+                    }
+                    break;
+                }
+                case types_js_1.FieldType.PHONE: {
+                    if (!PHONE_REGEX.test(cleanVal)) {
+                        isValid = false;
+                        reason = "invalid_phone_format";
+                    }
+                    break;
+                }
+                case types_js_1.FieldType.EMAIL: {
+                    if (!EMAIL_REGEX.test(cleanVal)) {
+                        isValid = false;
+                        reason = "invalid_email_format";
+                    }
+                    break;
+                }
+                case types_js_1.FieldType.ENUM: {
+                    if (fdef.allowedValues && !fdef.allowedValues.includes(cleanVal)) {
+                        isValid = false;
+                        reason = `value_not_in_allowed_enum`;
+                    }
+                    break;
+                }
+                case types_js_1.FieldType.DATE: {
+                    const parsed = Date.parse(cleanVal);
+                    if (isNaN(parsed)) {
+                        isValid = false;
+                        reason = "invalid_date_format";
+                    }
+                    break;
+                }
+                case types_js_1.FieldType.BOOLEAN: {
+                    const boolLower = cleanVal.toLowerCase();
+                    if (!["true", "false", "1", "0", "yes", "no"].includes(boolLower)) {
+                        isValid = false;
+                        reason = "invalid_boolean_format";
+                    }
+                    break;
+                }
+            }
+            if (!isValid) {
+                invalidSlots++;
+                issues.push({
+                    rowIndex,
+                    field: colName,
+                    dimension: "validity",
+                    reason,
+                    value: cleanVal,
+                });
+            }
+        }
+    });
+    const score = checkedSlots === 0 ? 100 : Math.max(0, Math.min(100, (1 - invalidSlots / checkedSlots) * 100));
+    return { score: Number(score.toFixed(2)), issues };
+}

package/dist/engine.d.ts

@@ -0,0 +1,7 @@
+import { Schema } from "./schema.js";
+import { DQIReport } from "./report.js";
+export declare class DQIEngine {
+    schema: Schema | null;
+    constructor(schema?: Schema | null);
+    evaluate(csvInput: string | Record<string, string>[]): DQIReport;
+}

package/dist/engine.js

@@ -0,0 +1,79 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DQIEngine = void 0;
+const schema_js_1 = require("./schema.js");
+const report_js_1 = require("./report.js");
+const completeness_js_1 = require("./dimensions/completeness.js");
+const validity_js_1 = require("./dimensions/validity.js");
+const uniqueness_js_1 = require("./dimensions/uniqueness.js");
+const outliers_js_1 = require("./dimensions/outliers.js");
+const papaparse_1 = __importDefault(require("papaparse"));
+class DQIEngine {
+    schema;
+    constructor(schema = null) {
+        this.schema = schema;
+    }
+    evaluate(csvInput) {
+        let rows;
+        if (typeof csvInput === "string") {
+            const parsed = papaparse_1.default.parse(csvInput, {
+                header: true,
+                skipEmptyLines: true,
+            });
+            rows = parsed.data;
+            if (!this.schema) {
+                this.schema = schema_js_1.Schema.infer(csvInput);
+            }
+        }
+        else {
+            rows = csvInput;
+            if (!this.schema) {
+                throw new Error("Schema must be provided when evaluating raw row arrays.");
+            }
+        }
+        const completeness = (0, completeness_js_1.scoreCompleteness)(rows, this.schema);
+        const validity = (0, validity_js_1.scoreValidity)(rows, this.schema);
+        const uniqueness = (0, uniqueness_js_1.scoreUniqueness)(rows, this.schema);
+        const outliers = (0, outliers_js_1.scoreOutliers)(rows, this.schema);
+        const w = this.schema.weights;
+        const overall = completeness.score * w.completeness +
+            validity.score * w.validity +
+            uniqueness.score * w.uniqueness +
+            outliers.score * w.outliers;
+        const roundOverall = Number(overall.toFixed(2));
+        let grade = "F";
+        if (roundOverall >= 90)
+            grade = "A";
+        else if (roundOverall >= 75)
+            grade = "B";
+        else if (roundOverall >= 60)
+            grade = "C";
+        else if (roundOverall >= 45)
+            grade = "D";
+        const allIssues = [
+            ...completeness.issues,
+            ...validity.issues,
+            ...uniqueness.issues,
+            ...outliers.issues,
+        ];
+        const reportData = {
+            overallScore: roundOverall,
+            grade,
+            totalRecords: rows.length,
+            dimensions: {
+                completeness,
+                validity,
+                uniqueness,
+                outliers,
+            },
+            weights: w,
+            allIssues,
+            generatedAt: new Date().toISOString(),
+        };
+        return new report_js_1.DQIReport(reportData);
+    }
+}
+exports.DQIEngine = DQIEngine;

package/dist/index.d.ts

@@ -0,0 +1,5 @@
+export { DQIEngine } from "./engine.js";
+export { Schema } from "./schema.js";
+export { DQIReport } from "./report.js";
+export * from "./types.js";
+export { jaroWinkler } from "./dimensions/uniqueness.js";

package/dist/index.js

@@ -0,0 +1,26 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.jaroWinkler = exports.DQIReport = exports.Schema = exports.DQIEngine = void 0;
+var engine_js_1 = require("./engine.js");
+Object.defineProperty(exports, "DQIEngine", { enumerable: true, get: function () { return engine_js_1.DQIEngine; } });
+var schema_js_1 = require("./schema.js");
+Object.defineProperty(exports, "Schema", { enumerable: true, get: function () { return schema_js_1.Schema; } });
+var report_js_1 = require("./report.js");
+Object.defineProperty(exports, "DQIReport", { enumerable: true, get: function () { return report_js_1.DQIReport; } });
+__exportStar(require("./types.js"), exports);
+var uniqueness_js_1 = require("./dimensions/uniqueness.js");
+Object.defineProperty(exports, "jaroWinkler", { enumerable: true, get: function () { return uniqueness_js_1.jaroWinkler; } });

package/dist/report.d.ts

@@ -0,0 +1,9 @@
+import { EvaluationReport } from "./types.js";
+export declare class DQIReport {
+    data: EvaluationReport;
+    constructor(data: EvaluationReport);
+    get overallScore(): number;
+    get grade(): string;
+    toJSON(): string;
+    toHTML(): string;
+}

package/dist/report.js

@@ -0,0 +1,91 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DQIReport = void 0;
+class DQIReport {
+    data;
+    constructor(data) {
+        this.data = data;
+    }
+    get overallScore() {
+        return this.data.overallScore;
+    }
+    get grade() {
+        return this.data.grade;
+    }
+    toJSON() {
+        return JSON.stringify(this.data, null, 2);
+    }
+    toHTML() {
+        const { overallScore, grade, totalRecords, dimensions, allIssues, generatedAt } = this.data;
+        return `<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="UTF-8">
+<meta name="viewport" content="width=device-width, initial-scale=1.0">
+<title>DQI Report</title>
+<style>
+  body { font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; background: #f4f6f9; color: #2d3748; padding: 2rem; margin: 0; }
+  .header { margin-bottom: 2rem; }
+  .card-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); gap: 1rem; margin-bottom: 2rem; }
+  .card { background: #fff; border-radius: 10px; padding: 1.5rem; text-align: center; box-shadow: 0 1px 3px rgba(0,0,0,0.1); }
+  .score-val { font-size: 2.2rem; font-weight: 800; color: #2b6cb0; }
+  .badge { display: inline-block; padding: 0.25rem 0.5rem; border-radius: 4px; font-weight: bold; }
+  .grade-A { background: #c6f6d5; color: #22543d; }
+  .grade-B { background: #bee3f8; color: #2b6cb0; }
+  .grade-C { background: #feebc8; color: #7b341e; }
+  .grade-D { background: #fed7d7; color: #742a2a; }
+  .grade-F { background: #e2e8f0; color: #1a202c; }
+  table { width: 100%; border-collapse: collapse; background: #fff; border-radius: 8px; overflow: hidden; margin-top: 1rem; }
+  th, td { padding: 0.75rem 1rem; text-align: left; border-bottom: 1px solid #edf2f7; font-size: 0.85rem; }
+  th { background: #edf2f7; color: #4a5568; }
+</style>
+</head>
+<body>
+<div class="header">
+  <h1>Data Quality Index Report</h1>
+  <p>Total records: ${totalRecords} | Generated: ${generatedAt}</p>
+</div>
+
+<div class="card-grid">
+  <div class="card">
+    <div>Overall DQI</div>
+    <div class="score-val">${overallScore}</div>
+    <span class="badge grade-${grade}">Grade ${grade}</span>
+  </div>
+  <div class="card">
+    <div>Completeness</div>
+    <div class="score-val">${dimensions.completeness.score}</div>
+  </div>
+  <div class="card">
+    <div>Validity</div>
+    <div class="score-val">${dimensions.validity.score}</div>
+  </div>
+  <div class="card">
+    <div>Uniqueness</div>
+    <div class="score-val">${dimensions.uniqueness.score}</div>
+  </div>
+  <div class="card">
+    <div>Outliers</div>
+    <div class="score-val">${dimensions.outliers.score}</div>
+  </div>
+</div>
+
+<h2>Audit Issues (${allIssues.length})</h2>
+${allIssues.length === 0
+            ? "<p>No issues detected.</p>"
+            : `<table>
+  <thead>
+    <tr><th>Row</th><th>Field</th><th>Dimension</th><th>Reason</th></tr>
+  </thead>
+  <tbody>
+    ${allIssues
+                .slice(0, 500)
+                .map((i) => `<tr><td>${i.rowIndex}</td><td>${i.field}</td><td>${i.dimension}</td><td>${i.reason}</td></tr>`)
+                .join("")}
+  </tbody>
+</table>`}
+</body>
+</html>`;
+    }
+}
+exports.DQIReport = DQIReport;

package/dist/schema.d.ts

@@ -0,0 +1,9 @@
+import { FieldDefinition, DimensionWeights } from "./types.js";
+export declare class Schema {
+    fields: Record<string, FieldDefinition>;
+    weights: DimensionWeights;
+    entityLabel: string;
+    constructor(fields: Record<string, FieldDefinition>, weights?: DimensionWeights, entityLabel?: string);
+    static fromYAML(yamlString: string): Schema;
+    static infer(csvContent: string): Schema;
+}