@futux/dqi 0.0.0-stage → 0.1.0

package/dist/schema.js

@@ -0,0 +1,101 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Schema = void 0;
+const types_js_1 = require("./types.js");
+const js_yaml_1 = __importDefault(require("js-yaml"));
+const papaparse_1 = __importDefault(require("papaparse"));
+class Schema {
+    fields;
+    weights;
+    entityLabel;
+    constructor(fields, weights = { completeness: 0.25, validity: 0.35, uniqueness: 0.25, outliers: 0.15 }, entityLabel = "Record") {
+        this.fields = fields;
+        this.weights = weights;
+        this.entityLabel = entityLabel;
+    }
+    static fromYAML(yamlString) {
+        const data = js_yaml_1.default.load(yamlString);
+        const entityLabel = data?.entity_label ?? "Record";
+        const weightsCfg = data?.dimensions?.weights ?? {};
+        const weights = {
+            completeness: weightsCfg.completeness ?? 0.25,
+            validity: weightsCfg.validity ?? 0.35,
+            uniqueness: weightsCfg.uniqueness ?? 0.25,
+            outliers: weightsCfg.outliers ?? 0.15,
+        };
+        const fields = {};
+        if (data?.fields) {
+            for (const [name, cfg] of Object.entries(data.fields)) {
+                fields[name] = {
+                    name,
+                    type: cfg["type"] || types_js_1.FieldType.TEXT,
+                    required: Boolean(cfg["required"]),
+                    uniqueKey: Boolean(cfg["unique_key"]),
+                    fuzzyKey: Boolean(cfg["fuzzy_key"]),
+                    min: cfg["min"],
+                    max: cfg["max"],
+                    pattern: cfg["pattern"],
+                    allowedValues: cfg["allowed_values"],
+                };
+            }
+        }
+        return new Schema(fields, weights, entityLabel);
+    }
+    static infer(csvContent) {
+        const parsed = papaparse_1.default.parse(csvContent, {
+            header: true,
+            skipEmptyLines: true,
+            preview: 500,
+        });
+        const rows = parsed.data;
+        const fields = {};
+        const columns = parsed.meta.fields ?? [];
+        for (const col of columns) {
+            const lower = col.toLowerCase();
+            let detectedType = types_js_1.FieldType.TEXT;
+            if (lower.includes("phone") || lower.includes("tel") || lower.includes("mobile")) {
+                detectedType = types_js_1.FieldType.PHONE;
+            }
+            else if (lower.includes("email")) {
+                detectedType = types_js_1.FieldType.EMAIL;
+            }
+            else if (lower.includes("lat") || lower.includes("latitude")) {
+                detectedType = types_js_1.FieldType.LATITUDE;
+            }
+            else if (lower.includes("lon") || lower.includes("lng") || lower.includes("longitude")) {
+                detectedType = types_js_1.FieldType.LONGITUDE;
+            }
+            else if (lower.includes("date") || lower.includes("dob") || lower.includes("birth")) {
+                detectedType = types_js_1.FieldType.DATE;
+            }
+            else if (lower.includes("id") || lower.includes("nin") || lower.includes("uuid") || lower.includes("ref")) {
+                detectedType = types_js_1.FieldType.ID;
+            }
+            else {
+                // Sample inspection
+                const sampleVals = rows.map((r) => r[col]).filter((v) => v !== undefined && v !== null && v.trim() !== "");
+                const allNumeric = sampleVals.length > 0 && sampleVals.every((v) => !isNaN(Number(v)));
+                if (allNumeric) {
+                    const allInts = sampleVals.every((v) => Number.isInteger(Number(v)));
+                    detectedType = allInts ? types_js_1.FieldType.INTEGER : types_js_1.FieldType.FLOAT;
+                }
+            }
+            const nonNullCount = rows.filter((r) => r[col] !== undefined && r[col] !== null && r[col].trim() !== "").length;
+            const isRequired = rows.length > 0 && nonNullCount / rows.length > 0.95;
+            const isUniqueKey = detectedType === types_js_1.FieldType.ID || lower.includes("id");
+            fields[col] = {
+                name: col,
+                type: detectedType,
+                required: isRequired,
+                uniqueKey: isUniqueKey,
+                fuzzyKey: detectedType === types_js_1.FieldType.TEXT && (lower.includes("name") || lower.includes("title")),
+                autoInferred: true,
+            };
+        }
+        return new Schema(fields);
+    }
+}
+exports.Schema = Schema;

package/dist/types.d.ts

@@ -0,0 +1,58 @@
+export declare enum FieldType {
+    TEXT = "text",
+    INTEGER = "integer",
+    FLOAT = "float",
+    DATE = "date",
+    PHONE = "phone",
+    EMAIL = "email",
+    LATITUDE = "latitude",
+    LONGITUDE = "longitude",
+    ENUM = "enum",
+    ID = "id",
+    BOOLEAN = "boolean"
+}
+export interface FieldDefinition {
+    name: string;
+    type: FieldType;
+    required?: boolean;
+    uniqueKey?: boolean;
+    fuzzyKey?: boolean;
+    min?: number | string;
+    max?: number | string;
+    pattern?: string;
+    allowedValues?: string[];
+    autoInferred?: boolean;
+}
+export interface DimensionWeights {
+    completeness: number;
+    validity: number;
+    uniqueness: number;
+    outliers: number;
+}
+export interface IssueItem {
+    rowIndex: number;
+    field: string;
+    dimension: "completeness" | "validity" | "uniqueness" | "outliers";
+    reason: string;
+    value?: string | number | null;
+    duplicateOf?: number;
+}
+export interface DimensionResult {
+    score: number;
+    issues: IssueItem[];
+    [key: string]: unknown;
+}
+export interface EvaluationReport {
+    overallScore: number;
+    grade: "A" | "B" | "C" | "D" | "F";
+    totalRecords: number;
+    dimensions: {
+        completeness: DimensionResult;
+        validity: DimensionResult;
+        uniqueness: DimensionResult;
+        outliers: DimensionResult;
+    };
+    weights: DimensionWeights;
+    allIssues: IssueItem[];
+    generatedAt: string;
+}

package/dist/types.js

@@ -0,0 +1,17 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.FieldType = void 0;
+var FieldType;
+(function (FieldType) {
+    FieldType["TEXT"] = "text";
+    FieldType["INTEGER"] = "integer";
+    FieldType["FLOAT"] = "float";
+    FieldType["DATE"] = "date";
+    FieldType["PHONE"] = "phone";
+    FieldType["EMAIL"] = "email";
+    FieldType["LATITUDE"] = "latitude";
+    FieldType["LONGITUDE"] = "longitude";
+    FieldType["ENUM"] = "enum";
+    FieldType["ID"] = "id";
+    FieldType["BOOLEAN"] = "boolean";
+})(FieldType || (exports.FieldType = FieldType = {}));

package/package.json

@@ -1,6 +1,32 @@
 {
   "name": "@futux/dqi",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.1.0",
+  "description": "Schema-agnostic Data Quality Index library for Node.js and browser environments",
+  "main": "dist/index.js",
+  "module": "dist/index.mjs",
+  "types": "dist/index.d.ts",
+  "scripts": {
+    "build": "tsc",
+    "test": "vitest run"
+  },
+  "keywords": [
+    "data-quality",
+    "dqi",
+    "data-governance",
+    "dpi",
+    "audit"
+  ],
+  "author": "Daniel Olagunju <d.olagunju@futuxconsult.com>, Temitope Olarewaju <t.olarewaju@futuxconsult.com>",
+  "license": "Apache-2.0",
+  "dependencies": {
+    "js-yaml": "^4.1.0",
+    "papaparse": "^5.4.1"
+  },
+  "devDependencies": {
+    "@types/js-yaml": "^4.0.9",
+    "@types/node": "^22.0.0",
+    "@types/papaparse": "^5.3.14",
+    "typescript": "^5.5.0",
+    "vitest": "^2.0.0"
+  }
+}

package/src/dimensions/completeness.ts

@@ -0,0 +1,35 @@
+import { Schema } from "../schema.js";
+import { DimensionResult, IssueItem } from "../types.js";
+
+export function scoreCompleteness(rows: Record<string, string>[], schema: Schema): DimensionResult {
+  const issues: IssueItem[] = [];
+  const requiredFields = Object.values(schema.fields)
+    .filter((f) => f.required)
+    .map((f) => f.name);
+
+  if (requiredFields.length === 0 || rows.length === 0) {
+    return { score: 100, issues: [] };
+  }
+
+  let totalRequiredSlots = rows.length * requiredFields.length;
+  let missingCount = 0;
+
+  rows.forEach((row, rowIndex) => {
+    for (const field of requiredFields) {
+      const val = row[field];
+      if (val === undefined || val === null || val.trim() === "") {
+        missingCount++;
+        issues.push({
+          rowIndex,
+          field,
+          dimension: "completeness",
+          reason: "required_field_empty",
+          value: val ?? null,
+        });
+      }
+    }
+  });
+
+  const score = Math.max(0, Math.min(100, (1 - missingCount / totalRequiredSlots) * 100));
+  return { score: Number(score.toFixed(2)), issues };
+}

package/src/dimensions/outliers.ts

@@ -0,0 +1,67 @@
+import { Schema } from "../schema.js";
+import { DimensionResult, IssueItem, FieldType } from "../types.js";
+
+function quantile(arr: number[], q: number): number {
+  const sorted = [...arr].sort((a, b) => a - b);
+  const pos = (sorted.length - 1) * q;
+  const base = Math.floor(pos);
+  const rest = pos - base;
+  if (sorted[base + 1] !== undefined) {
+    return sorted[base]! + rest * (sorted[base + 1]! - sorted[base]!);
+  }
+  return sorted[base]!;
+}
+
+export function scoreOutliers(rows: Record<string, string>[], schema: Schema): DimensionResult {
+  const issues: IssueItem[] = [];
+  const flaggedRows = new Set<number>();
+
+  const numericFields = Object.values(schema.fields)
+    .filter((f) => [FieldType.FLOAT, FieldType.INTEGER].includes(f.type))
+    .map((f) => f.name);
+
+  if (numericFields.length === 0 || rows.length < 4) {
+    return { score: 100, issues: [] };
+  }
+
+  for (const col of numericFields) {
+    const validEntries: Array<{ idx: number; val: number }> = [];
+    rows.forEach((row, rowIndex) => {
+      const v = row[col];
+      if (v !== undefined && v !== null && v.trim() !== "") {
+        const num = Number(v);
+        if (!isNaN(num)) {
+          validEntries.push({ idx: rowIndex, val: num });
+        }
+      }
+    });
+
+    if (validEntries.length < 4) continue;
+
+    const values = validEntries.map((e) => e.val);
+    const q1 = quantile(values, 0.25);
+    const q3 = quantile(values, 0.75);
+    const iqr = q3 - q1;
+    const lowerFence = q1 - 1.5 * iqr;
+    const upperFence = q3 + 1.5 * iqr;
+
+    for (const entry of validEntries) {
+      if (entry.val < lowerFence || entry.val > upperFence) {
+        flaggedRows.add(entry.idx);
+        issues.push({
+          rowIndex: entry.idx,
+          field: col,
+          dimension: "outliers",
+          reason: `iqr_outlier_[${lowerFence.toFixed(2)},${upperFence.toFixed(2)}]`,
+          value: entry.val,
+        });
+      }
+    }
+  }
+
+  const score = rows.length === 0 ? 100 : Math.max(0, (1 - flaggedRows.size / rows.length) * 100);
+  return {
+    score: Number(score.toFixed(2)),
+    issues,
+  };
+}

package/src/dimensions/uniqueness.ts

@@ -0,0 +1,121 @@
+import { Schema } from "../schema.js";
+import { DimensionResult, IssueItem } from "../types.js";
+
+export function jaroWinkler(s1: string, s2: string): number {
+  if (s1 === s2) return 1.0;
+  const len1 = s1.length;
+  const len2 = s2.length;
+  if (len1 === 0 || len2 === 0) return 0.0;
+
+  const matchDistance = Math.floor(Math.max(len1, len2) / 2) - 1;
+  const s1Matches = new Array(len1).fill(false);
+  const s2Matches = new Array(len2).fill(false);
+
+  let matches = 0;
+  for (let i = 0; i < len1; i++) {
+    const start = Math.max(0, i - matchDistance);
+    const end = Math.min(i + matchDistance + 1, len2);
+    for (let j = start; j < end; j++) {
+      if (s2Matches[j] || s1[i] !== s2[j]) continue;
+      s1Matches[i] = true;
+      s2Matches[j] = true;
+      matches++;
+      break;
+    }
+  }
+
+  if (matches === 0) return 0.0;
+
+  let transpositions = 0;
+  let k = 0;
+  for (let i = 0; i < len1; i++) {
+    if (!s1Matches[i]) continue;
+    while (!s2Matches[k]) k++;
+    if (s1[i] !== s2[k]) transpositions++;
+    k++;
+  }
+
+  const jaro = (matches / len1 + matches / len2 + (matches - transpositions / 2) / matches) / 3;
+  let prefix = 0;
+  for (let i = 0; i < Math.min(4, Math.min(len1, len2)); i++) {
+    if (s1[i] === s2[i]) prefix++;
+    else break;
+  }
+
+  return jaro + prefix * 0.1 * (1 - jaro);
+}
+
+export function scoreUniqueness(rows: Record<string, string>[], schema: Schema): DimensionResult {
+  const issues: IssueItem[] = [];
+  const flaggedRows = new Set<number>();
+  let exactDuplicates = 0;
+
+  const uniqueKeyFields = Object.values(schema.fields)
+    .filter((f) => f.uniqueKey)
+    .map((f) => f.name);
+
+  // Exact duplicates
+  for (const col of uniqueKeyFields) {
+    const seen = new Map<string, number>();
+    rows.forEach((row, rowIndex) => {
+      const val = row[col];
+      if (!val || val.trim() === "") return;
+      const key = val.trim();
+      if (seen.has(key)) {
+        exactDuplicates++;
+        flaggedRows.add(rowIndex);
+        issues.push({
+          rowIndex,
+          field: col,
+          dimension: "uniqueness",
+          reason: "exact_duplicate_key",
+          duplicateOf: seen.get(key),
+        });
+      } else {
+        seen.set(key, rowIndex);
+      }
+    });
+  }
+
+  // Soft fuzzy candidate pairs
+  const fuzzyFields = Object.values(schema.fields)
+    .filter((f) => f.fuzzyKey)
+    .map((f) => f.name);
+
+  for (const col of fuzzyFields) {
+    const blocks = new Map<string, Array<{ idx: number; val: string }>>();
+    rows.forEach((row, rowIndex) => {
+      const val = row[col];
+      if (!val || val.trim() === "") return;
+      const clean = val.trim().toLowerCase();
+      const bucket = clean[0] ?? "_";
+      if (!blocks.has(bucket)) blocks.set(bucket, []);
+      blocks.get(bucket)!.push({ idx: rowIndex, val: clean });
+    });
+
+    for (const entries of blocks.values()) {
+      for (let i = 0; i < entries.length; i++) {
+        for (let j = i + 1; j < entries.length; j++) {
+          const sim = jaroWinkler(entries[i]!.val, entries[j]!.val);
+          if (sim >= 0.85) {
+            flaggedRows.add(entries[j]!.idx);
+            issues.push({
+              rowIndex: entries[j]!.idx,
+              field: col,
+              dimension: "uniqueness",
+              reason: "fuzzy_duplicate_candidate",
+              duplicateOf: entries[i]!.idx,
+            });
+          }
+        }
+      }
+    }
+  }
+
+  const score = rows.length === 0 ? 100 : Math.max(0, (1 - flaggedRows.size / rows.length) * 100);
+  return {
+    score: Number(score.toFixed(2)),
+    exactDuplicateCount: exactDuplicates,
+    issues,
+  };
+}

package/src/dimensions/validity.ts

@@ -0,0 +1,133 @@
+import { Schema } from "../schema.js";
+import { DimensionResult, IssueItem, FieldType } from "../types.js";
+
+const PHONE_REGEX = /^\+?[\d\s\-().]{7,15}$/;
+const EMAIL_REGEX = /^[^@\s]+@[^@\s]+\.[^@\s]+$/;
+
+export function scoreValidity(rows: Record<string, string>[], schema: Schema): DimensionResult {
+  const issues: IssueItem[] = [];
+  let checkedSlots = 0;
+  let invalidSlots = 0;
+
+  rows.forEach((row, rowIndex) => {
+    for (const [colName, fdef] of Object.entries(schema.fields)) {
+      const val = row[colName];
+      if (val === undefined || val === null || val.trim() === "") {
+        continue;
+      }
+
+      checkedSlots++;
+      const cleanVal = val.trim();
+      let isValid = true;
+      let reason = "";
+
+      switch (fdef.type) {
+        case FieldType.INTEGER: {
+          const num = Number(cleanVal);
+          if (isNaN(num) || !Number.isInteger(num)) {
+            isValid = false;
+            reason = "invalid_integer_format";
+          } else {
+            if (fdef.min !== undefined && num < Number(fdef.min)) {
+              isValid = false;
+              reason = `integer_below_min_${fdef.min}`;
+            }
+            if (fdef.max !== undefined && num > Number(fdef.max)) {
+              isValid = false;
+              reason = `integer_above_max_${fdef.max}`;
+            }
+          }
+          break;
+        }
+        case FieldType.FLOAT: {
+          const num = Number(cleanVal);
+          if (isNaN(num)) {
+            isValid = false;
+            reason = "invalid_float_format";
+          } else {
+            if (fdef.min !== undefined && num < Number(fdef.min)) {
+              isValid = false;
+              reason = `float_below_min_${fdef.min}`;
+            }
+            if (fdef.max !== undefined && num > Number(fdef.max)) {
+              isValid = false;
+              reason = `float_above_max_${fdef.max}`;
+            }
+          }
+          break;
+        }
+        case FieldType.LATITUDE: {
+          const num = Number(cleanVal);
+          const minLat = fdef.min !== undefined ? Number(fdef.min) : -90.0;
+          const maxLat = fdef.max !== undefined ? Number(fdef.max) : 90.0;
+          if (isNaN(num) || num < minLat || num > maxLat) {
+            isValid = false;
+            reason = `latitude_out_of_bounds_[${minLat},${maxLat}]`;
+          }
+          break;
+        }
+        case FieldType.LONGITUDE: {
+          const num = Number(cleanVal);
+          const minLon = fdef.min !== undefined ? Number(fdef.min) : -180.0;
+          const maxLon = fdef.max !== undefined ? Number(fdef.max) : 180.0;
+          if (isNaN(num) || num < minLon || num > maxLon) {
+            isValid = false;
+            reason = `longitude_out_of_bounds_[${minLon},${maxLon}]`;
+          }
+          break;
+        }
+        case FieldType.PHONE: {
+          if (!PHONE_REGEX.test(cleanVal)) {
+            isValid = false;
+            reason = "invalid_phone_format";
+          }
+          break;
+        }
+        case FieldType.EMAIL: {
+          if (!EMAIL_REGEX.test(cleanVal)) {
+            isValid = false;
+            reason = "invalid_email_format";
+          }
+          break;
+        }
+        case FieldType.ENUM: {
+          if (fdef.allowedValues && !fdef.allowedValues.includes(cleanVal)) {
+            isValid = false;
+            reason = `value_not_in_allowed_enum`;
+          }
+          break;
+        }
+        case FieldType.DATE: {
+          const parsed = Date.parse(cleanVal);
+          if (isNaN(parsed)) {
+            isValid = false;
+            reason = "invalid_date_format";
+          }
+          break;
+        }
+        case FieldType.BOOLEAN: {
+          const boolLower = cleanVal.toLowerCase();
+          if (!["true", "false", "1", "0", "yes", "no"].includes(boolLower)) {
+            isValid = false;
+            reason = "invalid_boolean_format";
+          }
+          break;
+        }
+      }
+
+      if (!isValid) {
+        invalidSlots++;
+        issues.push({
+          rowIndex,
+          field: colName,
+          dimension: "validity",
+          reason,
+          value: cleanVal,
+        });
+      }
+    }
+  });
+
+  const score = checkedSlots === 0 ? 100 : Math.max(0, Math.min(100, (1 - invalidSlots / checkedSlots) * 100));
+  return { score: Number(score.toFixed(2)), issues };
+}

package/src/engine.ts

@@ -0,0 +1,79 @@
+import { Schema } from "./schema.js";
+import { DQIReport } from "./report.js";
+import { EvaluationReport, IssueItem } from "./types.js";
+import { scoreCompleteness } from "./dimensions/completeness.js";
+import { scoreValidity } from "./dimensions/validity.js";
+import { scoreUniqueness } from "./dimensions/uniqueness.js";
+import { scoreOutliers } from "./dimensions/outliers.js";
+import Papa from "papaparse";
+
+export class DQIEngine {
+  public schema: Schema | null;
+
+  constructor(schema: Schema | null = null) {
+    this.schema = schema;
+  }
+
+  evaluate(csvInput: string | Record<string, string>[]): DQIReport {
+    let rows: Record<string, string>[];
+
+    if (typeof csvInput === "string") {
+      const parsed = Papa.parse<Record<string, string>>(csvInput, {
+        header: true,
+        skipEmptyLines: true,
+      });
+      rows = parsed.data;
+      if (!this.schema) {
+        this.schema = Schema.infer(csvInput);
+      }
+    } else {
+      rows = csvInput;
+      if (!this.schema) {
+        throw new Error("Schema must be provided when evaluating raw row arrays.");
+      }
+    }
+
+    const completeness = scoreCompleteness(rows, this.schema);
+    const validity = scoreValidity(rows, this.schema);
+    const uniqueness = scoreUniqueness(rows, this.schema);
+    const outliers = scoreOutliers(rows, this.schema);
+
+    const w = this.schema.weights;
+    const overall =
+      completeness.score * w.completeness +
+      validity.score * w.validity +
+      uniqueness.score * w.uniqueness +
+      outliers.score * w.outliers;
+
+    const roundOverall = Number(overall.toFixed(2));
+    let grade: "A" | "B" | "C" | "D" | "F" = "F";
+    if (roundOverall >= 90) grade = "A";
+    else if (roundOverall >= 75) grade = "B";
+    else if (roundOverall >= 60) grade = "C";
+    else if (roundOverall >= 45) grade = "D";
+
+    const allIssues: IssueItem[] = [
+      ...completeness.issues,
+      ...validity.issues,
+      ...uniqueness.issues,
+      ...outliers.issues,
+    ];
+
+    const reportData: EvaluationReport = {
+      overallScore: roundOverall,
+      grade,
+      totalRecords: rows.length,
+      dimensions: {
+        completeness,
+        validity,
+        uniqueness,
+        outliers,
+      },
+      weights: w,
+      allIssues,
+      generatedAt: new Date().toISOString(),
+    };
+
+    return new DQIReport(reportData);
+  }
+}

package/src/index.ts

@@ -0,0 +1,5 @@
+export { DQIEngine } from "./engine.js";
+export { Schema } from "./schema.js";
+export { DQIReport } from "./report.js";
+export * from "./types.js";
+export { jaroWinkler } from "./dimensions/uniqueness.js";