@futux/dqi 0.0.0-stage → 0.1.1

package/src/engine.ts

@@ -0,0 +1,79 @@
+import { Schema } from "./schema.js";
+import { DQIReport } from "./report.js";
+import { EvaluationReport, IssueItem } from "./types.js";
+import { scoreCompleteness } from "./dimensions/completeness.js";
+import { scoreValidity } from "./dimensions/validity.js";
+import { scoreUniqueness } from "./dimensions/uniqueness.js";
+import { scoreOutliers } from "./dimensions/outliers.js";
+import Papa from "papaparse";
+
+export class DQIEngine {
+  public schema: Schema | null;
+
+  constructor(schema: Schema | null = null) {
+    this.schema = schema;
+  }
+
+  evaluate(csvInput: string | Record<string, string>[]): DQIReport {
+    let rows: Record<string, string>[];
+
+    if (typeof csvInput === "string") {
+      const parsed = Papa.parse<Record<string, string>>(csvInput, {
+        header: true,
+        skipEmptyLines: true,
+      });
+      rows = parsed.data;
+      if (!this.schema) {
+        this.schema = Schema.infer(csvInput);
+      }
+    } else {
+      rows = csvInput;
+      if (!this.schema) {
+        throw new Error("Schema must be provided when evaluating raw row arrays.");
+      }
+    }
+
+    const completeness = scoreCompleteness(rows, this.schema);
+    const validity = scoreValidity(rows, this.schema);
+    const uniqueness = scoreUniqueness(rows, this.schema);
+    const outliers = scoreOutliers(rows, this.schema);
+
+    const w = this.schema.weights;
+    const overall =
+      completeness.score * w.completeness +
+      validity.score * w.validity +
+      uniqueness.score * w.uniqueness +
+      outliers.score * w.outliers;
+
+    const roundOverall = Number(overall.toFixed(2));
+    let grade: "A" | "B" | "C" | "D" | "F" = "F";
+    if (roundOverall >= 90) grade = "A";
+    else if (roundOverall >= 75) grade = "B";
+    else if (roundOverall >= 60) grade = "C";
+    else if (roundOverall >= 45) grade = "D";
+
+    const allIssues: IssueItem[] = [
+      ...completeness.issues,
+      ...validity.issues,
+      ...uniqueness.issues,
+      ...outliers.issues,
+    ];
+
+    const reportData: EvaluationReport = {
+      overallScore: roundOverall,
+      grade,
+      totalRecords: rows.length,
+      dimensions: {
+        completeness,
+        validity,
+        uniqueness,
+        outliers,
+      },
+      weights: w,
+      allIssues,
+      generatedAt: new Date().toISOString(),
+    };
+
+    return new DQIReport(reportData);
+  }
+}

package/src/index.ts

@@ -0,0 +1,5 @@
+export { DQIEngine } from "./engine.js";
+export { Schema } from "./schema.js";
+export { DQIReport } from "./report.js";
+export * from "./types.js";
+export { jaroWinkler } from "./dimensions/uniqueness.js";

package/src/report.ts

@@ -0,0 +1,100 @@
+import { EvaluationReport } from "./types.js";
+
+export class DQIReport {
+  public data: EvaluationReport;
+
+  constructor(data: EvaluationReport) {
+    this.data = data;
+  }
+
+  get overallScore(): number {
+    return this.data.overallScore;
+  }
+
+  get grade(): string {
+    return this.data.grade;
+  }
+
+  toJSON(): string {
+    return JSON.stringify(this.data, null, 2);
+  }
+
+  toHTML(): string {
+    const { overallScore, grade, totalRecords, dimensions, allIssues, generatedAt } = this.data;
+
+    return `<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="UTF-8">
+<meta name="viewport" content="width=device-width, initial-scale=1.0">
+<title>DQI Report</title>
+<style>
+  body { font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; background: #f4f6f9; color: #2d3748; padding: 2rem; margin: 0; }
+  .header { margin-bottom: 2rem; }
+  .card-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); gap: 1rem; margin-bottom: 2rem; }
+  .card { background: #fff; border-radius: 10px; padding: 1.5rem; text-align: center; box-shadow: 0 1px 3px rgba(0,0,0,0.1); }
+  .score-val { font-size: 2.2rem; font-weight: 800; color: #2b6cb0; }
+  .badge { display: inline-block; padding: 0.25rem 0.5rem; border-radius: 4px; font-weight: bold; }
+  .grade-A { background: #c6f6d5; color: #22543d; }
+  .grade-B { background: #bee3f8; color: #2b6cb0; }
+  .grade-C { background: #feebc8; color: #7b341e; }
+  .grade-D { background: #fed7d7; color: #742a2a; }
+  .grade-F { background: #e2e8f0; color: #1a202c; }
+  table { width: 100%; border-collapse: collapse; background: #fff; border-radius: 8px; overflow: hidden; margin-top: 1rem; }
+  th, td { padding: 0.75rem 1rem; text-align: left; border-bottom: 1px solid #edf2f7; font-size: 0.85rem; }
+  th { background: #edf2f7; color: #4a5568; }
+</style>
+</head>
+<body>
+<div class="header">
+  <h1>Data Quality Index Report</h1>
+  <p>Total records: ${totalRecords} | Generated: ${generatedAt}</p>
+</div>
+
+<div class="card-grid">
+  <div class="card">
+    <div>Overall DQI</div>
+    <div class="score-val">${overallScore}</div>
+    <span class="badge grade-${grade}">Grade ${grade}</span>
+  </div>
+  <div class="card">
+    <div>Completeness</div>
+    <div class="score-val">${dimensions.completeness.score}</div>
+  </div>
+  <div class="card">
+    <div>Validity</div>
+    <div class="score-val">${dimensions.validity.score}</div>
+  </div>
+  <div class="card">
+    <div>Uniqueness</div>
+    <div class="score-val">${dimensions.uniqueness.score}</div>
+  </div>
+  <div class="card">
+    <div>Outliers</div>
+    <div class="score-val">${dimensions.outliers.score}</div>
+  </div>
+</div>
+
+<h2>Audit Issues (${allIssues.length})</h2>
+${
+  allIssues.length === 0
+    ? "<p>No issues detected.</p>"
+    : `<table>
+  <thead>
+    <tr><th>Row</th><th>Field</th><th>Dimension</th><th>Reason</th></tr>
+  </thead>
+  <tbody>
+    ${allIssues
+      .slice(0, 500)
+      .map(
+        (i) =>
+          `<tr><td>${i.rowIndex}</td><td>${i.field}</td><td>${i.dimension}</td><td>${i.reason}</td></tr>`
+      )
+      .join("")}
+  </tbody>
+</table>`
+}
+</body>
+</html>`;
+  }
+}

package/src/schema.ts

@@ -0,0 +1,109 @@
+import { FieldType, FieldDefinition, DimensionWeights } from "./types.js";
+import yaml from "js-yaml";
+import Papa from "papaparse";
+
+export class Schema {
+  public fields: Record<string, FieldDefinition>;
+  public weights: DimensionWeights;
+  public entityLabel: string;
+
+  constructor(
+    fields: Record<string, FieldDefinition>,
+    weights: DimensionWeights = { completeness: 0.25, validity: 0.35, uniqueness: 0.25, outliers: 0.15 },
+    entityLabel: string = "Record"
+  ) {
+    this.fields = fields;
+    this.weights = weights;
+    this.entityLabel = entityLabel;
+  }
+
+  static fromYAML(yamlString: string): Schema {
+    const data = yaml.load(yamlString) as {
+      entity_label?: string;
+      dimensions?: { weights?: Partial<DimensionWeights> };
+      fields?: Record<string, Record<string, unknown>>;
+    };
+
+    const entityLabel = data?.entity_label ?? "Record";
+    const weightsCfg = data?.dimensions?.weights ?? {};
+    const weights: DimensionWeights = {
+      completeness: weightsCfg.completeness ?? 0.25,
+      validity: weightsCfg.validity ?? 0.35,
+      uniqueness: weightsCfg.uniqueness ?? 0.25,
+      outliers: weightsCfg.outliers ?? 0.15,
+    };
+
+    const fields: Record<string, FieldDefinition> = {};
+    if (data?.fields) {
+      for (const [name, cfg] of Object.entries(data.fields)) {
+        fields[name] = {
+          name,
+          type: (cfg["type"] as FieldType) || FieldType.TEXT,
+          required: Boolean(cfg["required"]),
+          uniqueKey: Boolean(cfg["unique_key"]),
+          fuzzyKey: Boolean(cfg["fuzzy_key"]),
+          min: cfg["min"] as number | string | undefined,
+          max: cfg["max"] as number | string | undefined,
+          pattern: cfg["pattern"] as string | undefined,
+          allowedValues: cfg["allowed_values"] as string[] | undefined,
+        };
+      }
+    }
+
+    return new Schema(fields, weights, entityLabel);
+  }
+
+  static infer(csvContent: string): Schema {
+    const parsed = Papa.parse<Record<string, string>>(csvContent, {
+      header: true,
+      skipEmptyLines: true,
+      preview: 500,
+    });
+
+    const rows = parsed.data;
+    const fields: Record<string, FieldDefinition> = {};
+    const columns = parsed.meta.fields ?? [];
+
+    for (const col of columns) {
+      const lower = col.toLowerCase();
+      let detectedType = FieldType.TEXT;
+
+      if (lower.includes("phone") || lower.includes("tel") || lower.includes("mobile")) {
+        detectedType = FieldType.PHONE;
+      } else if (lower.includes("email")) {
+        detectedType = FieldType.EMAIL;
+      } else if (lower.includes("lat") || lower.includes("latitude")) {
+        detectedType = FieldType.LATITUDE;
+      } else if (lower.includes("lon") || lower.includes("lng") || lower.includes("longitude")) {
+        detectedType = FieldType.LONGITUDE;
+      } else if (lower.includes("date") || lower.includes("dob") || lower.includes("birth")) {
+        detectedType = FieldType.DATE;
+      } else if (lower.includes("id") || lower.includes("nin") || lower.includes("uuid") || lower.includes("ref")) {
+        detectedType = FieldType.ID;
+      } else {
+        // Sample inspection
+        const sampleVals = rows.map((r) => r[col]).filter((v) => v !== undefined && v !== null && v.trim() !== "");
+        const allNumeric = sampleVals.length > 0 && sampleVals.every((v) => !isNaN(Number(v)));
+        if (allNumeric) {
+          const allInts = sampleVals.every((v) => Number.isInteger(Number(v)));
+          detectedType = allInts ? FieldType.INTEGER : FieldType.FLOAT;
+        }
+      }
+
+      const nonNullCount = rows.filter((r) => r[col] !== undefined && r[col] !== null && r[col].trim() !== "").length;
+      const isRequired = rows.length > 0 && nonNullCount / rows.length > 0.95;
+      const isUniqueKey = detectedType === FieldType.ID || lower.includes("id");
+
+      fields[col] = {
+        name: col,
+        type: detectedType,
+        required: isRequired,
+        uniqueKey: isUniqueKey,
+        fuzzyKey: detectedType === FieldType.TEXT && (lower.includes("name") || lower.includes("title")),
+        autoInferred: true,
+      };
+    }
+
+    return new Schema(fields);
+  }
+}

package/src/types.ts

@@ -0,0 +1,63 @@
+export enum FieldType {
+  TEXT = "text",
+  INTEGER = "integer",
+  FLOAT = "float",
+  DATE = "date",
+  PHONE = "phone",
+  EMAIL = "email",
+  LATITUDE = "latitude",
+  LONGITUDE = "longitude",
+  ENUM = "enum",
+  ID = "id",
+  BOOLEAN = "boolean",
+}
+
+export interface FieldDefinition {
+  name: string;
+  type: FieldType;
+  required?: boolean;
+  uniqueKey?: boolean;
+  fuzzyKey?: boolean;
+  min?: number | string;
+  max?: number | string;
+  pattern?: string;
+  allowedValues?: string[];
+  autoInferred?: boolean;
+}
+
+export interface DimensionWeights {
+  completeness: number;
+  validity: number;
+  uniqueness: number;
+  outliers: number;
+}
+
+export interface IssueItem {
+  rowIndex: number;
+  field: string;
+  dimension: "completeness" | "validity" | "uniqueness" | "outliers";
+  reason: string;
+  value?: string | number | null;
+  duplicateOf?: number;
+}
+
+export interface DimensionResult {
+  score: number;
+  issues: IssueItem[];
+  [key: string]: unknown;
+}
+
+export interface EvaluationReport {
+  overallScore: number;
+  grade: "A" | "B" | "C" | "D" | "F";
+  totalRecords: number;
+  dimensions: {
+    completeness: DimensionResult;
+    validity: DimensionResult;
+    uniqueness: DimensionResult;
+    outliers: DimensionResult;
+  };
+  weights: DimensionWeights;
+  allIssues: IssueItem[];
+  generatedAt: string;
+}

package/tsconfig.json

@@ -0,0 +1,16 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
+    "declaration": true,
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "strict": true,
+    "noImplicitAny": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true
+  },
+  "include": ["src/**/*"]
+}