@futux/dqi 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +122 -2
- package/dist/dimensions/completeness.d.ts +3 -0
- package/dist/dimensions/completeness.js +31 -0
- package/dist/dimensions/outliers.d.ts +3 -0
- package/dist/dimensions/outliers.js +61 -0
- package/dist/dimensions/uniqueness.d.ts +4 -0
- package/dist/dimensions/uniqueness.js +122 -0
- package/dist/dimensions/validity.d.ts +3 -0
- package/dist/dimensions/validity.js +130 -0
- package/dist/engine.d.ts +7 -0
- package/dist/engine.js +79 -0
- package/dist/index.d.ts +5 -0
- package/dist/index.js +26 -0
- package/dist/report.d.ts +9 -0
- package/dist/report.js +91 -0
- package/dist/schema.d.ts +9 -0
- package/dist/schema.js +101 -0
- package/dist/types.d.ts +58 -0
- package/dist/types.js +17 -0
- package/package.json +30 -4
- package/src/dimensions/completeness.ts +35 -0
- package/src/dimensions/outliers.ts +67 -0
- package/src/dimensions/uniqueness.ts +121 -0
- package/src/dimensions/validity.ts +133 -0
- package/src/engine.ts +79 -0
- package/src/index.ts +5 -0
- package/src/report.ts +100 -0
- package/src/schema.ts +109 -0
- package/src/types.ts +63 -0
- package/tsconfig.json +16 -0
package/dist/report.js
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.DQIReport = void 0;
|
|
4
|
+
class DQIReport {
|
|
5
|
+
data;
|
|
6
|
+
constructor(data) {
|
|
7
|
+
this.data = data;
|
|
8
|
+
}
|
|
9
|
+
get overallScore() {
|
|
10
|
+
return this.data.overallScore;
|
|
11
|
+
}
|
|
12
|
+
get grade() {
|
|
13
|
+
return this.data.grade;
|
|
14
|
+
}
|
|
15
|
+
toJSON() {
|
|
16
|
+
return JSON.stringify(this.data, null, 2);
|
|
17
|
+
}
|
|
18
|
+
toHTML() {
|
|
19
|
+
const { overallScore, grade, totalRecords, dimensions, allIssues, generatedAt } = this.data;
|
|
20
|
+
return `<!DOCTYPE html>
|
|
21
|
+
<html lang="en">
|
|
22
|
+
<head>
|
|
23
|
+
<meta charset="UTF-8">
|
|
24
|
+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
25
|
+
<title>DQI Report</title>
|
|
26
|
+
<style>
|
|
27
|
+
body { font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; background: #f4f6f9; color: #2d3748; padding: 2rem; margin: 0; }
|
|
28
|
+
.header { margin-bottom: 2rem; }
|
|
29
|
+
.card-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); gap: 1rem; margin-bottom: 2rem; }
|
|
30
|
+
.card { background: #fff; border-radius: 10px; padding: 1.5rem; text-align: center; box-shadow: 0 1px 3px rgba(0,0,0,0.1); }
|
|
31
|
+
.score-val { font-size: 2.2rem; font-weight: 800; color: #2b6cb0; }
|
|
32
|
+
.badge { display: inline-block; padding: 0.25rem 0.5rem; border-radius: 4px; font-weight: bold; }
|
|
33
|
+
.grade-A { background: #c6f6d5; color: #22543d; }
|
|
34
|
+
.grade-B { background: #bee3f8; color: #2b6cb0; }
|
|
35
|
+
.grade-C { background: #feebc8; color: #7b341e; }
|
|
36
|
+
.grade-D { background: #fed7d7; color: #742a2a; }
|
|
37
|
+
.grade-F { background: #e2e8f0; color: #1a202c; }
|
|
38
|
+
table { width: 100%; border-collapse: collapse; background: #fff; border-radius: 8px; overflow: hidden; margin-top: 1rem; }
|
|
39
|
+
th, td { padding: 0.75rem 1rem; text-align: left; border-bottom: 1px solid #edf2f7; font-size: 0.85rem; }
|
|
40
|
+
th { background: #edf2f7; color: #4a5568; }
|
|
41
|
+
</style>
|
|
42
|
+
</head>
|
|
43
|
+
<body>
|
|
44
|
+
<div class="header">
|
|
45
|
+
<h1>Data Quality Index Report</h1>
|
|
46
|
+
<p>Total records: ${totalRecords} | Generated: ${generatedAt}</p>
|
|
47
|
+
</div>
|
|
48
|
+
|
|
49
|
+
<div class="card-grid">
|
|
50
|
+
<div class="card">
|
|
51
|
+
<div>Overall DQI</div>
|
|
52
|
+
<div class="score-val">${overallScore}</div>
|
|
53
|
+
<span class="badge grade-${grade}">Grade ${grade}</span>
|
|
54
|
+
</div>
|
|
55
|
+
<div class="card">
|
|
56
|
+
<div>Completeness</div>
|
|
57
|
+
<div class="score-val">${dimensions.completeness.score}</div>
|
|
58
|
+
</div>
|
|
59
|
+
<div class="card">
|
|
60
|
+
<div>Validity</div>
|
|
61
|
+
<div class="score-val">${dimensions.validity.score}</div>
|
|
62
|
+
</div>
|
|
63
|
+
<div class="card">
|
|
64
|
+
<div>Uniqueness</div>
|
|
65
|
+
<div class="score-val">${dimensions.uniqueness.score}</div>
|
|
66
|
+
</div>
|
|
67
|
+
<div class="card">
|
|
68
|
+
<div>Outliers</div>
|
|
69
|
+
<div class="score-val">${dimensions.outliers.score}</div>
|
|
70
|
+
</div>
|
|
71
|
+
</div>
|
|
72
|
+
|
|
73
|
+
<h2>Audit Issues (${allIssues.length})</h2>
|
|
74
|
+
${allIssues.length === 0
|
|
75
|
+
? "<p>No issues detected.</p>"
|
|
76
|
+
: `<table>
|
|
77
|
+
<thead>
|
|
78
|
+
<tr><th>Row</th><th>Field</th><th>Dimension</th><th>Reason</th></tr>
|
|
79
|
+
</thead>
|
|
80
|
+
<tbody>
|
|
81
|
+
${allIssues
|
|
82
|
+
.slice(0, 500)
|
|
83
|
+
.map((i) => `<tr><td>${i.rowIndex}</td><td>${i.field}</td><td>${i.dimension}</td><td>${i.reason}</td></tr>`)
|
|
84
|
+
.join("")}
|
|
85
|
+
</tbody>
|
|
86
|
+
</table>`}
|
|
87
|
+
</body>
|
|
88
|
+
</html>`;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
exports.DQIReport = DQIReport;
|
package/dist/schema.d.ts
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { FieldDefinition, DimensionWeights } from "./types.js";
|
|
2
|
+
export declare class Schema {
|
|
3
|
+
fields: Record<string, FieldDefinition>;
|
|
4
|
+
weights: DimensionWeights;
|
|
5
|
+
entityLabel: string;
|
|
6
|
+
constructor(fields: Record<string, FieldDefinition>, weights?: DimensionWeights, entityLabel?: string);
|
|
7
|
+
static fromYAML(yamlString: string): Schema;
|
|
8
|
+
static infer(csvContent: string): Schema;
|
|
9
|
+
}
|
package/dist/schema.js
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.Schema = void 0;
|
|
7
|
+
const types_js_1 = require("./types.js");
|
|
8
|
+
const js_yaml_1 = __importDefault(require("js-yaml"));
|
|
9
|
+
const papaparse_1 = __importDefault(require("papaparse"));
|
|
10
|
+
class Schema {
|
|
11
|
+
fields;
|
|
12
|
+
weights;
|
|
13
|
+
entityLabel;
|
|
14
|
+
constructor(fields, weights = { completeness: 0.25, validity: 0.35, uniqueness: 0.25, outliers: 0.15 }, entityLabel = "Record") {
|
|
15
|
+
this.fields = fields;
|
|
16
|
+
this.weights = weights;
|
|
17
|
+
this.entityLabel = entityLabel;
|
|
18
|
+
}
|
|
19
|
+
static fromYAML(yamlString) {
|
|
20
|
+
const data = js_yaml_1.default.load(yamlString);
|
|
21
|
+
const entityLabel = data?.entity_label ?? "Record";
|
|
22
|
+
const weightsCfg = data?.dimensions?.weights ?? {};
|
|
23
|
+
const weights = {
|
|
24
|
+
completeness: weightsCfg.completeness ?? 0.25,
|
|
25
|
+
validity: weightsCfg.validity ?? 0.35,
|
|
26
|
+
uniqueness: weightsCfg.uniqueness ?? 0.25,
|
|
27
|
+
outliers: weightsCfg.outliers ?? 0.15,
|
|
28
|
+
};
|
|
29
|
+
const fields = {};
|
|
30
|
+
if (data?.fields) {
|
|
31
|
+
for (const [name, cfg] of Object.entries(data.fields)) {
|
|
32
|
+
fields[name] = {
|
|
33
|
+
name,
|
|
34
|
+
type: cfg["type"] || types_js_1.FieldType.TEXT,
|
|
35
|
+
required: Boolean(cfg["required"]),
|
|
36
|
+
uniqueKey: Boolean(cfg["unique_key"]),
|
|
37
|
+
fuzzyKey: Boolean(cfg["fuzzy_key"]),
|
|
38
|
+
min: cfg["min"],
|
|
39
|
+
max: cfg["max"],
|
|
40
|
+
pattern: cfg["pattern"],
|
|
41
|
+
allowedValues: cfg["allowed_values"],
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
return new Schema(fields, weights, entityLabel);
|
|
46
|
+
}
|
|
47
|
+
static infer(csvContent) {
|
|
48
|
+
const parsed = papaparse_1.default.parse(csvContent, {
|
|
49
|
+
header: true,
|
|
50
|
+
skipEmptyLines: true,
|
|
51
|
+
preview: 500,
|
|
52
|
+
});
|
|
53
|
+
const rows = parsed.data;
|
|
54
|
+
const fields = {};
|
|
55
|
+
const columns = parsed.meta.fields ?? [];
|
|
56
|
+
for (const col of columns) {
|
|
57
|
+
const lower = col.toLowerCase();
|
|
58
|
+
let detectedType = types_js_1.FieldType.TEXT;
|
|
59
|
+
if (lower.includes("phone") || lower.includes("tel") || lower.includes("mobile")) {
|
|
60
|
+
detectedType = types_js_1.FieldType.PHONE;
|
|
61
|
+
}
|
|
62
|
+
else if (lower.includes("email")) {
|
|
63
|
+
detectedType = types_js_1.FieldType.EMAIL;
|
|
64
|
+
}
|
|
65
|
+
else if (lower.includes("lat") || lower.includes("latitude")) {
|
|
66
|
+
detectedType = types_js_1.FieldType.LATITUDE;
|
|
67
|
+
}
|
|
68
|
+
else if (lower.includes("lon") || lower.includes("lng") || lower.includes("longitude")) {
|
|
69
|
+
detectedType = types_js_1.FieldType.LONGITUDE;
|
|
70
|
+
}
|
|
71
|
+
else if (lower.includes("date") || lower.includes("dob") || lower.includes("birth")) {
|
|
72
|
+
detectedType = types_js_1.FieldType.DATE;
|
|
73
|
+
}
|
|
74
|
+
else if (lower.includes("id") || lower.includes("nin") || lower.includes("uuid") || lower.includes("ref")) {
|
|
75
|
+
detectedType = types_js_1.FieldType.ID;
|
|
76
|
+
}
|
|
77
|
+
else {
|
|
78
|
+
// Sample inspection
|
|
79
|
+
const sampleVals = rows.map((r) => r[col]).filter((v) => v !== undefined && v !== null && v.trim() !== "");
|
|
80
|
+
const allNumeric = sampleVals.length > 0 && sampleVals.every((v) => !isNaN(Number(v)));
|
|
81
|
+
if (allNumeric) {
|
|
82
|
+
const allInts = sampleVals.every((v) => Number.isInteger(Number(v)));
|
|
83
|
+
detectedType = allInts ? types_js_1.FieldType.INTEGER : types_js_1.FieldType.FLOAT;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
const nonNullCount = rows.filter((r) => r[col] !== undefined && r[col] !== null && r[col].trim() !== "").length;
|
|
87
|
+
const isRequired = rows.length > 0 && nonNullCount / rows.length > 0.95;
|
|
88
|
+
const isUniqueKey = detectedType === types_js_1.FieldType.ID || lower.includes("id");
|
|
89
|
+
fields[col] = {
|
|
90
|
+
name: col,
|
|
91
|
+
type: detectedType,
|
|
92
|
+
required: isRequired,
|
|
93
|
+
uniqueKey: isUniqueKey,
|
|
94
|
+
fuzzyKey: detectedType === types_js_1.FieldType.TEXT && (lower.includes("name") || lower.includes("title")),
|
|
95
|
+
autoInferred: true,
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
return new Schema(fields);
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
exports.Schema = Schema;
|
package/dist/types.d.ts
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
export declare enum FieldType {
|
|
2
|
+
TEXT = "text",
|
|
3
|
+
INTEGER = "integer",
|
|
4
|
+
FLOAT = "float",
|
|
5
|
+
DATE = "date",
|
|
6
|
+
PHONE = "phone",
|
|
7
|
+
EMAIL = "email",
|
|
8
|
+
LATITUDE = "latitude",
|
|
9
|
+
LONGITUDE = "longitude",
|
|
10
|
+
ENUM = "enum",
|
|
11
|
+
ID = "id",
|
|
12
|
+
BOOLEAN = "boolean"
|
|
13
|
+
}
|
|
14
|
+
export interface FieldDefinition {
|
|
15
|
+
name: string;
|
|
16
|
+
type: FieldType;
|
|
17
|
+
required?: boolean;
|
|
18
|
+
uniqueKey?: boolean;
|
|
19
|
+
fuzzyKey?: boolean;
|
|
20
|
+
min?: number | string;
|
|
21
|
+
max?: number | string;
|
|
22
|
+
pattern?: string;
|
|
23
|
+
allowedValues?: string[];
|
|
24
|
+
autoInferred?: boolean;
|
|
25
|
+
}
|
|
26
|
+
export interface DimensionWeights {
|
|
27
|
+
completeness: number;
|
|
28
|
+
validity: number;
|
|
29
|
+
uniqueness: number;
|
|
30
|
+
outliers: number;
|
|
31
|
+
}
|
|
32
|
+
export interface IssueItem {
|
|
33
|
+
rowIndex: number;
|
|
34
|
+
field: string;
|
|
35
|
+
dimension: "completeness" | "validity" | "uniqueness" | "outliers";
|
|
36
|
+
reason: string;
|
|
37
|
+
value?: string | number | null;
|
|
38
|
+
duplicateOf?: number;
|
|
39
|
+
}
|
|
40
|
+
export interface DimensionResult {
|
|
41
|
+
score: number;
|
|
42
|
+
issues: IssueItem[];
|
|
43
|
+
[key: string]: unknown;
|
|
44
|
+
}
|
|
45
|
+
export interface EvaluationReport {
|
|
46
|
+
overallScore: number;
|
|
47
|
+
grade: "A" | "B" | "C" | "D" | "F";
|
|
48
|
+
totalRecords: number;
|
|
49
|
+
dimensions: {
|
|
50
|
+
completeness: DimensionResult;
|
|
51
|
+
validity: DimensionResult;
|
|
52
|
+
uniqueness: DimensionResult;
|
|
53
|
+
outliers: DimensionResult;
|
|
54
|
+
};
|
|
55
|
+
weights: DimensionWeights;
|
|
56
|
+
allIssues: IssueItem[];
|
|
57
|
+
generatedAt: string;
|
|
58
|
+
}
|
package/dist/types.js
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.FieldType = void 0;
|
|
4
|
+
var FieldType;
|
|
5
|
+
(function (FieldType) {
|
|
6
|
+
FieldType["TEXT"] = "text";
|
|
7
|
+
FieldType["INTEGER"] = "integer";
|
|
8
|
+
FieldType["FLOAT"] = "float";
|
|
9
|
+
FieldType["DATE"] = "date";
|
|
10
|
+
FieldType["PHONE"] = "phone";
|
|
11
|
+
FieldType["EMAIL"] = "email";
|
|
12
|
+
FieldType["LATITUDE"] = "latitude";
|
|
13
|
+
FieldType["LONGITUDE"] = "longitude";
|
|
14
|
+
FieldType["ENUM"] = "enum";
|
|
15
|
+
FieldType["ID"] = "id";
|
|
16
|
+
FieldType["BOOLEAN"] = "boolean";
|
|
17
|
+
})(FieldType || (exports.FieldType = FieldType = {}));
|
package/package.json
CHANGED
|
@@ -1,6 +1,32 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@futux/dqi",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "0.1.1",
|
|
4
|
+
"description": "Schema-agnostic Data Quality Index library for Node.js and browser environments",
|
|
5
|
+
"main": "dist/index.js",
|
|
6
|
+
"module": "dist/index.mjs",
|
|
7
|
+
"types": "dist/index.d.ts",
|
|
8
|
+
"scripts": {
|
|
9
|
+
"build": "tsc",
|
|
10
|
+
"test": "vitest run"
|
|
11
|
+
},
|
|
12
|
+
"keywords": [
|
|
13
|
+
"data-quality",
|
|
14
|
+
"dqi",
|
|
15
|
+
"data-governance",
|
|
16
|
+
"dpi",
|
|
17
|
+
"audit"
|
|
18
|
+
],
|
|
19
|
+
"author": "Daniel Olagunju <d.olagunju@futuxconsult.com>, Temitope Olarewaju <t.olarewaju@futuxconsult.com>",
|
|
20
|
+
"license": "Apache-2.0",
|
|
21
|
+
"dependencies": {
|
|
22
|
+
"js-yaml": "^4.1.0",
|
|
23
|
+
"papaparse": "^5.4.1"
|
|
24
|
+
},
|
|
25
|
+
"devDependencies": {
|
|
26
|
+
"@types/js-yaml": "^4.0.9",
|
|
27
|
+
"@types/node": "^22.0.0",
|
|
28
|
+
"@types/papaparse": "^5.3.14",
|
|
29
|
+
"typescript": "^5.5.0",
|
|
30
|
+
"vitest": "^2.0.0"
|
|
31
|
+
}
|
|
32
|
+
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { Schema } from "../schema.js";
|
|
2
|
+
import { DimensionResult, IssueItem } from "../types.js";
|
|
3
|
+
|
|
4
|
+
export function scoreCompleteness(rows: Record<string, string>[], schema: Schema): DimensionResult {
|
|
5
|
+
const issues: IssueItem[] = [];
|
|
6
|
+
const requiredFields = Object.values(schema.fields)
|
|
7
|
+
.filter((f) => f.required)
|
|
8
|
+
.map((f) => f.name);
|
|
9
|
+
|
|
10
|
+
if (requiredFields.length === 0 || rows.length === 0) {
|
|
11
|
+
return { score: 100, issues: [] };
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
let totalRequiredSlots = rows.length * requiredFields.length;
|
|
15
|
+
let missingCount = 0;
|
|
16
|
+
|
|
17
|
+
rows.forEach((row, rowIndex) => {
|
|
18
|
+
for (const field of requiredFields) {
|
|
19
|
+
const val = row[field];
|
|
20
|
+
if (val === undefined || val === null || val.trim() === "") {
|
|
21
|
+
missingCount++;
|
|
22
|
+
issues.push({
|
|
23
|
+
rowIndex,
|
|
24
|
+
field,
|
|
25
|
+
dimension: "completeness",
|
|
26
|
+
reason: "required_field_empty",
|
|
27
|
+
value: val ?? null,
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
const score = Math.max(0, Math.min(100, (1 - missingCount / totalRequiredSlots) * 100));
|
|
34
|
+
return { score: Number(score.toFixed(2)), issues };
|
|
35
|
+
}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { Schema } from "../schema.js";
|
|
2
|
+
import { DimensionResult, IssueItem, FieldType } from "../types.js";
|
|
3
|
+
|
|
4
|
+
function quantile(arr: number[], q: number): number {
|
|
5
|
+
const sorted = [...arr].sort((a, b) => a - b);
|
|
6
|
+
const pos = (sorted.length - 1) * q;
|
|
7
|
+
const base = Math.floor(pos);
|
|
8
|
+
const rest = pos - base;
|
|
9
|
+
if (sorted[base + 1] !== undefined) {
|
|
10
|
+
return sorted[base]! + rest * (sorted[base + 1]! - sorted[base]!);
|
|
11
|
+
}
|
|
12
|
+
return sorted[base]!;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export function scoreOutliers(rows: Record<string, string>[], schema: Schema): DimensionResult {
|
|
16
|
+
const issues: IssueItem[] = [];
|
|
17
|
+
const flaggedRows = new Set<number>();
|
|
18
|
+
|
|
19
|
+
const numericFields = Object.values(schema.fields)
|
|
20
|
+
.filter((f) => [FieldType.FLOAT, FieldType.INTEGER].includes(f.type))
|
|
21
|
+
.map((f) => f.name);
|
|
22
|
+
|
|
23
|
+
if (numericFields.length === 0 || rows.length < 4) {
|
|
24
|
+
return { score: 100, issues: [] };
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
for (const col of numericFields) {
|
|
28
|
+
const validEntries: Array<{ idx: number; val: number }> = [];
|
|
29
|
+
rows.forEach((row, rowIndex) => {
|
|
30
|
+
const v = row[col];
|
|
31
|
+
if (v !== undefined && v !== null && v.trim() !== "") {
|
|
32
|
+
const num = Number(v);
|
|
33
|
+
if (!isNaN(num)) {
|
|
34
|
+
validEntries.push({ idx: rowIndex, val: num });
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
if (validEntries.length < 4) continue;
|
|
40
|
+
|
|
41
|
+
const values = validEntries.map((e) => e.val);
|
|
42
|
+
const q1 = quantile(values, 0.25);
|
|
43
|
+
const q3 = quantile(values, 0.75);
|
|
44
|
+
const iqr = q3 - q1;
|
|
45
|
+
const lowerFence = q1 - 1.5 * iqr;
|
|
46
|
+
const upperFence = q3 + 1.5 * iqr;
|
|
47
|
+
|
|
48
|
+
for (const entry of validEntries) {
|
|
49
|
+
if (entry.val < lowerFence || entry.val > upperFence) {
|
|
50
|
+
flaggedRows.add(entry.idx);
|
|
51
|
+
issues.push({
|
|
52
|
+
rowIndex: entry.idx,
|
|
53
|
+
field: col,
|
|
54
|
+
dimension: "outliers",
|
|
55
|
+
reason: `iqr_outlier_[${lowerFence.toFixed(2)},${upperFence.toFixed(2)}]`,
|
|
56
|
+
value: entry.val,
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
const score = rows.length === 0 ? 100 : Math.max(0, (1 - flaggedRows.size / rows.length) * 100);
|
|
63
|
+
return {
|
|
64
|
+
score: Number(score.toFixed(2)),
|
|
65
|
+
issues,
|
|
66
|
+
};
|
|
67
|
+
}
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
import { Schema } from "../schema.js";
|
|
2
|
+
import { DimensionResult, IssueItem } from "../types.js";
|
|
3
|
+
|
|
4
|
+
export function jaroWinkler(s1: string, s2: string): number {
|
|
5
|
+
if (s1 === s2) return 1.0;
|
|
6
|
+
const len1 = s1.length;
|
|
7
|
+
const len2 = s2.length;
|
|
8
|
+
if (len1 === 0 || len2 === 0) return 0.0;
|
|
9
|
+
|
|
10
|
+
const matchDistance = Math.floor(Math.max(len1, len2) / 2) - 1;
|
|
11
|
+
const s1Matches = new Array(len1).fill(false);
|
|
12
|
+
const s2Matches = new Array(len2).fill(false);
|
|
13
|
+
|
|
14
|
+
let matches = 0;
|
|
15
|
+
for (let i = 0; i < len1; i++) {
|
|
16
|
+
const start = Math.max(0, i - matchDistance);
|
|
17
|
+
const end = Math.min(i + matchDistance + 1, len2);
|
|
18
|
+
for (let j = start; j < end; j++) {
|
|
19
|
+
if (s2Matches[j] || s1[i] !== s2[j]) continue;
|
|
20
|
+
s1Matches[i] = true;
|
|
21
|
+
s2Matches[j] = true;
|
|
22
|
+
matches++;
|
|
23
|
+
break;
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
if (matches === 0) return 0.0;
|
|
28
|
+
|
|
29
|
+
let transpositions = 0;
|
|
30
|
+
let k = 0;
|
|
31
|
+
for (let i = 0; i < len1; i++) {
|
|
32
|
+
if (!s1Matches[i]) continue;
|
|
33
|
+
while (!s2Matches[k]) k++;
|
|
34
|
+
if (s1[i] !== s2[k]) transpositions++;
|
|
35
|
+
k++;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
const jaro = (matches / len1 + matches / len2 + (matches - transpositions / 2) / matches) / 3;
|
|
39
|
+
let prefix = 0;
|
|
40
|
+
for (let i = 0; i < Math.min(4, Math.min(len1, len2)); i++) {
|
|
41
|
+
if (s1[i] === s2[i]) prefix++;
|
|
42
|
+
else break;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
return jaro + prefix * 0.1 * (1 - jaro);
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export function scoreUniqueness(rows: Record<string, string>[], schema: Schema): DimensionResult {
|
|
49
|
+
const issues: IssueItem[] = [];
|
|
50
|
+
const flaggedRows = new Set<number>();
|
|
51
|
+
let exactDuplicates = 0;
|
|
52
|
+
|
|
53
|
+
const uniqueKeyFields = Object.values(schema.fields)
|
|
54
|
+
.filter((f) => f.uniqueKey)
|
|
55
|
+
.map((f) => f.name);
|
|
56
|
+
|
|
57
|
+
// Exact duplicates
|
|
58
|
+
for (const col of uniqueKeyFields) {
|
|
59
|
+
const seen = new Map<string, number>();
|
|
60
|
+
rows.forEach((row, rowIndex) => {
|
|
61
|
+
const val = row[col];
|
|
62
|
+
if (!val || val.trim() === "") return;
|
|
63
|
+
const key = val.trim();
|
|
64
|
+
if (seen.has(key)) {
|
|
65
|
+
exactDuplicates++;
|
|
66
|
+
flaggedRows.add(rowIndex);
|
|
67
|
+
issues.push({
|
|
68
|
+
rowIndex,
|
|
69
|
+
field: col,
|
|
70
|
+
dimension: "uniqueness",
|
|
71
|
+
reason: "exact_duplicate_key",
|
|
72
|
+
duplicateOf: seen.get(key),
|
|
73
|
+
});
|
|
74
|
+
} else {
|
|
75
|
+
seen.set(key, rowIndex);
|
|
76
|
+
}
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
// Soft fuzzy candidate pairs
|
|
81
|
+
const fuzzyFields = Object.values(schema.fields)
|
|
82
|
+
.filter((f) => f.fuzzyKey)
|
|
83
|
+
.map((f) => f.name);
|
|
84
|
+
|
|
85
|
+
for (const col of fuzzyFields) {
|
|
86
|
+
const blocks = new Map<string, Array<{ idx: number; val: string }>>();
|
|
87
|
+
rows.forEach((row, rowIndex) => {
|
|
88
|
+
const val = row[col];
|
|
89
|
+
if (!val || val.trim() === "") return;
|
|
90
|
+
const clean = val.trim().toLowerCase();
|
|
91
|
+
const bucket = clean[0] ?? "_";
|
|
92
|
+
if (!blocks.has(bucket)) blocks.set(bucket, []);
|
|
93
|
+
blocks.get(bucket)!.push({ idx: rowIndex, val: clean });
|
|
94
|
+
});
|
|
95
|
+
|
|
96
|
+
for (const entries of blocks.values()) {
|
|
97
|
+
for (let i = 0; i < entries.length; i++) {
|
|
98
|
+
for (let j = i + 1; j < entries.length; j++) {
|
|
99
|
+
const sim = jaroWinkler(entries[i]!.val, entries[j]!.val);
|
|
100
|
+
if (sim >= 0.85) {
|
|
101
|
+
flaggedRows.add(entries[j]!.idx);
|
|
102
|
+
issues.push({
|
|
103
|
+
rowIndex: entries[j]!.idx,
|
|
104
|
+
field: col,
|
|
105
|
+
dimension: "uniqueness",
|
|
106
|
+
reason: "fuzzy_duplicate_candidate",
|
|
107
|
+
duplicateOf: entries[i]!.idx,
|
|
108
|
+
});
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
const score = rows.length === 0 ? 100 : Math.max(0, (1 - flaggedRows.size / rows.length) * 100);
|
|
116
|
+
return {
|
|
117
|
+
score: Number(score.toFixed(2)),
|
|
118
|
+
exactDuplicateCount: exactDuplicates,
|
|
119
|
+
issues,
|
|
120
|
+
};
|
|
121
|
+
}
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
import { Schema } from "../schema.js";
|
|
2
|
+
import { DimensionResult, IssueItem, FieldType } from "../types.js";
|
|
3
|
+
|
|
4
|
+
const PHONE_REGEX = /^\+?[\d\s\-().]{7,15}$/;
|
|
5
|
+
const EMAIL_REGEX = /^[^@\s]+@[^@\s]+\.[^@\s]+$/;
|
|
6
|
+
|
|
7
|
+
export function scoreValidity(rows: Record<string, string>[], schema: Schema): DimensionResult {
|
|
8
|
+
const issues: IssueItem[] = [];
|
|
9
|
+
let checkedSlots = 0;
|
|
10
|
+
let invalidSlots = 0;
|
|
11
|
+
|
|
12
|
+
rows.forEach((row, rowIndex) => {
|
|
13
|
+
for (const [colName, fdef] of Object.entries(schema.fields)) {
|
|
14
|
+
const val = row[colName];
|
|
15
|
+
if (val === undefined || val === null || val.trim() === "") {
|
|
16
|
+
continue;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
checkedSlots++;
|
|
20
|
+
const cleanVal = val.trim();
|
|
21
|
+
let isValid = true;
|
|
22
|
+
let reason = "";
|
|
23
|
+
|
|
24
|
+
switch (fdef.type) {
|
|
25
|
+
case FieldType.INTEGER: {
|
|
26
|
+
const num = Number(cleanVal);
|
|
27
|
+
if (isNaN(num) || !Number.isInteger(num)) {
|
|
28
|
+
isValid = false;
|
|
29
|
+
reason = "invalid_integer_format";
|
|
30
|
+
} else {
|
|
31
|
+
if (fdef.min !== undefined && num < Number(fdef.min)) {
|
|
32
|
+
isValid = false;
|
|
33
|
+
reason = `integer_below_min_${fdef.min}`;
|
|
34
|
+
}
|
|
35
|
+
if (fdef.max !== undefined && num > Number(fdef.max)) {
|
|
36
|
+
isValid = false;
|
|
37
|
+
reason = `integer_above_max_${fdef.max}`;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
break;
|
|
41
|
+
}
|
|
42
|
+
case FieldType.FLOAT: {
|
|
43
|
+
const num = Number(cleanVal);
|
|
44
|
+
if (isNaN(num)) {
|
|
45
|
+
isValid = false;
|
|
46
|
+
reason = "invalid_float_format";
|
|
47
|
+
} else {
|
|
48
|
+
if (fdef.min !== undefined && num < Number(fdef.min)) {
|
|
49
|
+
isValid = false;
|
|
50
|
+
reason = `float_below_min_${fdef.min}`;
|
|
51
|
+
}
|
|
52
|
+
if (fdef.max !== undefined && num > Number(fdef.max)) {
|
|
53
|
+
isValid = false;
|
|
54
|
+
reason = `float_above_max_${fdef.max}`;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
break;
|
|
58
|
+
}
|
|
59
|
+
case FieldType.LATITUDE: {
|
|
60
|
+
const num = Number(cleanVal);
|
|
61
|
+
const minLat = fdef.min !== undefined ? Number(fdef.min) : -90.0;
|
|
62
|
+
const maxLat = fdef.max !== undefined ? Number(fdef.max) : 90.0;
|
|
63
|
+
if (isNaN(num) || num < minLat || num > maxLat) {
|
|
64
|
+
isValid = false;
|
|
65
|
+
reason = `latitude_out_of_bounds_[${minLat},${maxLat}]`;
|
|
66
|
+
}
|
|
67
|
+
break;
|
|
68
|
+
}
|
|
69
|
+
case FieldType.LONGITUDE: {
|
|
70
|
+
const num = Number(cleanVal);
|
|
71
|
+
const minLon = fdef.min !== undefined ? Number(fdef.min) : -180.0;
|
|
72
|
+
const maxLon = fdef.max !== undefined ? Number(fdef.max) : 180.0;
|
|
73
|
+
if (isNaN(num) || num < minLon || num > maxLon) {
|
|
74
|
+
isValid = false;
|
|
75
|
+
reason = `longitude_out_of_bounds_[${minLon},${maxLon}]`;
|
|
76
|
+
}
|
|
77
|
+
break;
|
|
78
|
+
}
|
|
79
|
+
case FieldType.PHONE: {
|
|
80
|
+
if (!PHONE_REGEX.test(cleanVal)) {
|
|
81
|
+
isValid = false;
|
|
82
|
+
reason = "invalid_phone_format";
|
|
83
|
+
}
|
|
84
|
+
break;
|
|
85
|
+
}
|
|
86
|
+
case FieldType.EMAIL: {
|
|
87
|
+
if (!EMAIL_REGEX.test(cleanVal)) {
|
|
88
|
+
isValid = false;
|
|
89
|
+
reason = "invalid_email_format";
|
|
90
|
+
}
|
|
91
|
+
break;
|
|
92
|
+
}
|
|
93
|
+
case FieldType.ENUM: {
|
|
94
|
+
if (fdef.allowedValues && !fdef.allowedValues.includes(cleanVal)) {
|
|
95
|
+
isValid = false;
|
|
96
|
+
reason = `value_not_in_allowed_enum`;
|
|
97
|
+
}
|
|
98
|
+
break;
|
|
99
|
+
}
|
|
100
|
+
case FieldType.DATE: {
|
|
101
|
+
const parsed = Date.parse(cleanVal);
|
|
102
|
+
if (isNaN(parsed)) {
|
|
103
|
+
isValid = false;
|
|
104
|
+
reason = "invalid_date_format";
|
|
105
|
+
}
|
|
106
|
+
break;
|
|
107
|
+
}
|
|
108
|
+
case FieldType.BOOLEAN: {
|
|
109
|
+
const boolLower = cleanVal.toLowerCase();
|
|
110
|
+
if (!["true", "false", "1", "0", "yes", "no"].includes(boolLower)) {
|
|
111
|
+
isValid = false;
|
|
112
|
+
reason = "invalid_boolean_format";
|
|
113
|
+
}
|
|
114
|
+
break;
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
if (!isValid) {
|
|
119
|
+
invalidSlots++;
|
|
120
|
+
issues.push({
|
|
121
|
+
rowIndex,
|
|
122
|
+
field: colName,
|
|
123
|
+
dimension: "validity",
|
|
124
|
+
reason,
|
|
125
|
+
value: cleanVal,
|
|
126
|
+
});
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
});
|
|
130
|
+
|
|
131
|
+
const score = checkedSlots === 0 ? 100 : Math.max(0, Math.min(100, (1 - invalidSlots / checkedSlots) * 100));
|
|
132
|
+
return { score: Number(score.toFixed(2)), issues };
|
|
133
|
+
}
|