@futux/dqi 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +122 -2
- package/dist/dimensions/completeness.d.ts +3 -0
- package/dist/dimensions/completeness.js +31 -0
- package/dist/dimensions/outliers.d.ts +3 -0
- package/dist/dimensions/outliers.js +61 -0
- package/dist/dimensions/uniqueness.d.ts +4 -0
- package/dist/dimensions/uniqueness.js +122 -0
- package/dist/dimensions/validity.d.ts +3 -0
- package/dist/dimensions/validity.js +130 -0
- package/dist/engine.d.ts +7 -0
- package/dist/engine.js +79 -0
- package/dist/index.d.ts +5 -0
- package/dist/index.js +26 -0
- package/dist/report.d.ts +9 -0
- package/dist/report.js +91 -0
- package/dist/schema.d.ts +9 -0
- package/dist/schema.js +101 -0
- package/dist/types.d.ts +58 -0
- package/dist/types.js +17 -0
- package/package.json +30 -4
- package/src/dimensions/completeness.ts +35 -0
- package/src/dimensions/outliers.ts +67 -0
- package/src/dimensions/uniqueness.ts +121 -0
- package/src/dimensions/validity.ts +133 -0
- package/src/engine.ts +79 -0
- package/src/index.ts +5 -0
- package/src/report.ts +100 -0
- package/src/schema.ts +109 -0
- package/src/types.ts +63 -0
- package/tsconfig.json +16 -0
package/README.md
CHANGED
|
@@ -1,3 +1,123 @@
|
|
|
1
|
-
#
|
|
1
|
+
# @futux/dqi (Data Quality Index for JavaScript & TypeScript)
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
[](https://www.npmjs.com/package/@futux/dqi)
|
|
4
|
+
[](https://www.npmjs.com/package/@futux/dqi)
|
|
5
|
+
[](https://opensource.org/licenses/Apache-2.0)
|
|
6
|
+
[](https://futuxagriconsult.github.io/open-dqi-core/)
|
|
7
|
+
|
|
8
|
+
> **Lightweight, schema-agnostic Data Quality Index (DQI) library for Node.js, Next.js, and modern browser runtimes.** Score any tabular dataset (CSV, JSON, array of objects) across four weighted dimensions: **Completeness**, **Validity**, **Uniqueness**, and **Outliers** in milliseconds.
|
|
9
|
+
|
|
10
|
+
---
|
|
11
|
+
|
|
12
|
+
### 🌐 Try the Interactive Web Playground
|
|
13
|
+
|
|
14
|
+
Test `@futux/dqi` directly in your browser:
|
|
15
|
+
👉 **[Launch Live Web Demo (https://futuxagriconsult.github.io/open-dqi-core/)](https://futuxagriconsult.github.io/open-dqi-core/)**
|
|
16
|
+
|
|
17
|
+
---
|
|
18
|
+
|
|
19
|
+
## 🚀 Key Highlights
|
|
20
|
+
|
|
21
|
+
- **Zero-Config Auto-Inference:** Evaluates arbitrary JSON/CSV records out-of-the-box without requiring predefined schemas.
|
|
22
|
+
- **Universal Single Metric:** Outputs an intuitive 0–100 score and academic letter grade (A–F) for easy communication with non-technical stakeholders.
|
|
23
|
+
- **Client & Server Ready:** Runs natively in Node.js, Bun, Deno, Web Workers, and modern browser single-page applications.
|
|
24
|
+
- **Pure JavaScript / Zero Heavy Native Addons:** Extremely small bundle footprint, ultra-fast cold start.
|
|
25
|
+
- **Auditable Defect Breakdown:** Pinpoints exact row numbers, defect types, and violating values for instant remediation.
|
|
26
|
+
|
|
27
|
+
---
|
|
28
|
+
|
|
29
|
+
## 📦 Installation
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
# npm
|
|
33
|
+
npm install @futux/dqi
|
|
34
|
+
|
|
35
|
+
# yarn
|
|
36
|
+
yarn add @futux/dqi
|
|
37
|
+
|
|
38
|
+
# pnpm
|
|
39
|
+
pnpm add @futux/dqi
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
---
|
|
43
|
+
|
|
44
|
+
## 💡 Quick Start
|
|
45
|
+
|
|
46
|
+
### 1. Zero-Config Evaluation
|
|
47
|
+
|
|
48
|
+
Pass any array of tabular objects. `@futux/dqi` automatically infers data types, detects sentinel nulls, checks formats, and flags anomalies:
|
|
49
|
+
|
|
50
|
+
```typescript
|
|
51
|
+
import { DQIEngine } from "@futux/dqi";
|
|
52
|
+
|
|
53
|
+
const dataset = [
|
|
54
|
+
{ id: "REC-101", name: "Ibrahim Musa", phone: "+254712345678", yield_kg: 42.5 },
|
|
55
|
+
{ id: "REC-102", name: "Sarah Cherono", phone: "+254787654321", yield_kg: 38.0 },
|
|
56
|
+
{ id: "REC-103", name: "N/A", phone: "invalid-phone", yield_kg: 9999.0 },
|
|
57
|
+
{ id: "REC-101", name: "Ibrahim Musa", phone: "+254712345678", yield_kg: 42.5 } // Duplicate
|
|
58
|
+
];
|
|
59
|
+
|
|
60
|
+
const engine = new DQIEngine();
|
|
61
|
+
const report = engine.evaluate(dataset);
|
|
62
|
+
|
|
63
|
+
console.log(`Overall DQI: ${report.overall.toFixed(2)}/100 (Grade: ${report.grade})`);
|
|
64
|
+
console.log(`- Completeness: ${report.dimensions.completeness.score.toFixed(1)}%`);
|
|
65
|
+
console.log(`- Validity: ${report.dimensions.validity.score.toFixed(1)}%`);
|
|
66
|
+
console.log(`- Uniqueness: ${report.dimensions.uniqueness.score.toFixed(1)}%`);
|
|
67
|
+
console.log(`- Outliers: ${report.dimensions.outliers.score.toFixed(1)}%`);
|
|
68
|
+
|
|
69
|
+
// List flagged defects
|
|
70
|
+
console.log("Flagged Defects:", report.defects);
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
### 2. Custom Schema & Dimension Weights
|
|
74
|
+
|
|
75
|
+
Configure custom validation rules and weighting:
|
|
76
|
+
|
|
77
|
+
```typescript
|
|
78
|
+
import { DQIEngine, Schema } from "@futux/dqi";
|
|
79
|
+
|
|
80
|
+
const schema = new Schema({
|
|
81
|
+
name: "farmer_registry",
|
|
82
|
+
weights: {
|
|
83
|
+
completeness: 0.35,
|
|
84
|
+
validity: 0.35,
|
|
85
|
+
uniqueness: 0.15,
|
|
86
|
+
outliers: 0.15
|
|
87
|
+
},
|
|
88
|
+
fields: {
|
|
89
|
+
id: { type: "id", required: true, unique: true },
|
|
90
|
+
name: { type: "text", required: true },
|
|
91
|
+
phone: { type: "phone", required: true },
|
|
92
|
+
yield_kg: { type: "numeric", min: 1.0, max: 500.0 }
|
|
93
|
+
}
|
|
94
|
+
});
|
|
95
|
+
|
|
96
|
+
const engine = new DQIEngine(schema);
|
|
97
|
+
const report = engine.evaluate(dataset);
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
---
|
|
101
|
+
|
|
102
|
+
## 🔬 Four Quality Dimensions
|
|
103
|
+
|
|
104
|
+
1. **Completeness (35% default):** Measures missing, null, empty, whitespace-only, and sentinel placeholder strings (`N/A`, `NULL`, `none`, `-`, `?`).
|
|
105
|
+
2. **Validity (35% default):** Audits regex conformance, ISO-8601 dates, E.164 phone formats, GPS coordinate boundaries, and enum lists.
|
|
106
|
+
3. **Uniqueness (15% default):** Identifies exact duplicate rows and primary key collisions.
|
|
107
|
+
4. **Outlier Detection (15% default):** Detects statistical extreme anomalies using Interquartile Range ($1.5 \times \text{IQR}$).
|
|
108
|
+
|
|
109
|
+
---
|
|
110
|
+
|
|
111
|
+
## 🌐 Also Available in Python
|
|
112
|
+
|
|
113
|
+
For Python data pipelines and pandas integration, install the companion package:
|
|
114
|
+
```bash
|
|
115
|
+
pip install open-dqi-core
|
|
116
|
+
```
|
|
117
|
+
See [GitHub Repository](https://github.com/futuxagriconsult/open-dqi-core) for Python CLI and offline HTML reporting details.
|
|
118
|
+
|
|
119
|
+
---
|
|
120
|
+
|
|
121
|
+
## 📄 License
|
|
122
|
+
|
|
123
|
+
Apache-2.0. Developed and maintained by **FutuX Agri-consult Limited** ([info@futuxconsult.com](mailto:info@futuxconsult.com)).
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.scoreCompleteness = scoreCompleteness;
|
|
4
|
+
function scoreCompleteness(rows, schema) {
|
|
5
|
+
const issues = [];
|
|
6
|
+
const requiredFields = Object.values(schema.fields)
|
|
7
|
+
.filter((f) => f.required)
|
|
8
|
+
.map((f) => f.name);
|
|
9
|
+
if (requiredFields.length === 0 || rows.length === 0) {
|
|
10
|
+
return { score: 100, issues: [] };
|
|
11
|
+
}
|
|
12
|
+
let totalRequiredSlots = rows.length * requiredFields.length;
|
|
13
|
+
let missingCount = 0;
|
|
14
|
+
rows.forEach((row, rowIndex) => {
|
|
15
|
+
for (const field of requiredFields) {
|
|
16
|
+
const val = row[field];
|
|
17
|
+
if (val === undefined || val === null || val.trim() === "") {
|
|
18
|
+
missingCount++;
|
|
19
|
+
issues.push({
|
|
20
|
+
rowIndex,
|
|
21
|
+
field,
|
|
22
|
+
dimension: "completeness",
|
|
23
|
+
reason: "required_field_empty",
|
|
24
|
+
value: val ?? null,
|
|
25
|
+
});
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
});
|
|
29
|
+
const score = Math.max(0, Math.min(100, (1 - missingCount / totalRequiredSlots) * 100));
|
|
30
|
+
return { score: Number(score.toFixed(2)), issues };
|
|
31
|
+
}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.scoreOutliers = scoreOutliers;
|
|
4
|
+
const types_js_1 = require("../types.js");
|
|
5
|
+
function quantile(arr, q) {
|
|
6
|
+
const sorted = [...arr].sort((a, b) => a - b);
|
|
7
|
+
const pos = (sorted.length - 1) * q;
|
|
8
|
+
const base = Math.floor(pos);
|
|
9
|
+
const rest = pos - base;
|
|
10
|
+
if (sorted[base + 1] !== undefined) {
|
|
11
|
+
return sorted[base] + rest * (sorted[base + 1] - sorted[base]);
|
|
12
|
+
}
|
|
13
|
+
return sorted[base];
|
|
14
|
+
}
|
|
15
|
+
function scoreOutliers(rows, schema) {
|
|
16
|
+
const issues = [];
|
|
17
|
+
const flaggedRows = new Set();
|
|
18
|
+
const numericFields = Object.values(schema.fields)
|
|
19
|
+
.filter((f) => [types_js_1.FieldType.FLOAT, types_js_1.FieldType.INTEGER].includes(f.type))
|
|
20
|
+
.map((f) => f.name);
|
|
21
|
+
if (numericFields.length === 0 || rows.length < 4) {
|
|
22
|
+
return { score: 100, issues: [] };
|
|
23
|
+
}
|
|
24
|
+
for (const col of numericFields) {
|
|
25
|
+
const validEntries = [];
|
|
26
|
+
rows.forEach((row, rowIndex) => {
|
|
27
|
+
const v = row[col];
|
|
28
|
+
if (v !== undefined && v !== null && v.trim() !== "") {
|
|
29
|
+
const num = Number(v);
|
|
30
|
+
if (!isNaN(num)) {
|
|
31
|
+
validEntries.push({ idx: rowIndex, val: num });
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
});
|
|
35
|
+
if (validEntries.length < 4)
|
|
36
|
+
continue;
|
|
37
|
+
const values = validEntries.map((e) => e.val);
|
|
38
|
+
const q1 = quantile(values, 0.25);
|
|
39
|
+
const q3 = quantile(values, 0.75);
|
|
40
|
+
const iqr = q3 - q1;
|
|
41
|
+
const lowerFence = q1 - 1.5 * iqr;
|
|
42
|
+
const upperFence = q3 + 1.5 * iqr;
|
|
43
|
+
for (const entry of validEntries) {
|
|
44
|
+
if (entry.val < lowerFence || entry.val > upperFence) {
|
|
45
|
+
flaggedRows.add(entry.idx);
|
|
46
|
+
issues.push({
|
|
47
|
+
rowIndex: entry.idx,
|
|
48
|
+
field: col,
|
|
49
|
+
dimension: "outliers",
|
|
50
|
+
reason: `iqr_outlier_[${lowerFence.toFixed(2)},${upperFence.toFixed(2)}]`,
|
|
51
|
+
value: entry.val,
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
const score = rows.length === 0 ? 100 : Math.max(0, (1 - flaggedRows.size / rows.length) * 100);
|
|
57
|
+
return {
|
|
58
|
+
score: Number(score.toFixed(2)),
|
|
59
|
+
issues,
|
|
60
|
+
};
|
|
61
|
+
}
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.jaroWinkler = jaroWinkler;
|
|
4
|
+
exports.scoreUniqueness = scoreUniqueness;
|
|
5
|
+
function jaroWinkler(s1, s2) {
|
|
6
|
+
if (s1 === s2)
|
|
7
|
+
return 1.0;
|
|
8
|
+
const len1 = s1.length;
|
|
9
|
+
const len2 = s2.length;
|
|
10
|
+
if (len1 === 0 || len2 === 0)
|
|
11
|
+
return 0.0;
|
|
12
|
+
const matchDistance = Math.floor(Math.max(len1, len2) / 2) - 1;
|
|
13
|
+
const s1Matches = new Array(len1).fill(false);
|
|
14
|
+
const s2Matches = new Array(len2).fill(false);
|
|
15
|
+
let matches = 0;
|
|
16
|
+
for (let i = 0; i < len1; i++) {
|
|
17
|
+
const start = Math.max(0, i - matchDistance);
|
|
18
|
+
const end = Math.min(i + matchDistance + 1, len2);
|
|
19
|
+
for (let j = start; j < end; j++) {
|
|
20
|
+
if (s2Matches[j] || s1[i] !== s2[j])
|
|
21
|
+
continue;
|
|
22
|
+
s1Matches[i] = true;
|
|
23
|
+
s2Matches[j] = true;
|
|
24
|
+
matches++;
|
|
25
|
+
break;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
if (matches === 0)
|
|
29
|
+
return 0.0;
|
|
30
|
+
let transpositions = 0;
|
|
31
|
+
let k = 0;
|
|
32
|
+
for (let i = 0; i < len1; i++) {
|
|
33
|
+
if (!s1Matches[i])
|
|
34
|
+
continue;
|
|
35
|
+
while (!s2Matches[k])
|
|
36
|
+
k++;
|
|
37
|
+
if (s1[i] !== s2[k])
|
|
38
|
+
transpositions++;
|
|
39
|
+
k++;
|
|
40
|
+
}
|
|
41
|
+
const jaro = (matches / len1 + matches / len2 + (matches - transpositions / 2) / matches) / 3;
|
|
42
|
+
let prefix = 0;
|
|
43
|
+
for (let i = 0; i < Math.min(4, Math.min(len1, len2)); i++) {
|
|
44
|
+
if (s1[i] === s2[i])
|
|
45
|
+
prefix++;
|
|
46
|
+
else
|
|
47
|
+
break;
|
|
48
|
+
}
|
|
49
|
+
return jaro + prefix * 0.1 * (1 - jaro);
|
|
50
|
+
}
|
|
51
|
+
function scoreUniqueness(rows, schema) {
|
|
52
|
+
const issues = [];
|
|
53
|
+
const flaggedRows = new Set();
|
|
54
|
+
let exactDuplicates = 0;
|
|
55
|
+
const uniqueKeyFields = Object.values(schema.fields)
|
|
56
|
+
.filter((f) => f.uniqueKey)
|
|
57
|
+
.map((f) => f.name);
|
|
58
|
+
// Exact duplicates
|
|
59
|
+
for (const col of uniqueKeyFields) {
|
|
60
|
+
const seen = new Map();
|
|
61
|
+
rows.forEach((row, rowIndex) => {
|
|
62
|
+
const val = row[col];
|
|
63
|
+
if (!val || val.trim() === "")
|
|
64
|
+
return;
|
|
65
|
+
const key = val.trim();
|
|
66
|
+
if (seen.has(key)) {
|
|
67
|
+
exactDuplicates++;
|
|
68
|
+
flaggedRows.add(rowIndex);
|
|
69
|
+
issues.push({
|
|
70
|
+
rowIndex,
|
|
71
|
+
field: col,
|
|
72
|
+
dimension: "uniqueness",
|
|
73
|
+
reason: "exact_duplicate_key",
|
|
74
|
+
duplicateOf: seen.get(key),
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
else {
|
|
78
|
+
seen.set(key, rowIndex);
|
|
79
|
+
}
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
// Soft fuzzy candidate pairs
|
|
83
|
+
const fuzzyFields = Object.values(schema.fields)
|
|
84
|
+
.filter((f) => f.fuzzyKey)
|
|
85
|
+
.map((f) => f.name);
|
|
86
|
+
for (const col of fuzzyFields) {
|
|
87
|
+
const blocks = new Map();
|
|
88
|
+
rows.forEach((row, rowIndex) => {
|
|
89
|
+
const val = row[col];
|
|
90
|
+
if (!val || val.trim() === "")
|
|
91
|
+
return;
|
|
92
|
+
const clean = val.trim().toLowerCase();
|
|
93
|
+
const bucket = clean[0] ?? "_";
|
|
94
|
+
if (!blocks.has(bucket))
|
|
95
|
+
blocks.set(bucket, []);
|
|
96
|
+
blocks.get(bucket).push({ idx: rowIndex, val: clean });
|
|
97
|
+
});
|
|
98
|
+
for (const entries of blocks.values()) {
|
|
99
|
+
for (let i = 0; i < entries.length; i++) {
|
|
100
|
+
for (let j = i + 1; j < entries.length; j++) {
|
|
101
|
+
const sim = jaroWinkler(entries[i].val, entries[j].val);
|
|
102
|
+
if (sim >= 0.85) {
|
|
103
|
+
flaggedRows.add(entries[j].idx);
|
|
104
|
+
issues.push({
|
|
105
|
+
rowIndex: entries[j].idx,
|
|
106
|
+
field: col,
|
|
107
|
+
dimension: "uniqueness",
|
|
108
|
+
reason: "fuzzy_duplicate_candidate",
|
|
109
|
+
duplicateOf: entries[i].idx,
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
const score = rows.length === 0 ? 100 : Math.max(0, (1 - flaggedRows.size / rows.length) * 100);
|
|
117
|
+
return {
|
|
118
|
+
score: Number(score.toFixed(2)),
|
|
119
|
+
exactDuplicateCount: exactDuplicates,
|
|
120
|
+
issues,
|
|
121
|
+
};
|
|
122
|
+
}
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.scoreValidity = scoreValidity;
|
|
4
|
+
const types_js_1 = require("../types.js");
|
|
5
|
+
const PHONE_REGEX = /^\+?[\d\s\-().]{7,15}$/;
|
|
6
|
+
const EMAIL_REGEX = /^[^@\s]+@[^@\s]+\.[^@\s]+$/;
|
|
7
|
+
function scoreValidity(rows, schema) {
|
|
8
|
+
const issues = [];
|
|
9
|
+
let checkedSlots = 0;
|
|
10
|
+
let invalidSlots = 0;
|
|
11
|
+
rows.forEach((row, rowIndex) => {
|
|
12
|
+
for (const [colName, fdef] of Object.entries(schema.fields)) {
|
|
13
|
+
const val = row[colName];
|
|
14
|
+
if (val === undefined || val === null || val.trim() === "") {
|
|
15
|
+
continue;
|
|
16
|
+
}
|
|
17
|
+
checkedSlots++;
|
|
18
|
+
const cleanVal = val.trim();
|
|
19
|
+
let isValid = true;
|
|
20
|
+
let reason = "";
|
|
21
|
+
switch (fdef.type) {
|
|
22
|
+
case types_js_1.FieldType.INTEGER: {
|
|
23
|
+
const num = Number(cleanVal);
|
|
24
|
+
if (isNaN(num) || !Number.isInteger(num)) {
|
|
25
|
+
isValid = false;
|
|
26
|
+
reason = "invalid_integer_format";
|
|
27
|
+
}
|
|
28
|
+
else {
|
|
29
|
+
if (fdef.min !== undefined && num < Number(fdef.min)) {
|
|
30
|
+
isValid = false;
|
|
31
|
+
reason = `integer_below_min_${fdef.min}`;
|
|
32
|
+
}
|
|
33
|
+
if (fdef.max !== undefined && num > Number(fdef.max)) {
|
|
34
|
+
isValid = false;
|
|
35
|
+
reason = `integer_above_max_${fdef.max}`;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
break;
|
|
39
|
+
}
|
|
40
|
+
case types_js_1.FieldType.FLOAT: {
|
|
41
|
+
const num = Number(cleanVal);
|
|
42
|
+
if (isNaN(num)) {
|
|
43
|
+
isValid = false;
|
|
44
|
+
reason = "invalid_float_format";
|
|
45
|
+
}
|
|
46
|
+
else {
|
|
47
|
+
if (fdef.min !== undefined && num < Number(fdef.min)) {
|
|
48
|
+
isValid = false;
|
|
49
|
+
reason = `float_below_min_${fdef.min}`;
|
|
50
|
+
}
|
|
51
|
+
if (fdef.max !== undefined && num > Number(fdef.max)) {
|
|
52
|
+
isValid = false;
|
|
53
|
+
reason = `float_above_max_${fdef.max}`;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
break;
|
|
57
|
+
}
|
|
58
|
+
case types_js_1.FieldType.LATITUDE: {
|
|
59
|
+
const num = Number(cleanVal);
|
|
60
|
+
const minLat = fdef.min !== undefined ? Number(fdef.min) : -90.0;
|
|
61
|
+
const maxLat = fdef.max !== undefined ? Number(fdef.max) : 90.0;
|
|
62
|
+
if (isNaN(num) || num < minLat || num > maxLat) {
|
|
63
|
+
isValid = false;
|
|
64
|
+
reason = `latitude_out_of_bounds_[${minLat},${maxLat}]`;
|
|
65
|
+
}
|
|
66
|
+
break;
|
|
67
|
+
}
|
|
68
|
+
case types_js_1.FieldType.LONGITUDE: {
|
|
69
|
+
const num = Number(cleanVal);
|
|
70
|
+
const minLon = fdef.min !== undefined ? Number(fdef.min) : -180.0;
|
|
71
|
+
const maxLon = fdef.max !== undefined ? Number(fdef.max) : 180.0;
|
|
72
|
+
if (isNaN(num) || num < minLon || num > maxLon) {
|
|
73
|
+
isValid = false;
|
|
74
|
+
reason = `longitude_out_of_bounds_[${minLon},${maxLon}]`;
|
|
75
|
+
}
|
|
76
|
+
break;
|
|
77
|
+
}
|
|
78
|
+
case types_js_1.FieldType.PHONE: {
|
|
79
|
+
if (!PHONE_REGEX.test(cleanVal)) {
|
|
80
|
+
isValid = false;
|
|
81
|
+
reason = "invalid_phone_format";
|
|
82
|
+
}
|
|
83
|
+
break;
|
|
84
|
+
}
|
|
85
|
+
case types_js_1.FieldType.EMAIL: {
|
|
86
|
+
if (!EMAIL_REGEX.test(cleanVal)) {
|
|
87
|
+
isValid = false;
|
|
88
|
+
reason = "invalid_email_format";
|
|
89
|
+
}
|
|
90
|
+
break;
|
|
91
|
+
}
|
|
92
|
+
case types_js_1.FieldType.ENUM: {
|
|
93
|
+
if (fdef.allowedValues && !fdef.allowedValues.includes(cleanVal)) {
|
|
94
|
+
isValid = false;
|
|
95
|
+
reason = `value_not_in_allowed_enum`;
|
|
96
|
+
}
|
|
97
|
+
break;
|
|
98
|
+
}
|
|
99
|
+
case types_js_1.FieldType.DATE: {
|
|
100
|
+
const parsed = Date.parse(cleanVal);
|
|
101
|
+
if (isNaN(parsed)) {
|
|
102
|
+
isValid = false;
|
|
103
|
+
reason = "invalid_date_format";
|
|
104
|
+
}
|
|
105
|
+
break;
|
|
106
|
+
}
|
|
107
|
+
case types_js_1.FieldType.BOOLEAN: {
|
|
108
|
+
const boolLower = cleanVal.toLowerCase();
|
|
109
|
+
if (!["true", "false", "1", "0", "yes", "no"].includes(boolLower)) {
|
|
110
|
+
isValid = false;
|
|
111
|
+
reason = "invalid_boolean_format";
|
|
112
|
+
}
|
|
113
|
+
break;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
if (!isValid) {
|
|
117
|
+
invalidSlots++;
|
|
118
|
+
issues.push({
|
|
119
|
+
rowIndex,
|
|
120
|
+
field: colName,
|
|
121
|
+
dimension: "validity",
|
|
122
|
+
reason,
|
|
123
|
+
value: cleanVal,
|
|
124
|
+
});
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
});
|
|
128
|
+
const score = checkedSlots === 0 ? 100 : Math.max(0, Math.min(100, (1 - invalidSlots / checkedSlots) * 100));
|
|
129
|
+
return { score: Number(score.toFixed(2)), issues };
|
|
130
|
+
}
|
package/dist/engine.d.ts
ADDED
package/dist/engine.js
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.DQIEngine = void 0;
|
|
7
|
+
const schema_js_1 = require("./schema.js");
|
|
8
|
+
const report_js_1 = require("./report.js");
|
|
9
|
+
const completeness_js_1 = require("./dimensions/completeness.js");
|
|
10
|
+
const validity_js_1 = require("./dimensions/validity.js");
|
|
11
|
+
const uniqueness_js_1 = require("./dimensions/uniqueness.js");
|
|
12
|
+
const outliers_js_1 = require("./dimensions/outliers.js");
|
|
13
|
+
const papaparse_1 = __importDefault(require("papaparse"));
|
|
14
|
+
class DQIEngine {
|
|
15
|
+
schema;
|
|
16
|
+
constructor(schema = null) {
|
|
17
|
+
this.schema = schema;
|
|
18
|
+
}
|
|
19
|
+
evaluate(csvInput) {
|
|
20
|
+
let rows;
|
|
21
|
+
if (typeof csvInput === "string") {
|
|
22
|
+
const parsed = papaparse_1.default.parse(csvInput, {
|
|
23
|
+
header: true,
|
|
24
|
+
skipEmptyLines: true,
|
|
25
|
+
});
|
|
26
|
+
rows = parsed.data;
|
|
27
|
+
if (!this.schema) {
|
|
28
|
+
this.schema = schema_js_1.Schema.infer(csvInput);
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
else {
|
|
32
|
+
rows = csvInput;
|
|
33
|
+
if (!this.schema) {
|
|
34
|
+
throw new Error("Schema must be provided when evaluating raw row arrays.");
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
const completeness = (0, completeness_js_1.scoreCompleteness)(rows, this.schema);
|
|
38
|
+
const validity = (0, validity_js_1.scoreValidity)(rows, this.schema);
|
|
39
|
+
const uniqueness = (0, uniqueness_js_1.scoreUniqueness)(rows, this.schema);
|
|
40
|
+
const outliers = (0, outliers_js_1.scoreOutliers)(rows, this.schema);
|
|
41
|
+
const w = this.schema.weights;
|
|
42
|
+
const overall = completeness.score * w.completeness +
|
|
43
|
+
validity.score * w.validity +
|
|
44
|
+
uniqueness.score * w.uniqueness +
|
|
45
|
+
outliers.score * w.outliers;
|
|
46
|
+
const roundOverall = Number(overall.toFixed(2));
|
|
47
|
+
let grade = "F";
|
|
48
|
+
if (roundOverall >= 90)
|
|
49
|
+
grade = "A";
|
|
50
|
+
else if (roundOverall >= 75)
|
|
51
|
+
grade = "B";
|
|
52
|
+
else if (roundOverall >= 60)
|
|
53
|
+
grade = "C";
|
|
54
|
+
else if (roundOverall >= 45)
|
|
55
|
+
grade = "D";
|
|
56
|
+
const allIssues = [
|
|
57
|
+
...completeness.issues,
|
|
58
|
+
...validity.issues,
|
|
59
|
+
...uniqueness.issues,
|
|
60
|
+
...outliers.issues,
|
|
61
|
+
];
|
|
62
|
+
const reportData = {
|
|
63
|
+
overallScore: roundOverall,
|
|
64
|
+
grade,
|
|
65
|
+
totalRecords: rows.length,
|
|
66
|
+
dimensions: {
|
|
67
|
+
completeness,
|
|
68
|
+
validity,
|
|
69
|
+
uniqueness,
|
|
70
|
+
outliers,
|
|
71
|
+
},
|
|
72
|
+
weights: w,
|
|
73
|
+
allIssues,
|
|
74
|
+
generatedAt: new Date().toISOString(),
|
|
75
|
+
};
|
|
76
|
+
return new report_js_1.DQIReport(reportData);
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
exports.DQIEngine = DQIEngine;
|
package/dist/index.d.ts
ADDED
package/dist/index.js
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
+
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
+
};
|
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
exports.jaroWinkler = exports.DQIReport = exports.Schema = exports.DQIEngine = void 0;
|
|
18
|
+
var engine_js_1 = require("./engine.js");
|
|
19
|
+
Object.defineProperty(exports, "DQIEngine", { enumerable: true, get: function () { return engine_js_1.DQIEngine; } });
|
|
20
|
+
var schema_js_1 = require("./schema.js");
|
|
21
|
+
Object.defineProperty(exports, "Schema", { enumerable: true, get: function () { return schema_js_1.Schema; } });
|
|
22
|
+
var report_js_1 = require("./report.js");
|
|
23
|
+
Object.defineProperty(exports, "DQIReport", { enumerable: true, get: function () { return report_js_1.DQIReport; } });
|
|
24
|
+
__exportStar(require("./types.js"), exports);
|
|
25
|
+
var uniqueness_js_1 = require("./dimensions/uniqueness.js");
|
|
26
|
+
Object.defineProperty(exports, "jaroWinkler", { enumerable: true, get: function () { return uniqueness_js_1.jaroWinkler; } });
|
package/dist/report.d.ts
ADDED