@liuhange/dsh-data-quality-scoring 2.0.1 → 2.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +53 -39
- package/lib/index.js +0 -662
- package/lib/invariant.js +0 -5
- package/lib/types/accuracyScorer.d.ts +0 -5
- package/lib/types/completenessScorer.d.ts +0 -6
- package/lib/types/consistencyScorer.d.ts +0 -6
- package/lib/types/defaultScoringRules.d.ts +0 -3
- package/lib/types/index.d.ts +0 -5
- package/lib/types/invariant.d.ts +0 -3
- package/lib/types/issueCollector.d.ts +0 -5
- package/lib/types/qualityReportGenerator.d.ts +0 -6
- package/lib/types/suggestionGenerator.d.ts +0 -6
- package/lib/types/timelinessScorer.d.ts +0 -5
- package/lib/types/types.d.ts +0 -54
- package/lib/types/weightedScoreCalculator.d.ts +0 -10
package/package.json
CHANGED
|
@@ -1,39 +1,53 @@
|
|
|
1
|
-
{
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
1
|
+
{
|
|
2
|
+
"name": "@liuhange/dsh-data-quality-scoring",
|
|
3
|
+
"description": "Data quality scoring plugin: multi-dimensional scoring (completeness/accuracy/consistency/timeliness) with weighted total and improvement suggestions Also known as @deepseek-ai/dsh-data-quality-scoring.",
|
|
4
|
+
"version": "2.0.2",
|
|
5
|
+
"publishConfig": {
|
|
6
|
+
"access": "public"
|
|
7
|
+
},
|
|
8
|
+
"repository": {
|
|
9
|
+
"type": "git",
|
|
10
|
+
"url": "git+https://github.com/liuhange789/data-asset-inspector.git",
|
|
11
|
+
"directory": "packages/data-asset/data-quality-scoring"
|
|
12
|
+
},
|
|
13
|
+
"type": "module",
|
|
14
|
+
"main": "lib/index.js",
|
|
15
|
+
"types": "lib/types/index.d.ts",
|
|
16
|
+
"exports": {
|
|
17
|
+
".": {
|
|
18
|
+
"types": "./lib/types/index.d.ts",
|
|
19
|
+
"default": "./lib/index.js"
|
|
20
|
+
},
|
|
21
|
+
"./invariant": {
|
|
22
|
+
"types": "./lib/types/invariant.d.ts",
|
|
23
|
+
"default": "./lib/invariant.js"
|
|
24
|
+
},
|
|
25
|
+
"./src/*": "./src/*",
|
|
26
|
+
"./package.json": "./package.json"
|
|
27
|
+
},
|
|
28
|
+
"files": [
|
|
29
|
+
"lib/index.js",
|
|
30
|
+
"lib/invariant.js",
|
|
31
|
+
"lib/types/**/*.d.ts"
|
|
32
|
+
],
|
|
33
|
+
"license": "MIT",
|
|
34
|
+
"scripts": {
|
|
35
|
+
"typecheck": "tsc --noEmit",
|
|
36
|
+
"test": "vitest run",
|
|
37
|
+
"build": "tsc"
|
|
38
|
+
},
|
|
39
|
+
"peerDependencies": {
|
|
40
|
+
"@deepseek-ai/cordis": "^4.0.0",
|
|
41
|
+
"@deepseek-ai/dsh-tools": "0.0.1-rc.1",
|
|
42
|
+
"@liuhange/dsh-data-asset-shared": "^2.0.0"
|
|
43
|
+
},
|
|
44
|
+
"keywords": [
|
|
45
|
+
"deepseek-harness",
|
|
46
|
+
"dsh-plugin",
|
|
47
|
+
"data-asset",
|
|
48
|
+
"data-asset-inspector",
|
|
49
|
+
"shujintong",
|
|
50
|
+
"数据资产"
|
|
51
|
+
],
|
|
52
|
+
"homepage": "https://github.com/liuhange789/data-asset-inspector#readme"
|
|
53
|
+
}
|
package/lib/index.js
DELETED
|
@@ -1,662 +0,0 @@
|
|
|
1
|
-
import { defineTool } from "@deepseek-ai/dsh-tools";
|
|
2
|
-
import * as fs from "fs";
|
|
3
|
-
import * as path from "path";
|
|
4
|
-
import { AuditLogger, BusinessRulesLoader, FileFormatAdapter, PathValidator } from "@liuhange/dsh-data-asset-shared";
|
|
5
|
-
//#region lib/types/defaultScoringRules.js
|
|
6
|
-
const defaultScoringRules = {
|
|
7
|
-
weights: {
|
|
8
|
-
completeness: .25,
|
|
9
|
-
accuracy: .25,
|
|
10
|
-
consistency: .25,
|
|
11
|
-
timeliness: .25
|
|
12
|
-
},
|
|
13
|
-
thresholds: {
|
|
14
|
-
completeness: 60,
|
|
15
|
-
accuracy: 60,
|
|
16
|
-
consistency: 60,
|
|
17
|
-
timeliness: 60
|
|
18
|
-
},
|
|
19
|
-
completeness: { missingMarkers: [
|
|
20
|
-
"null",
|
|
21
|
-
"undefined",
|
|
22
|
-
"N/A",
|
|
23
|
-
"",
|
|
24
|
-
"NULL",
|
|
25
|
-
"NaN"
|
|
26
|
-
] },
|
|
27
|
-
accuracy: {
|
|
28
|
-
formatRules: [{
|
|
29
|
-
fieldName: "phone",
|
|
30
|
-
pattern: "1[3-9]\\d{9}",
|
|
31
|
-
weight: .5
|
|
32
|
-
}, {
|
|
33
|
-
fieldName: "email",
|
|
34
|
-
pattern: "[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}",
|
|
35
|
-
weight: .5
|
|
36
|
-
}],
|
|
37
|
-
domainRules: [{
|
|
38
|
-
fieldName: "age",
|
|
39
|
-
min: 0,
|
|
40
|
-
max: 150,
|
|
41
|
-
weight: .5
|
|
42
|
-
}, {
|
|
43
|
-
fieldName: "score",
|
|
44
|
-
min: 0,
|
|
45
|
-
max: 100,
|
|
46
|
-
weight: .5
|
|
47
|
-
}]
|
|
48
|
-
},
|
|
49
|
-
consistency: { crossFieldRules: [{
|
|
50
|
-
name: "endDateAfterStartDate",
|
|
51
|
-
fields: ["startDate", "endDate"],
|
|
52
|
-
constraint: "after"
|
|
53
|
-
}] },
|
|
54
|
-
timeliness: {
|
|
55
|
-
timestampField: "updatedAt",
|
|
56
|
-
freshnessThresholdHours: 720
|
|
57
|
-
}
|
|
58
|
-
};
|
|
59
|
-
//#endregion
|
|
60
|
-
//#region lib/types/completenessScorer.js
|
|
61
|
-
var CompletenessScorer = class {
|
|
62
|
-
score(context) {
|
|
63
|
-
const missingMarkers = context.config.completeness.missingMarkers;
|
|
64
|
-
const issues = [];
|
|
65
|
-
if (context.records.length === 0 || context.fieldNames.length === 0) return {
|
|
66
|
-
score: 0,
|
|
67
|
-
issues
|
|
68
|
-
};
|
|
69
|
-
let totalValues = 0;
|
|
70
|
-
let missingValues = 0;
|
|
71
|
-
for (let i = 0; i < context.records.length; i++) {
|
|
72
|
-
const record = context.records[i];
|
|
73
|
-
for (const fieldName of context.fieldNames) {
|
|
74
|
-
totalValues++;
|
|
75
|
-
const value = record[fieldName];
|
|
76
|
-
if (this.isMissing(value, missingMarkers)) {
|
|
77
|
-
missingValues++;
|
|
78
|
-
issues.push({
|
|
79
|
-
dimension: "completeness",
|
|
80
|
-
description: `字段 ${fieldName} 缺失`,
|
|
81
|
-
location: `行 ${i + 1}, 列 ${fieldName}`,
|
|
82
|
-
severity: "medium"
|
|
83
|
-
});
|
|
84
|
-
}
|
|
85
|
-
}
|
|
86
|
-
}
|
|
87
|
-
if (totalValues === 0) return {
|
|
88
|
-
score: 0,
|
|
89
|
-
issues
|
|
90
|
-
};
|
|
91
|
-
return {
|
|
92
|
-
score: Math.floor((totalValues - missingValues) / totalValues * 100),
|
|
93
|
-
issues
|
|
94
|
-
};
|
|
95
|
-
}
|
|
96
|
-
isMissing(value, markers) {
|
|
97
|
-
if (value === null || value === void 0) return true;
|
|
98
|
-
const strValue = String(value).trim();
|
|
99
|
-
return markers.includes(strValue);
|
|
100
|
-
}
|
|
101
|
-
};
|
|
102
|
-
//#endregion
|
|
103
|
-
//#region lib/types/accuracyScorer.js
|
|
104
|
-
var AccuracyScorer = class {
|
|
105
|
-
score(context) {
|
|
106
|
-
const config = context.config;
|
|
107
|
-
const issues = [];
|
|
108
|
-
if (context.records.length === 0) return {
|
|
109
|
-
score: 0,
|
|
110
|
-
issues
|
|
111
|
-
};
|
|
112
|
-
const formatRules = config.accuracy.formatRules;
|
|
113
|
-
const domainRules = config.accuracy.domainRules;
|
|
114
|
-
let formatCompliance = 1;
|
|
115
|
-
let formatWeightSum = 0;
|
|
116
|
-
let formatWeightedCompliance = 0;
|
|
117
|
-
for (const rule of formatRules) {
|
|
118
|
-
if (!context.fieldNames.includes(rule.fieldName)) {
|
|
119
|
-
issues.push({
|
|
120
|
-
dimension: "accuracy",
|
|
121
|
-
description: `格式规则引用字段 ${rule.fieldName} 不存在,已跳过`,
|
|
122
|
-
location: `格式规则 ${rule.fieldName}`,
|
|
123
|
-
severity: "low"
|
|
124
|
-
});
|
|
125
|
-
continue;
|
|
126
|
-
}
|
|
127
|
-
let compliant = 0;
|
|
128
|
-
let total = 0;
|
|
129
|
-
for (let i = 0; i < context.records.length; i++) {
|
|
130
|
-
const value = context.records[i][rule.fieldName];
|
|
131
|
-
if (value === null || value === void 0 || String(value).trim() === "") continue;
|
|
132
|
-
total++;
|
|
133
|
-
try {
|
|
134
|
-
if (new RegExp(rule.pattern).test(String(value))) compliant++;
|
|
135
|
-
else issues.push({
|
|
136
|
-
dimension: "accuracy",
|
|
137
|
-
description: `字段 ${rule.fieldName} 值 "${value}" 不符合格式规则`,
|
|
138
|
-
location: `行 ${i + 1}, 列 ${rule.fieldName}`,
|
|
139
|
-
severity: "medium"
|
|
140
|
-
});
|
|
141
|
-
} catch {
|
|
142
|
-
issues.push({
|
|
143
|
-
dimension: "accuracy",
|
|
144
|
-
description: `格式规则 ${rule.fieldName} 正则编译失败`,
|
|
145
|
-
location: `格式规则 ${rule.fieldName}`,
|
|
146
|
-
severity: "high"
|
|
147
|
-
});
|
|
148
|
-
}
|
|
149
|
-
}
|
|
150
|
-
if (total > 0) {
|
|
151
|
-
formatWeightedCompliance += compliant / total * rule.weight;
|
|
152
|
-
formatWeightSum += rule.weight;
|
|
153
|
-
}
|
|
154
|
-
}
|
|
155
|
-
if (formatWeightSum > 0) formatCompliance = formatWeightedCompliance / formatWeightSum;
|
|
156
|
-
let domainCompliance = 1;
|
|
157
|
-
let domainWeightSum = 0;
|
|
158
|
-
let domainWeightedCompliance = 0;
|
|
159
|
-
for (const rule of domainRules) {
|
|
160
|
-
if (!context.fieldNames.includes(rule.fieldName)) {
|
|
161
|
-
issues.push({
|
|
162
|
-
dimension: "accuracy",
|
|
163
|
-
description: `值域规则引用字段 ${rule.fieldName} 不存在,已跳过`,
|
|
164
|
-
location: `值域规则 ${rule.fieldName}`,
|
|
165
|
-
severity: "low"
|
|
166
|
-
});
|
|
167
|
-
continue;
|
|
168
|
-
}
|
|
169
|
-
let compliant = 0;
|
|
170
|
-
let total = 0;
|
|
171
|
-
for (let i = 0; i < context.records.length; i++) {
|
|
172
|
-
const value = context.records[i][rule.fieldName];
|
|
173
|
-
if (value === null || value === void 0 || String(value).trim() === "") continue;
|
|
174
|
-
total++;
|
|
175
|
-
const numValue = Number(value);
|
|
176
|
-
if (isNaN(numValue)) {
|
|
177
|
-
issues.push({
|
|
178
|
-
dimension: "accuracy",
|
|
179
|
-
description: `字段 ${rule.fieldName} 值 "${value}" 不是数字,无法校验值域`,
|
|
180
|
-
location: `行 ${i + 1}, 列 ${rule.fieldName}`,
|
|
181
|
-
severity: "medium"
|
|
182
|
-
});
|
|
183
|
-
continue;
|
|
184
|
-
}
|
|
185
|
-
if (rule.min !== void 0 && rule.max !== void 0) if (numValue >= rule.min && numValue <= rule.max) compliant++;
|
|
186
|
-
else issues.push({
|
|
187
|
-
dimension: "accuracy",
|
|
188
|
-
description: `字段 ${rule.fieldName} 值 ${numValue} 超出值域 [${rule.min}, ${rule.max}]`,
|
|
189
|
-
location: `行 ${i + 1}, 列 ${rule.fieldName}`,
|
|
190
|
-
severity: "medium"
|
|
191
|
-
});
|
|
192
|
-
else if (rule.allowedValues !== void 0) if (rule.allowedValues.includes(value)) compliant++;
|
|
193
|
-
else issues.push({
|
|
194
|
-
dimension: "accuracy",
|
|
195
|
-
description: `字段 ${rule.fieldName} 值 "${value}" 不在允许值列表中`,
|
|
196
|
-
location: `行 ${i + 1}, 列 ${rule.fieldName}`,
|
|
197
|
-
severity: "medium"
|
|
198
|
-
});
|
|
199
|
-
}
|
|
200
|
-
if (total > 0) {
|
|
201
|
-
domainWeightedCompliance += compliant / total * rule.weight;
|
|
202
|
-
domainWeightSum += rule.weight;
|
|
203
|
-
}
|
|
204
|
-
}
|
|
205
|
-
if (domainWeightSum > 0) domainCompliance = domainWeightedCompliance / domainWeightSum;
|
|
206
|
-
return {
|
|
207
|
-
score: Math.floor((formatCompliance * .5 + domainCompliance * .5) * 100),
|
|
208
|
-
issues
|
|
209
|
-
};
|
|
210
|
-
}
|
|
211
|
-
};
|
|
212
|
-
//#endregion
|
|
213
|
-
//#region lib/types/consistencyScorer.js
|
|
214
|
-
var ConsistencyScorer = class {
|
|
215
|
-
score(context) {
|
|
216
|
-
const config = context.config;
|
|
217
|
-
const issues = [];
|
|
218
|
-
if (context.records.length === 0) return {
|
|
219
|
-
score: 0,
|
|
220
|
-
issues
|
|
221
|
-
};
|
|
222
|
-
const rules = config.consistency.crossFieldRules;
|
|
223
|
-
let totalChecks = 0;
|
|
224
|
-
let satisfiedChecks = 0;
|
|
225
|
-
for (const rule of rules) {
|
|
226
|
-
const missingFields = rule.fields.filter((f) => !context.fieldNames.includes(f));
|
|
227
|
-
if (missingFields.length > 0) {
|
|
228
|
-
issues.push({
|
|
229
|
-
dimension: "consistency",
|
|
230
|
-
description: `约束规则引用字段 ${missingFields.join(", ")} 不存在,已跳过`,
|
|
231
|
-
location: `约束规则 ${rule.name}`,
|
|
232
|
-
severity: "low"
|
|
233
|
-
});
|
|
234
|
-
continue;
|
|
235
|
-
}
|
|
236
|
-
for (let i = 0; i < context.records.length; i++) {
|
|
237
|
-
const record = context.records[i];
|
|
238
|
-
const result = this.checkConstraint(record, rule);
|
|
239
|
-
totalChecks++;
|
|
240
|
-
if (result.satisfied) satisfiedChecks++;
|
|
241
|
-
else issues.push({
|
|
242
|
-
dimension: "consistency",
|
|
243
|
-
description: `约束规则 ${rule.name} 不满足: ${result.reason}`,
|
|
244
|
-
location: `行 ${i + 1}`,
|
|
245
|
-
severity: "medium"
|
|
246
|
-
});
|
|
247
|
-
}
|
|
248
|
-
}
|
|
249
|
-
if (totalChecks === 0) return {
|
|
250
|
-
score: 100,
|
|
251
|
-
issues
|
|
252
|
-
};
|
|
253
|
-
return {
|
|
254
|
-
score: Math.floor(satisfiedChecks / totalChecks * 100),
|
|
255
|
-
issues
|
|
256
|
-
};
|
|
257
|
-
}
|
|
258
|
-
checkConstraint(record, rule) {
|
|
259
|
-
const fields = rule.fields;
|
|
260
|
-
const constraint = rule.constraint;
|
|
261
|
-
if (constraint === "after" && fields.length >= 2) {
|
|
262
|
-
const before = record[fields[0]];
|
|
263
|
-
const after = record[fields[1]];
|
|
264
|
-
if (before === null || before === void 0 || after === null || after === void 0) return {
|
|
265
|
-
satisfied: true,
|
|
266
|
-
reason: "字段为空,跳过约束"
|
|
267
|
-
};
|
|
268
|
-
const beforeDate = new Date(String(before));
|
|
269
|
-
if (new Date(String(after)) >= beforeDate) return {
|
|
270
|
-
satisfied: true,
|
|
271
|
-
reason: ""
|
|
272
|
-
};
|
|
273
|
-
return {
|
|
274
|
-
satisfied: false,
|
|
275
|
-
reason: `${fields[1]}(${after}) 早于 ${fields[0]}(${before})`
|
|
276
|
-
};
|
|
277
|
-
}
|
|
278
|
-
if (constraint === "before" && fields.length >= 2) {
|
|
279
|
-
const before = record[fields[0]];
|
|
280
|
-
const after = record[fields[1]];
|
|
281
|
-
if (before === null || before === void 0 || after === null || after === void 0) return {
|
|
282
|
-
satisfied: true,
|
|
283
|
-
reason: "字段为空,跳过约束"
|
|
284
|
-
};
|
|
285
|
-
if (new Date(String(before)) < new Date(String(after))) return {
|
|
286
|
-
satisfied: true,
|
|
287
|
-
reason: ""
|
|
288
|
-
};
|
|
289
|
-
return {
|
|
290
|
-
satisfied: false,
|
|
291
|
-
reason: `${fields[0]}(${before}) 不早于 ${fields[1]}(${after})`
|
|
292
|
-
};
|
|
293
|
-
}
|
|
294
|
-
if (constraint === "equal" && fields.length >= 2) {
|
|
295
|
-
const v1 = record[fields[0]];
|
|
296
|
-
const v2 = record[fields[1]];
|
|
297
|
-
if (v1 === v2) return {
|
|
298
|
-
satisfied: true,
|
|
299
|
-
reason: ""
|
|
300
|
-
};
|
|
301
|
-
return {
|
|
302
|
-
satisfied: false,
|
|
303
|
-
reason: `${fields[0]}(${v1}) 不等于 ${fields[1]}(${v2})`
|
|
304
|
-
};
|
|
305
|
-
}
|
|
306
|
-
if (constraint === "notEqual" && fields.length >= 2) {
|
|
307
|
-
const v1 = record[fields[0]];
|
|
308
|
-
const v2 = record[fields[1]];
|
|
309
|
-
if (v1 !== v2) return {
|
|
310
|
-
satisfied: true,
|
|
311
|
-
reason: ""
|
|
312
|
-
};
|
|
313
|
-
return {
|
|
314
|
-
satisfied: false,
|
|
315
|
-
reason: `${fields[0]}(${v1}) 等于 ${fields[1]}(${v2})`
|
|
316
|
-
};
|
|
317
|
-
}
|
|
318
|
-
if (constraint === "greaterThan" && fields.length >= 1) {
|
|
319
|
-
const v = record[fields[0]];
|
|
320
|
-
const threshold = rule.value;
|
|
321
|
-
if (v === null || v === void 0 || threshold === void 0) return {
|
|
322
|
-
satisfied: true,
|
|
323
|
-
reason: "字段为空,跳过约束"
|
|
324
|
-
};
|
|
325
|
-
if (Number(v) > Number(threshold)) return {
|
|
326
|
-
satisfied: true,
|
|
327
|
-
reason: ""
|
|
328
|
-
};
|
|
329
|
-
return {
|
|
330
|
-
satisfied: false,
|
|
331
|
-
reason: `${fields[0]}(${v}) 不大于 ${threshold}`
|
|
332
|
-
};
|
|
333
|
-
}
|
|
334
|
-
if (constraint === "lessThan" && fields.length >= 1) {
|
|
335
|
-
const v = record[fields[0]];
|
|
336
|
-
const threshold = rule.value;
|
|
337
|
-
if (v === null || v === void 0 || threshold === void 0) return {
|
|
338
|
-
satisfied: true,
|
|
339
|
-
reason: "字段为空,跳过约束"
|
|
340
|
-
};
|
|
341
|
-
if (Number(v) < Number(threshold)) return {
|
|
342
|
-
satisfied: true,
|
|
343
|
-
reason: ""
|
|
344
|
-
};
|
|
345
|
-
return {
|
|
346
|
-
satisfied: false,
|
|
347
|
-
reason: `${fields[0]}(${v}) 不小于 ${threshold}`
|
|
348
|
-
};
|
|
349
|
-
}
|
|
350
|
-
return {
|
|
351
|
-
satisfied: true,
|
|
352
|
-
reason: `未知约束类型: ${constraint}`
|
|
353
|
-
};
|
|
354
|
-
}
|
|
355
|
-
};
|
|
356
|
-
//#endregion
|
|
357
|
-
//#region lib/types/timelinessScorer.js
|
|
358
|
-
var TimelinessScorer = class {
|
|
359
|
-
score(context) {
|
|
360
|
-
const config = context.config;
|
|
361
|
-
const issues = [];
|
|
362
|
-
if (context.records.length === 0) return {
|
|
363
|
-
score: 0,
|
|
364
|
-
issues
|
|
365
|
-
};
|
|
366
|
-
const timestampField = config.timeliness.timestampField;
|
|
367
|
-
const thresholdHours = config.timeliness.freshnessThresholdHours;
|
|
368
|
-
if (!context.fieldNames.includes(timestampField)) {
|
|
369
|
-
issues.push({
|
|
370
|
-
dimension: "timeliness",
|
|
371
|
-
description: `时间戳字段 ${timestampField} 不存在`,
|
|
372
|
-
location: `时间戳字段 ${timestampField}`,
|
|
373
|
-
severity: "high"
|
|
374
|
-
});
|
|
375
|
-
return {
|
|
376
|
-
score: 0,
|
|
377
|
-
issues
|
|
378
|
-
};
|
|
379
|
-
}
|
|
380
|
-
const now = Date.now();
|
|
381
|
-
const thresholdMs = thresholdHours * 60 * 60 * 1e3;
|
|
382
|
-
let staleCount = 0;
|
|
383
|
-
let validCount = 0;
|
|
384
|
-
for (let i = 0; i < context.records.length; i++) {
|
|
385
|
-
const timestampValue = context.records[i][timestampField];
|
|
386
|
-
if (timestampValue === null || timestampValue === void 0 || String(timestampValue).trim() === "") continue;
|
|
387
|
-
const recordTime = new Date(String(timestampValue)).getTime();
|
|
388
|
-
if (isNaN(recordTime)) {
|
|
389
|
-
issues.push({
|
|
390
|
-
dimension: "timeliness",
|
|
391
|
-
description: `时间戳值 "${timestampValue}" 无法解析为日期`,
|
|
392
|
-
location: `行 ${i + 1}, 列 ${timestampField}`,
|
|
393
|
-
severity: "medium"
|
|
394
|
-
});
|
|
395
|
-
continue;
|
|
396
|
-
}
|
|
397
|
-
validCount++;
|
|
398
|
-
const age = now - recordTime;
|
|
399
|
-
if (age > thresholdMs) {
|
|
400
|
-
staleCount++;
|
|
401
|
-
issues.push({
|
|
402
|
-
dimension: "timeliness",
|
|
403
|
-
description: `数据已过期,距今 ${Math.floor(age / (3600 * 1e3))} 小时`,
|
|
404
|
-
location: `行 ${i + 1}, 列 ${timestampField}`,
|
|
405
|
-
severity: "medium"
|
|
406
|
-
});
|
|
407
|
-
}
|
|
408
|
-
}
|
|
409
|
-
if (validCount === 0) return {
|
|
410
|
-
score: 0,
|
|
411
|
-
issues
|
|
412
|
-
};
|
|
413
|
-
const staleRatio = staleCount / validCount;
|
|
414
|
-
return {
|
|
415
|
-
score: Math.floor((1 - staleRatio) * 100),
|
|
416
|
-
issues
|
|
417
|
-
};
|
|
418
|
-
}
|
|
419
|
-
};
|
|
420
|
-
//#endregion
|
|
421
|
-
//#region lib/types/weightedScoreCalculator.js
|
|
422
|
-
var WeightedScoreCalculator = class {
|
|
423
|
-
calculate(dimensions, weights) {
|
|
424
|
-
const weightSum = weights.completeness + weights.accuracy + weights.consistency + weights.timeliness;
|
|
425
|
-
let normalizedWeights = { ...weights };
|
|
426
|
-
let weightsNormalized = false;
|
|
427
|
-
if (weightSum !== 1) {
|
|
428
|
-
normalizedWeights = {
|
|
429
|
-
completeness: weights.completeness / weightSum,
|
|
430
|
-
accuracy: weights.accuracy / weightSum,
|
|
431
|
-
consistency: weights.consistency / weightSum,
|
|
432
|
-
timeliness: weights.timeliness / weightSum
|
|
433
|
-
};
|
|
434
|
-
weightsNormalized = true;
|
|
435
|
-
}
|
|
436
|
-
return {
|
|
437
|
-
totalScore: Math.floor(dimensions.completeness.score * normalizedWeights.completeness + dimensions.accuracy.score * normalizedWeights.accuracy + dimensions.consistency.score * normalizedWeights.consistency + dimensions.timeliness.score * normalizedWeights.timeliness),
|
|
438
|
-
weightsNormalized,
|
|
439
|
-
normalizedWeights
|
|
440
|
-
};
|
|
441
|
-
}
|
|
442
|
-
};
|
|
443
|
-
//#endregion
|
|
444
|
-
//#region lib/types/issueCollector.js
|
|
445
|
-
var IssueCollector = class {
|
|
446
|
-
collect(dimensionResults) {
|
|
447
|
-
const issues = [];
|
|
448
|
-
for (const result of dimensionResults) issues.push(...result.issues);
|
|
449
|
-
return issues;
|
|
450
|
-
}
|
|
451
|
-
};
|
|
452
|
-
//#endregion
|
|
453
|
-
//#region lib/types/suggestionGenerator.js
|
|
454
|
-
var SuggestionGenerator = class {
|
|
455
|
-
generate(dimensions, thresholds) {
|
|
456
|
-
const suggestions = [];
|
|
457
|
-
if (dimensions.completeness.score < thresholds.completeness) suggestions.push(`完整性得分 ${dimensions.completeness.score} 低于阈值 ${thresholds.completeness},建议补充缺失字段数据或使用插值/众数填充`);
|
|
458
|
-
if (dimensions.accuracy.score < thresholds.accuracy) suggestions.push(`准确性得分 ${dimensions.accuracy.score} 低于阈值 ${thresholds.accuracy},建议校验数据格式和值域,修正不合规记录`);
|
|
459
|
-
if (dimensions.consistency.score < thresholds.consistency) suggestions.push(`一致性得分 ${dimensions.consistency.score} 低于阈值 ${thresholds.consistency},建议检查跨字段约束规则,修正逻辑冲突记录`);
|
|
460
|
-
if (dimensions.timeliness.score < thresholds.timeliness) suggestions.push(`时效性得分 ${dimensions.timeliness.score} 低于阈值 ${thresholds.timeliness},建议更新过期数据或调整时效阈值`);
|
|
461
|
-
return suggestions;
|
|
462
|
-
}
|
|
463
|
-
};
|
|
464
|
-
//#endregion
|
|
465
|
-
//#region lib/types/qualityReportGenerator.js
|
|
466
|
-
var QualityReportGenerator = class {
|
|
467
|
-
async writeJson(filePath, result) {
|
|
468
|
-
const dir = path.dirname(filePath);
|
|
469
|
-
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
|
|
470
|
-
const jsonContent = {
|
|
471
|
-
dimensions: result.dimensions,
|
|
472
|
-
totalScore: result.totalScore,
|
|
473
|
-
issues: result.issues,
|
|
474
|
-
suggestions: result.suggestions,
|
|
475
|
-
weightsNormalized: result.weightsNormalized,
|
|
476
|
-
scoredAt: result.scoredAt
|
|
477
|
-
};
|
|
478
|
-
fs.writeFileSync(filePath, JSON.stringify(jsonContent, null, 2), "utf-8");
|
|
479
|
-
}
|
|
480
|
-
async writeMarkdown(filePath, result) {
|
|
481
|
-
const dir = path.dirname(filePath);
|
|
482
|
-
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
|
|
483
|
-
const lines = [];
|
|
484
|
-
lines.push("【数据质量评分报告】");
|
|
485
|
-
lines.push("");
|
|
486
|
-
lines.push(`> 评分时间: ${result.scoredAt}`);
|
|
487
|
-
lines.push(`> 总分: ${result.totalScore} 分`);
|
|
488
|
-
lines.push("");
|
|
489
|
-
lines.push("## 维度得分");
|
|
490
|
-
lines.push("");
|
|
491
|
-
lines.push("| 维度 | 得分 | 权重 | 是否归一化 |");
|
|
492
|
-
lines.push("|------|------|------|-----------|");
|
|
493
|
-
lines.push(`| 完整性 | ${result.dimensions.completeness.score} | ${result.dimensions.completeness.weight} | ${result.weightsNormalized ? "是" : "否"} |`);
|
|
494
|
-
lines.push(`| 准确性 | ${result.dimensions.accuracy.score} | ${result.dimensions.accuracy.weight} | ${result.weightsNormalized ? "是" : "否"} |`);
|
|
495
|
-
lines.push(`| 一致性 | ${result.dimensions.consistency.score} | ${result.dimensions.consistency.weight} | ${result.weightsNormalized ? "是" : "否"} |`);
|
|
496
|
-
lines.push(`| 时效性 | ${result.dimensions.timeliness.score} | ${result.dimensions.timeliness.weight} | ${result.weightsNormalized ? "是" : "否"} |`);
|
|
497
|
-
lines.push("");
|
|
498
|
-
lines.push("## 问题明细");
|
|
499
|
-
lines.push("");
|
|
500
|
-
if (result.issues.length === 0) lines.push("无问题");
|
|
501
|
-
else {
|
|
502
|
-
lines.push("| 维度 | 描述 | 位置 | 严重程度 |");
|
|
503
|
-
lines.push("|------|------|------|---------|");
|
|
504
|
-
for (const issue of result.issues) lines.push(`| ${issue.dimension} | ${issue.description} | ${issue.location} | ${issue.severity} |`);
|
|
505
|
-
}
|
|
506
|
-
lines.push("");
|
|
507
|
-
lines.push("## 改进建议");
|
|
508
|
-
lines.push("");
|
|
509
|
-
if (result.suggestions.length === 0) lines.push("各维度得分均高于阈值,无需改进");
|
|
510
|
-
else for (const suggestion of result.suggestions) lines.push(`- ${suggestion}`);
|
|
511
|
-
lines.push("");
|
|
512
|
-
lines.push("## 配置状态");
|
|
513
|
-
lines.push("");
|
|
514
|
-
lines.push(`- 权重是否归一化: ${result.weightsNormalized ? "是" : "否"}`);
|
|
515
|
-
lines.push(`- 问题总数: ${result.issues.length}`);
|
|
516
|
-
lines.push(`- 建议总数: ${result.suggestions.length}`);
|
|
517
|
-
fs.writeFileSync(filePath, lines.join("\n"), "utf-8");
|
|
518
|
-
}
|
|
519
|
-
};
|
|
520
|
-
//#endregion
|
|
521
|
-
//#region lib/types/index.js
|
|
522
|
-
const name = "data-quality-scoring";
|
|
523
|
-
const inject = ["tools"];
|
|
524
|
-
function parseRecords(lines, format) {
|
|
525
|
-
if (format === "csv" && lines.length > 0) {
|
|
526
|
-
const header = lines[0].split(",");
|
|
527
|
-
const records = [];
|
|
528
|
-
for (let i = 1; i < lines.length; i++) {
|
|
529
|
-
const values = lines[i].split(",");
|
|
530
|
-
const record = {};
|
|
531
|
-
for (let j = 0; j < header.length; j++) record[header[j]] = values[j] ?? "";
|
|
532
|
-
records.push(record);
|
|
533
|
-
}
|
|
534
|
-
return records;
|
|
535
|
-
}
|
|
536
|
-
if (format === "json") {
|
|
537
|
-
const records = [];
|
|
538
|
-
for (const line of lines) try {
|
|
539
|
-
const parsed = JSON.parse(line);
|
|
540
|
-
if (typeof parsed === "object" && parsed !== null) records.push(parsed);
|
|
541
|
-
} catch {}
|
|
542
|
-
return records;
|
|
543
|
-
}
|
|
544
|
-
return [];
|
|
545
|
-
}
|
|
546
|
-
function buildDimensionScore(result, weight, normalizedWeight) {
|
|
547
|
-
return {
|
|
548
|
-
score: result.score,
|
|
549
|
-
weight,
|
|
550
|
-
weightNormalized: normalizedWeight,
|
|
551
|
-
issues: result.issues
|
|
552
|
-
};
|
|
553
|
-
}
|
|
554
|
-
function buildDimensions(completeness, accuracy, consistency, timeliness, weights, normalizedWeights) {
|
|
555
|
-
return {
|
|
556
|
-
completeness: buildDimensionScore(completeness, weights.completeness, normalizedWeights.completeness),
|
|
557
|
-
accuracy: buildDimensionScore(accuracy, weights.accuracy, normalizedWeights.accuracy),
|
|
558
|
-
consistency: buildDimensionScore(consistency, weights.consistency, normalizedWeights.consistency),
|
|
559
|
-
timeliness: buildDimensionScore(timeliness, weights.timeliness, normalizedWeights.timeliness)
|
|
560
|
-
};
|
|
561
|
-
}
|
|
562
|
-
function apply(ctx) {
|
|
563
|
-
const loader = new BusinessRulesLoader();
|
|
564
|
-
const formatAdapter = new FileFormatAdapter();
|
|
565
|
-
const pathValidator = new PathValidator();
|
|
566
|
-
const auditLogger = new AuditLogger();
|
|
567
|
-
const completenessScorer = new CompletenessScorer();
|
|
568
|
-
const accuracyScorer = new AccuracyScorer();
|
|
569
|
-
const consistencyScorer = new ConsistencyScorer();
|
|
570
|
-
const timelinessScorer = new TimelinessScorer();
|
|
571
|
-
const weightedCalculator = new WeightedScoreCalculator();
|
|
572
|
-
const issueCollector = new IssueCollector();
|
|
573
|
-
const suggestionGenerator = new SuggestionGenerator();
|
|
574
|
-
const reportGenerator = new QualityReportGenerator();
|
|
575
|
-
ctx.tools.register(defineTool({
|
|
576
|
-
name: "score_data_quality",
|
|
577
|
-
description: "对数据文件进行多维度质量评分(完整性/准确性/一致性/时效性),生成质量评分报告",
|
|
578
|
-
parameters: {
|
|
579
|
-
filePath: {
|
|
580
|
-
type: "string",
|
|
581
|
-
required: true,
|
|
582
|
-
description: "待评分数据文件的路径"
|
|
583
|
-
},
|
|
584
|
-
outputPathDir: {
|
|
585
|
-
type: "string",
|
|
586
|
-
description: "报告输出目录(默认数据文件所在目录)"
|
|
587
|
-
}
|
|
588
|
-
},
|
|
589
|
-
output: {
|
|
590
|
-
schema: { type: "string" },
|
|
591
|
-
render: (_args, value) => [{
|
|
592
|
-
type: "text",
|
|
593
|
-
text: value
|
|
594
|
-
}]
|
|
595
|
-
},
|
|
596
|
-
async execute(args) {
|
|
597
|
-
const { config, status: configStatus } = loader.load();
|
|
598
|
-
const workingDir = process.cwd();
|
|
599
|
-
const filePath = args.filePath;
|
|
600
|
-
const outputPathDir = args.outputPathDir ?? path.dirname(path.resolve(filePath));
|
|
601
|
-
try {
|
|
602
|
-
pathValidator.validate(filePath, workingDir);
|
|
603
|
-
} catch {
|
|
604
|
-
return `错误:路径不合法 - ${filePath}`;
|
|
605
|
-
}
|
|
606
|
-
const fullPath = path.resolve(filePath);
|
|
607
|
-
if (!fs.existsSync(fullPath)) return `错误:文件不存在 - ${fullPath}`;
|
|
608
|
-
const scoringConfig = config.qualityScoring ?? defaultScoringRules;
|
|
609
|
-
const readResult = await formatAdapter.read(fullPath);
|
|
610
|
-
const records = parseRecords(readResult.lines, readResult.format);
|
|
611
|
-
const context = {
|
|
612
|
-
records,
|
|
613
|
-
fieldNames: records.length > 0 ? Object.keys(records[0]) : [],
|
|
614
|
-
config: scoringConfig
|
|
615
|
-
};
|
|
616
|
-
const completenessResult = completenessScorer.score(context);
|
|
617
|
-
const accuracyResult = accuracyScorer.score(context);
|
|
618
|
-
const consistencyResult = consistencyScorer.score(context);
|
|
619
|
-
const timelinessResult = timelinessScorer.score(context);
|
|
620
|
-
const weights = scoringConfig.weights;
|
|
621
|
-
const tempDimensions = buildDimensions(completenessResult, accuracyResult, consistencyResult, timelinessResult, weights, weights);
|
|
622
|
-
const { totalScore, weightsNormalized, normalizedWeights } = weightedCalculator.calculate(tempDimensions, weights);
|
|
623
|
-
const dimensions = buildDimensions(completenessResult, accuracyResult, consistencyResult, timelinessResult, weights, normalizedWeights);
|
|
624
|
-
const scoringResult = {
|
|
625
|
-
totalScore,
|
|
626
|
-
dimensions,
|
|
627
|
-
issues: issueCollector.collect([
|
|
628
|
-
completenessResult,
|
|
629
|
-
accuracyResult,
|
|
630
|
-
consistencyResult,
|
|
631
|
-
timelinessResult
|
|
632
|
-
]),
|
|
633
|
-
suggestions: suggestionGenerator.generate(dimensions, scoringConfig.thresholds),
|
|
634
|
-
weightsNormalized,
|
|
635
|
-
scoredAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
636
|
-
};
|
|
637
|
-
const jsonReportPath = path.join(outputPathDir, "quality_report.json");
|
|
638
|
-
const markdownReportPath = path.join(outputPathDir, "quality_report.md");
|
|
639
|
-
await reportGenerator.writeJson(jsonReportPath, scoringResult);
|
|
640
|
-
await reportGenerator.writeMarkdown(markdownReportPath, scoringResult);
|
|
641
|
-
auditLogger.log({
|
|
642
|
-
pluginName: "data-quality-scoring",
|
|
643
|
-
operation: "score_data_quality",
|
|
644
|
-
inputPath: fullPath,
|
|
645
|
-
outputPath: jsonReportPath,
|
|
646
|
-
result: "SUCCESS"
|
|
647
|
-
});
|
|
648
|
-
return JSON.stringify({
|
|
649
|
-
jsonReportPath,
|
|
650
|
-
markdownReportPath,
|
|
651
|
-
report: scoringResult,
|
|
652
|
-
totalScore,
|
|
653
|
-
dimensions,
|
|
654
|
-
status: "SUCCESS",
|
|
655
|
-
configStatus
|
|
656
|
-
}, null, 2);
|
|
657
|
-
}
|
|
658
|
-
}));
|
|
659
|
-
console.log("[data-quality-scoring] 数据质量评分插件已加载");
|
|
660
|
-
}
|
|
661
|
-
//#endregion
|
|
662
|
-
export { apply, inject, name };
|
package/lib/invariant.js
DELETED
package/lib/types/index.d.ts
DELETED
package/lib/types/invariant.d.ts
DELETED
|
@@ -1,6 +0,0 @@
|
|
|
1
|
-
import type { QualityScoringResult } from './types.js';
|
|
2
|
-
export declare class QualityReportGenerator {
|
|
3
|
-
writeJson(filePath: string, result: QualityScoringResult): Promise<void>;
|
|
4
|
-
writeMarkdown(filePath: string, result: QualityScoringResult): Promise<void>;
|
|
5
|
-
}
|
|
6
|
-
//# sourceMappingURL=qualityReportGenerator.d.ts.map
|
|
@@ -1,6 +0,0 @@
|
|
|
1
|
-
import type { QualityDimensions } from './types.js';
|
|
2
|
-
import type { QualityScoringConfig } from '@liuhange/dsh-data-asset-shared';
|
|
3
|
-
export declare class SuggestionGenerator {
|
|
4
|
-
generate(dimensions: QualityDimensions, thresholds: QualityScoringConfig['thresholds']): string[];
|
|
5
|
-
}
|
|
6
|
-
//# sourceMappingURL=suggestionGenerator.d.ts.map
|
package/lib/types/types.d.ts
DELETED
|
@@ -1,54 +0,0 @@
|
|
|
1
|
-
export type QualityDimension = 'completeness' | 'accuracy' | 'consistency' | 'timeliness';
|
|
2
|
-
export type IssueSeverity = 'low' | 'medium' | 'high';
|
|
3
|
-
export interface ScoreDataQualityParams {
|
|
4
|
-
filePath: string;
|
|
5
|
-
outputPathDir?: string;
|
|
6
|
-
}
|
|
7
|
-
export interface QualityIssue {
|
|
8
|
-
dimension: QualityDimension;
|
|
9
|
-
description: string;
|
|
10
|
-
location: string;
|
|
11
|
-
severity: IssueSeverity;
|
|
12
|
-
}
|
|
13
|
-
export interface DimensionScore {
|
|
14
|
-
score: number;
|
|
15
|
-
weight: number;
|
|
16
|
-
weightNormalized: number;
|
|
17
|
-
subScores?: Record<string, number>;
|
|
18
|
-
issues: QualityIssue[];
|
|
19
|
-
suggestion?: string;
|
|
20
|
-
}
|
|
21
|
-
export interface QualityDimensions {
|
|
22
|
-
completeness: DimensionScore;
|
|
23
|
-
accuracy: DimensionScore;
|
|
24
|
-
consistency: DimensionScore;
|
|
25
|
-
timeliness: DimensionScore;
|
|
26
|
-
}
|
|
27
|
-
export interface QualityScoringResult {
|
|
28
|
-
totalScore: number;
|
|
29
|
-
dimensions: QualityDimensions;
|
|
30
|
-
issues: QualityIssue[];
|
|
31
|
-
suggestions: string[];
|
|
32
|
-
weightsNormalized: boolean;
|
|
33
|
-
scoredAt: string;
|
|
34
|
-
}
|
|
35
|
-
export interface ScoreDataQualityResult {
|
|
36
|
-
jsonReportPath: string;
|
|
37
|
-
markdownReportPath: string;
|
|
38
|
-
report: QualityScoringResult;
|
|
39
|
-
totalScore: number;
|
|
40
|
-
dimensions: QualityDimensions;
|
|
41
|
-
status: 'SUCCESS' | 'FAILED';
|
|
42
|
-
configStatus: string;
|
|
43
|
-
}
|
|
44
|
-
export interface ScoringContext {
|
|
45
|
-
records: Record<string, unknown>[];
|
|
46
|
-
fieldNames: string[];
|
|
47
|
-
config: unknown;
|
|
48
|
-
}
|
|
49
|
-
export interface DimensionScoringResult {
|
|
50
|
-
score: number;
|
|
51
|
-
issues: QualityIssue[];
|
|
52
|
-
subScores?: Record<string, number>;
|
|
53
|
-
}
|
|
54
|
-
//# sourceMappingURL=types.d.ts.map
|
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
import type { QualityDimensions } from './types.js';
|
|
2
|
-
import type { QualityScoringConfig } from '@liuhange/dsh-data-asset-shared';
|
|
3
|
-
export declare class WeightedScoreCalculator {
|
|
4
|
-
calculate(dimensions: QualityDimensions, weights: QualityScoringConfig['weights']): {
|
|
5
|
-
totalScore: number;
|
|
6
|
-
weightsNormalized: boolean;
|
|
7
|
-
normalizedWeights: QualityScoringConfig['weights'];
|
|
8
|
-
};
|
|
9
|
-
}
|
|
10
|
-
//# sourceMappingURL=weightedScoreCalculator.d.ts.map
|