@liuhange/dsh-data-asset-shared 2.0.1 → 2.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -1,4 +1,4 @@
1
- # @deepseek-ai/dsh-data-asset-shared
1
+ # @liuhange/dsh-data-asset-shared
2
2
 
3
3
  Shared infrastructure for data assetization plugins.
4
4
 
@@ -13,7 +13,7 @@ Shared infrastructure for data assetization plugins.
13
13
  ## Usage
14
14
 
15
15
  ```typescript
16
- import { BusinessRulesLoader, FileFormatAdapter, ReportGenerator, PathValidator, AuditLogger } from '@deepseek-ai/dsh-data-asset-shared'
16
+ import { BusinessRulesLoader, FileFormatAdapter, ReportGenerator, PathValidator, AuditLogger } from '@liuhange/dsh-data-asset-shared'
17
17
 
18
18
  const loader = new BusinessRulesLoader()
19
19
  const { config, status } = loader.load()
@@ -0,0 +1,26 @@
1
+ import * as fs from 'fs';
2
+ import * as path from 'path';
3
+ export class AuditLogger {
4
+ logDir;
5
+ constructor(logDir) {
6
+ this.logDir = logDir ?? path.resolve(process.cwd(), 'logs');
7
+ }
8
+ log(params) {
9
+ const timestamp = new Date().toISOString();
10
+ const operator = params.operator ?? 'system';
11
+ const logLine = `[${params.pluginName}] ${timestamp} | 操作: ${params.operation} | 操作人: ${operator} | 输入: ${params.inputPath} | 输出: ${params.outputPath} | 结果: ${params.result}`;
12
+ console.log(logLine);
13
+ if (params.writeToFile) {
14
+ this.writeToFile(logLine);
15
+ }
16
+ }
17
+ writeToFile(logLine) {
18
+ if (!fs.existsSync(this.logDir)) {
19
+ fs.mkdirSync(this.logDir, { recursive: true });
20
+ }
21
+ const date = new Date().toISOString().split('T')[0];
22
+ const logFile = path.join(this.logDir, `audit-${date}.log`);
23
+ fs.appendFileSync(logFile, logLine + '\n', 'utf-8');
24
+ }
25
+ }
26
+ //# sourceMappingURL=auditLogger.js.map
@@ -0,0 +1,86 @@
1
+ import * as fs from 'fs';
2
+ import * as path from 'path';
3
+ import { defaultBusinessRules } from './defaultBusinessRules.js';
4
+ export class BusinessRulesLoader {
5
+ cachedResult = null;
6
+ configPath;
7
+ constructor(configPath) {
8
+ this.configPath = configPath
9
+ ? path.resolve(configPath)
10
+ : path.resolve(process.cwd(), 'config', 'business-rules.json');
11
+ }
12
+ load() {
13
+ if (this.cachedResult) {
14
+ return this.cachedResult;
15
+ }
16
+ const result = this.loadInternal();
17
+ this.cachedResult = result;
18
+ return result;
19
+ }
20
+ loadInternal() {
21
+ if (!fs.existsSync(this.configPath)) {
22
+ return this.createDefaultResult('DEFAULT_MISSING');
23
+ }
24
+ let raw;
25
+ try {
26
+ raw = fs.readFileSync(this.configPath, 'utf-8');
27
+ }
28
+ catch {
29
+ return this.createDefaultResult('DEFAULT_PARSE');
30
+ }
31
+ let parsed;
32
+ try {
33
+ parsed = JSON.parse(raw);
34
+ }
35
+ catch {
36
+ return this.createDefaultResult('DEFAULT_PARSE');
37
+ }
38
+ if (!this.isBusinessRulesConfig(parsed)) {
39
+ return this.createDefaultResult('DEFAULT_PARSE');
40
+ }
41
+ if (!this.isVersionCompatible(parsed.version)) {
42
+ return this.createDefaultResult('DEFAULT_VERSION');
43
+ }
44
+ if (!this.hasRequiredNodes(parsed)) {
45
+ return this.createDefaultResult('DEFAULT_PARTIAL');
46
+ }
47
+ return {
48
+ config: parsed,
49
+ status: 'CONFIG_LOADED',
50
+ };
51
+ }
52
+ createDefaultResult(status) {
53
+ return {
54
+ config: defaultBusinessRules,
55
+ status,
56
+ };
57
+ }
58
+ isBusinessRulesConfig(value) {
59
+ return (typeof value === 'object' &&
60
+ value !== null &&
61
+ 'version' in value &&
62
+ 'valueAssessment' in value &&
63
+ 'sensitivePatterns' in value &&
64
+ 'cleaningRules' in value &&
65
+ 'compliance' in value &&
66
+ 'packaging' in value);
67
+ }
68
+ isVersionCompatible(version) {
69
+ if (typeof version !== 'string') {
70
+ return false;
71
+ }
72
+ return version.startsWith('1.');
73
+ }
74
+ hasRequiredNodes(config) {
75
+ return (config.valueAssessment !== undefined &&
76
+ config.sensitivePatterns !== undefined &&
77
+ config.maskingLevels !== undefined &&
78
+ config.cleaningRules !== undefined &&
79
+ config.compliance !== undefined &&
80
+ config.packaging !== undefined);
81
+ }
82
+ clearCache() {
83
+ this.cachedResult = null;
84
+ }
85
+ }
86
+ //# sourceMappingURL=businessRulesLoader.js.map
@@ -0,0 +1,221 @@
1
+ export const defaultQualityScoringConfig = {
2
+ weights: {
3
+ completeness: 0.25,
4
+ accuracy: 0.25,
5
+ consistency: 0.25,
6
+ timeliness: 0.25,
7
+ },
8
+ thresholds: {
9
+ completeness: 60,
10
+ accuracy: 60,
11
+ consistency: 60,
12
+ timeliness: 60,
13
+ },
14
+ completeness: {
15
+ missingMarkers: ['null', 'undefined', 'N/A', '', 'NULL', 'NaN'],
16
+ },
17
+ accuracy: {
18
+ formatRules: [
19
+ { fieldName: 'phone', pattern: '1[3-9]\\d{9}', weight: 0.5 },
20
+ { fieldName: 'email', pattern: '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', weight: 0.5 },
21
+ ],
22
+ domainRules: [
23
+ { fieldName: 'age', min: 0, max: 150, weight: 0.5 },
24
+ { fieldName: 'score', min: 0, max: 100, weight: 0.5 },
25
+ ],
26
+ },
27
+ consistency: {
28
+ crossFieldRules: [
29
+ { name: 'endDateAfterStartDate', fields: ['startDate', 'endDate'], constraint: 'after' },
30
+ ],
31
+ },
32
+ timeliness: {
33
+ timestampField: 'updatedAt',
34
+ freshnessThresholdHours: 720,
35
+ },
36
+ };
37
+ export const defaultLineageConfig = {
38
+ enabled: false,
39
+ storagePath: 'lineage',
40
+ hashAlgorithm: 'SHA-256',
41
+ };
42
+ export const defaultAdvancedMaskingConfig = {
43
+ defaultAlgorithm: 'hash',
44
+ fieldAlgorithms: {},
45
+ fpe: {
46
+ key: '',
47
+ radix: 10,
48
+ },
49
+ kAnonymity: {
50
+ kValue: 2,
51
+ quasiIdentifiers: ['zipCode', 'age', 'gender'],
52
+ },
53
+ differentialPrivacy: {
54
+ epsilon: 1.0,
55
+ totalBudget: 5.0,
56
+ sensitivity: 1.0,
57
+ },
58
+ hash: {
59
+ algorithm: 'SHA-256',
60
+ salt: '',
61
+ },
62
+ };
63
+ export const defaultVisualizationConfig = {
64
+ charts: {
65
+ bar: { enabled: true, title: '脱敏统计' },
66
+ comparison: { enabled: true, title: '清洗前后对比' },
67
+ pie: { enabled: true, title: '数据资产分布' },
68
+ radar: { enabled: true, title: '质量评分' },
69
+ flowchart: { enabled: true, title: '血缘追踪' },
70
+ },
71
+ layout: {
72
+ columns: 2,
73
+ responsive: true,
74
+ },
75
+ colorScheme: {
76
+ primary: '#4A90D9',
77
+ secondary: '#F5A623',
78
+ success: '#7ED321',
79
+ warning: '#F8E71C',
80
+ danger: '#D0021B',
81
+ background: '#F5F5F5',
82
+ text: '#333333',
83
+ },
84
+ interactions: {
85
+ expandCollapse: true,
86
+ chartSwitch: true,
87
+ tooltip: true,
88
+ },
89
+ templateVersion: '1.0',
90
+ };
91
+ export const defaultSensitivityClassificationConfig = {
92
+ fieldNameRules: [
93
+ { name: 'idCard', pattern: 'id_?card|身份证|identity', level: 'Secret' },
94
+ { name: 'phone', pattern: 'phone|手机|mobile|tel', level: 'Confidential' },
95
+ { name: 'email', pattern: 'email|邮箱|mail', level: 'Confidential' },
96
+ { name: 'bankCard', pattern: 'bank_?card|银行卡|account', level: 'Secret' },
97
+ { name: 'address', pattern: 'address|地址|addr', level: 'Internal' },
98
+ { name: 'name', pattern: '^name$|姓名|username', level: 'Internal' },
99
+ { name: 'public', pattern: 'status|type|category|label|tag|level', level: 'Public' },
100
+ ],
101
+ fieldValueRules: [
102
+ { name: 'idCardValue', pattern: '[1-9]\\d{5}(18|19|20)?\\d{2}(0[1-9]|1[0-2])(0[1-9]|[12]\\d|3[01])\\d{3}[\\dXx]', level: 'Secret' },
103
+ { name: 'phoneValue', pattern: '1[3-9]\\d{9}', level: 'Confidential' },
104
+ { name: 'emailValue', pattern: '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', level: 'Confidential' },
105
+ { name: 'bankCardValue', pattern: '[1-9]\\d{15,18}', level: 'Secret' },
106
+ ],
107
+ levelMapping: {
108
+ Public: 'none',
109
+ Internal: 'partial',
110
+ Confidential: 'full',
111
+ Secret: 'encrypt',
112
+ },
113
+ defaultLevel: 'Internal',
114
+ };
115
+ export const defaultIncrementalSchedulingConfig = {
116
+ incrementalMode: false,
117
+ cronExpression: '0 2 * * *',
118
+ hashAlgorithm: 'SHA-256',
119
+ watchDirectory: 'input',
120
+ stateFilePath: 'state/incremental-state.json',
121
+ lockTimeout: 3600,
122
+ };
123
+ export const defaultBusinessRules = {
124
+ version: '1.0',
125
+ lastUpdated: '2026-08-17',
126
+ valueAssessment: {
127
+ highValue: {
128
+ keywords: ['供应链', '销售明细', '设备工况', '交易流水'],
129
+ score: 5,
130
+ label: '★★★★★',
131
+ recommendation: '优先交易,建议入表',
132
+ },
133
+ mediumValue: {
134
+ keywords: ['用户行为', '产品数据', '行业报告'],
135
+ score: 3,
136
+ label: '★★★',
137
+ recommendation: '脱敏后交易',
138
+ },
139
+ lowValue: {
140
+ keywords: ['日志', '备份', '临时'],
141
+ score: 1,
142
+ label: '★',
143
+ recommendation: '评估处理成本',
144
+ },
145
+ defaultValue: {
146
+ score: 3,
147
+ label: '★★★',
148
+ recommendation: '需进一步评估',
149
+ },
150
+ },
151
+ sensitivePatterns: {
152
+ idCard: {
153
+ pattern: '[1-9]\\d{5}(18|19|20)?\\d{2}(0[1-9]|1[0-2])(0[1-9]|[12]\\d|3[01])\\d{3}[\\dXx]',
154
+ level: 'FULL',
155
+ },
156
+ phone: {
157
+ pattern: '1[3-9]\\d{9}',
158
+ level: 'PARTIAL',
159
+ },
160
+ bankCard: {
161
+ pattern: '[1-9]\\d{15,18}',
162
+ level: 'PARTIAL',
163
+ },
164
+ email: {
165
+ pattern: '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}',
166
+ level: 'PARTIAL',
167
+ },
168
+ },
169
+ maskingLevels: {
170
+ FULL: {
171
+ description: '完全匿名化',
172
+ applyTo: ['idCard'],
173
+ },
174
+ PARTIAL: {
175
+ description: '假名化,保留前3后4',
176
+ applyTo: ['phone', 'bankCard'],
177
+ },
178
+ GENERALIZE: {
179
+ description: '泛化处理',
180
+ applyTo: ['email'],
181
+ },
182
+ },
183
+ cleaningRules: {
184
+ missingValueStrategy: {
185
+ government: 'interpolation',
186
+ financial: 'manual_review',
187
+ behavior: 'mode_fill',
188
+ },
189
+ qualityThreshold: {
190
+ A: { missingRate: 0.05, errorRate: 0.01, label: '优秀' },
191
+ B: { missingRate: 0.10, errorRate: 0.03, label: '良好' },
192
+ C: { missingRate: 0.20, errorRate: 0.05, label: '合格' },
193
+ D: { missingRate: 0.20, errorRate: 0.05, label: '不合格', action: '不建议交易' },
194
+ },
195
+ },
196
+ compliance: {
197
+ policies: [
198
+ { name: '数据产权登记工作指引', doc: '国数综政策〔2026〕35号', check: '数据来源可登记' },
199
+ { name: '数据流通安全治理', doc: '发改数据〔2025〕18号', check: '来源合法性' },
200
+ { name: '数据脱敏工具标准', doc: 'YD/T 6225-2024', check: '脱敏效果达标' },
201
+ ],
202
+ },
203
+ packaging: {
204
+ complianceStatements: [
205
+ '本数据产品已完成脱敏处理',
206
+ '不包含个人身份信息',
207
+ '数据来源合法',
208
+ '建议交易前完成数据产权登记',
209
+ ],
210
+ pricingRules: [
211
+ { valueLevel: '★★★★★', suggestion: '根据数据质量、稀缺性和应用场景综合定价,建议高价交易' },
212
+ { valueLevel: '★★★★', suggestion: '根据数据质量和应用场景定价,建议中高价交易' },
213
+ { valueLevel: '★★★', suggestion: '根据数据质量和应用场景综合定价' },
214
+ { valueLevel: '★★', suggestion: '建议低价交易或内部使用' },
215
+ { valueLevel: '★', suggestion: '建议评估处理成本后决定是否交易' },
216
+ ],
217
+ defaultDescription: '适用于数据分析、模型训练、行业研究等场景',
218
+ version: 'V1.0',
219
+ },
220
+ };
221
+ //# sourceMappingURL=defaultBusinessRules.js.map
@@ -0,0 +1,98 @@
1
+ import * as fs from 'fs';
2
+ import * as path from 'path';
3
+ export class FileFormatAdapter {
4
+ detectFormat(filePath) {
5
+ const ext = path.extname(filePath).toLowerCase().slice(1);
6
+ if (ext === 'csv' || ext === 'json' || ext === 'txt' || ext === 'xlsx') {
7
+ return ext;
8
+ }
9
+ return 'unknown';
10
+ }
11
+ async read(filePath) {
12
+ const format = this.detectFormat(filePath);
13
+ const fullPath = path.resolve(filePath);
14
+ if (!fs.existsSync(fullPath)) {
15
+ throw new Error(`错误:文件不存在 - ${fullPath}`);
16
+ }
17
+ const stats = fs.statSync(fullPath);
18
+ const isLargeFile = stats.size >= 10 * 1024 * 1024;
19
+ switch (format) {
20
+ case 'csv':
21
+ return this.readCsv(fullPath);
22
+ case 'json':
23
+ return this.readJson(fullPath);
24
+ case 'xlsx':
25
+ return this.readXlsx(fullPath);
26
+ case 'txt':
27
+ case 'unknown':
28
+ default:
29
+ return this.readTxt(fullPath, format, isLargeFile);
30
+ }
31
+ }
32
+ async write(filePath, lines, format) {
33
+ const fullPath = path.resolve(filePath);
34
+ const dir = path.dirname(fullPath);
35
+ if (!fs.existsSync(dir)) {
36
+ fs.mkdirSync(dir, { recursive: true });
37
+ }
38
+ switch (format) {
39
+ case 'csv':
40
+ this.writeCsv(fullPath, lines);
41
+ break;
42
+ case 'json':
43
+ this.writeJson(fullPath, lines);
44
+ break;
45
+ case 'xlsx':
46
+ this.writeXlsx(fullPath, lines);
47
+ break;
48
+ case 'txt':
49
+ case 'unknown':
50
+ default:
51
+ fs.writeFileSync(fullPath, lines.join('\n'), 'utf-8');
52
+ break;
53
+ }
54
+ }
55
+ readTxt(filePath, format, _isLargeFile) {
56
+ const content = fs.readFileSync(filePath, 'utf-8');
57
+ const lines = content.split('\n').filter(line => line.trim());
58
+ return { lines, format, raw: content };
59
+ }
60
+ readCsv(filePath) {
61
+ const content = fs.readFileSync(filePath, 'utf-8');
62
+ const lines = content.split('\n').filter(line => line.trim());
63
+ return { lines, format: 'csv', raw: content };
64
+ }
65
+ readJson(filePath) {
66
+ const content = fs.readFileSync(filePath, 'utf-8');
67
+ const parsed = JSON.parse(content);
68
+ if (Array.isArray(parsed)) {
69
+ const lines = parsed.map(item => JSON.stringify(item));
70
+ return { lines, format: 'json', raw: parsed };
71
+ }
72
+ const lines = [JSON.stringify(parsed)];
73
+ return { lines, format: 'json', raw: parsed };
74
+ }
75
+ readXlsx(filePath) {
76
+ const content = fs.readFileSync(filePath, 'utf-8');
77
+ const lines = content.split('\n').filter(line => line.trim());
78
+ return { lines, format: 'xlsx', raw: content };
79
+ }
80
+ writeCsv(filePath, lines) {
81
+ fs.writeFileSync(filePath, lines.join('\n'), 'utf-8');
82
+ }
83
+ writeJson(filePath, lines) {
84
+ const objects = lines.map((line) => {
85
+ try {
86
+ return JSON.parse(line);
87
+ }
88
+ catch {
89
+ return line;
90
+ }
91
+ });
92
+ fs.writeFileSync(filePath, JSON.stringify(objects, null, 2), 'utf-8');
93
+ }
94
+ writeXlsx(filePath, lines) {
95
+ fs.writeFileSync(filePath, lines.join('\n'), 'utf-8');
96
+ }
97
+ }
98
+ //# sourceMappingURL=fileFormatAdapter.js.map
package/lib/index.js CHANGED
@@ -1,617 +1,9 @@
1
- import * as fs from "fs";
2
- import * as path from "path";
3
- //#region lib/types/defaultBusinessRules.js
4
- const defaultQualityScoringConfig = {
5
- weights: {
6
- completeness: .25,
7
- accuracy: .25,
8
- consistency: .25,
9
- timeliness: .25
10
- },
11
- thresholds: {
12
- completeness: 60,
13
- accuracy: 60,
14
- consistency: 60,
15
- timeliness: 60
16
- },
17
- completeness: { missingMarkers: [
18
- "null",
19
- "undefined",
20
- "N/A",
21
- "",
22
- "NULL",
23
- "NaN"
24
- ] },
25
- accuracy: {
26
- formatRules: [{
27
- fieldName: "phone",
28
- pattern: "1[3-9]\\d{9}",
29
- weight: .5
30
- }, {
31
- fieldName: "email",
32
- pattern: "[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}",
33
- weight: .5
34
- }],
35
- domainRules: [{
36
- fieldName: "age",
37
- min: 0,
38
- max: 150,
39
- weight: .5
40
- }, {
41
- fieldName: "score",
42
- min: 0,
43
- max: 100,
44
- weight: .5
45
- }]
46
- },
47
- consistency: { crossFieldRules: [{
48
- name: "endDateAfterStartDate",
49
- fields: ["startDate", "endDate"],
50
- constraint: "after"
51
- }] },
52
- timeliness: {
53
- timestampField: "updatedAt",
54
- freshnessThresholdHours: 720
55
- }
56
- };
57
- const defaultLineageConfig = {
58
- enabled: false,
59
- storagePath: "lineage",
60
- hashAlgorithm: "SHA-256"
61
- };
62
- const defaultAdvancedMaskingConfig = {
63
- defaultAlgorithm: "hash",
64
- fieldAlgorithms: {},
65
- fpe: {
66
- key: "",
67
- radix: 10
68
- },
69
- kAnonymity: {
70
- kValue: 2,
71
- quasiIdentifiers: [
72
- "zipCode",
73
- "age",
74
- "gender"
75
- ]
76
- },
77
- differentialPrivacy: {
78
- epsilon: 1,
79
- totalBudget: 5,
80
- sensitivity: 1
81
- },
82
- hash: {
83
- algorithm: "SHA-256",
84
- salt: ""
85
- }
86
- };
87
- const defaultVisualizationConfig = {
88
- charts: {
89
- bar: {
90
- enabled: true,
91
- title: "脱敏统计"
92
- },
93
- comparison: {
94
- enabled: true,
95
- title: "清洗前后对比"
96
- },
97
- pie: {
98
- enabled: true,
99
- title: "数据资产分布"
100
- },
101
- radar: {
102
- enabled: true,
103
- title: "质量评分"
104
- },
105
- flowchart: {
106
- enabled: true,
107
- title: "血缘追踪"
108
- }
109
- },
110
- layout: {
111
- columns: 2,
112
- responsive: true
113
- },
114
- colorScheme: {
115
- primary: "#4A90D9",
116
- secondary: "#F5A623",
117
- success: "#7ED321",
118
- warning: "#F8E71C",
119
- danger: "#D0021B",
120
- background: "#F5F5F5",
121
- text: "#333333"
122
- },
123
- interactions: {
124
- expandCollapse: true,
125
- chartSwitch: true,
126
- tooltip: true
127
- },
128
- templateVersion: "1.0"
129
- };
130
- const defaultSensitivityClassificationConfig = {
131
- fieldNameRules: [
132
- {
133
- name: "idCard",
134
- pattern: "id_?card|身份证|identity",
135
- level: "Secret"
136
- },
137
- {
138
- name: "phone",
139
- pattern: "phone|手机|mobile|tel",
140
- level: "Confidential"
141
- },
142
- {
143
- name: "email",
144
- pattern: "email|邮箱|mail",
145
- level: "Confidential"
146
- },
147
- {
148
- name: "bankCard",
149
- pattern: "bank_?card|银行卡|account",
150
- level: "Secret"
151
- },
152
- {
153
- name: "address",
154
- pattern: "address|地址|addr",
155
- level: "Internal"
156
- },
157
- {
158
- name: "name",
159
- pattern: "^name$|姓名|username",
160
- level: "Internal"
161
- },
162
- {
163
- name: "public",
164
- pattern: "status|type|category|label|tag|level",
165
- level: "Public"
166
- }
167
- ],
168
- fieldValueRules: [
169
- {
170
- name: "idCardValue",
171
- pattern: "[1-9]\\d{5}(18|19|20)?\\d{2}(0[1-9]|1[0-2])(0[1-9]|[12]\\d|3[01])\\d{3}[\\dXx]",
172
- level: "Secret"
173
- },
174
- {
175
- name: "phoneValue",
176
- pattern: "1[3-9]\\d{9}",
177
- level: "Confidential"
178
- },
179
- {
180
- name: "emailValue",
181
- pattern: "[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}",
182
- level: "Confidential"
183
- },
184
- {
185
- name: "bankCardValue",
186
- pattern: "[1-9]\\d{15,18}",
187
- level: "Secret"
188
- }
189
- ],
190
- levelMapping: {
191
- Public: "none",
192
- Internal: "partial",
193
- Confidential: "full",
194
- Secret: "encrypt"
195
- },
196
- defaultLevel: "Internal"
197
- };
198
- const defaultIncrementalSchedulingConfig = {
199
- incrementalMode: false,
200
- cronExpression: "0 2 * * *",
201
- hashAlgorithm: "SHA-256",
202
- watchDirectory: "input",
203
- stateFilePath: "state/incremental-state.json",
204
- lockTimeout: 3600
205
- };
206
- const defaultBusinessRules = {
207
- version: "1.0",
208
- lastUpdated: "2026-08-17",
209
- valueAssessment: {
210
- highValue: {
211
- keywords: [
212
- "供应链",
213
- "销售明细",
214
- "设备工况",
215
- "交易流水"
216
- ],
217
- score: 5,
218
- label: "★★★★★",
219
- recommendation: "优先交易,建议入表"
220
- },
221
- mediumValue: {
222
- keywords: [
223
- "用户行为",
224
- "产品数据",
225
- "行业报告"
226
- ],
227
- score: 3,
228
- label: "★★★",
229
- recommendation: "脱敏后交易"
230
- },
231
- lowValue: {
232
- keywords: [
233
- "日志",
234
- "备份",
235
- "临时"
236
- ],
237
- score: 1,
238
- label: "★",
239
- recommendation: "评估处理成本"
240
- },
241
- defaultValue: {
242
- score: 3,
243
- label: "★★★",
244
- recommendation: "需进一步评估"
245
- }
246
- },
247
- sensitivePatterns: {
248
- idCard: {
249
- pattern: "[1-9]\\d{5}(18|19|20)?\\d{2}(0[1-9]|1[0-2])(0[1-9]|[12]\\d|3[01])\\d{3}[\\dXx]",
250
- level: "FULL"
251
- },
252
- phone: {
253
- pattern: "1[3-9]\\d{9}",
254
- level: "PARTIAL"
255
- },
256
- bankCard: {
257
- pattern: "[1-9]\\d{15,18}",
258
- level: "PARTIAL"
259
- },
260
- email: {
261
- pattern: "[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}",
262
- level: "PARTIAL"
263
- }
264
- },
265
- maskingLevels: {
266
- FULL: {
267
- description: "完全匿名化",
268
- applyTo: ["idCard"]
269
- },
270
- PARTIAL: {
271
- description: "假名化,保留前3后4",
272
- applyTo: ["phone", "bankCard"]
273
- },
274
- GENERALIZE: {
275
- description: "泛化处理",
276
- applyTo: ["email"]
277
- }
278
- },
279
- cleaningRules: {
280
- missingValueStrategy: {
281
- government: "interpolation",
282
- financial: "manual_review",
283
- behavior: "mode_fill"
284
- },
285
- qualityThreshold: {
286
- A: {
287
- missingRate: .05,
288
- errorRate: .01,
289
- label: "优秀"
290
- },
291
- B: {
292
- missingRate: .1,
293
- errorRate: .03,
294
- label: "良好"
295
- },
296
- C: {
297
- missingRate: .2,
298
- errorRate: .05,
299
- label: "合格"
300
- },
301
- D: {
302
- missingRate: .2,
303
- errorRate: .05,
304
- label: "不合格",
305
- action: "不建议交易"
306
- }
307
- }
308
- },
309
- compliance: { policies: [
310
- {
311
- name: "数据产权登记工作指引",
312
- doc: "国数综政策〔2026〕35号",
313
- check: "数据来源可登记"
314
- },
315
- {
316
- name: "数据流通安全治理",
317
- doc: "发改数据〔2025〕18号",
318
- check: "来源合法性"
319
- },
320
- {
321
- name: "数据脱敏工具标准",
322
- doc: "YD/T 6225-2024",
323
- check: "脱敏效果达标"
324
- }
325
- ] },
326
- packaging: {
327
- complianceStatements: [
328
- "本数据产品已完成脱敏处理",
329
- "不包含个人身份信息",
330
- "数据来源合法",
331
- "建议交易前完成数据产权登记"
332
- ],
333
- pricingRules: [
334
- {
335
- valueLevel: "★★★★★",
336
- suggestion: "根据数据质量、稀缺性和应用场景综合定价,建议高价交易"
337
- },
338
- {
339
- valueLevel: "★★★★",
340
- suggestion: "根据数据质量和应用场景定价,建议中高价交易"
341
- },
342
- {
343
- valueLevel: "★★★",
344
- suggestion: "根据数据质量和应用场景综合定价"
345
- },
346
- {
347
- valueLevel: "★★",
348
- suggestion: "建议低价交易或内部使用"
349
- },
350
- {
351
- valueLevel: "★",
352
- suggestion: "建议评估处理成本后决定是否交易"
353
- }
354
- ],
355
- defaultDescription: "适用于数据分析、模型训练、行业研究等场景",
356
- version: "V1.0"
357
- }
358
- };
359
- //#endregion
360
- //#region lib/types/businessRulesLoader.js
361
- var BusinessRulesLoader = class {
362
- cachedResult = null;
363
- configPath;
364
- constructor(configPath) {
365
- this.configPath = configPath ? path.resolve(configPath) : path.resolve(process.cwd(), "config", "business-rules.json");
366
- }
367
- load() {
368
- if (this.cachedResult) return this.cachedResult;
369
- const result = this.loadInternal();
370
- this.cachedResult = result;
371
- return result;
372
- }
373
- loadInternal() {
374
- if (!fs.existsSync(this.configPath)) return this.createDefaultResult("DEFAULT_MISSING");
375
- let raw;
376
- try {
377
- raw = fs.readFileSync(this.configPath, "utf-8");
378
- } catch {
379
- return this.createDefaultResult("DEFAULT_PARSE");
380
- }
381
- let parsed;
382
- try {
383
- parsed = JSON.parse(raw);
384
- } catch {
385
- return this.createDefaultResult("DEFAULT_PARSE");
386
- }
387
- if (!this.isBusinessRulesConfig(parsed)) return this.createDefaultResult("DEFAULT_PARSE");
388
- if (!this.isVersionCompatible(parsed.version)) return this.createDefaultResult("DEFAULT_VERSION");
389
- if (!this.hasRequiredNodes(parsed)) return this.createDefaultResult("DEFAULT_PARTIAL");
390
- return {
391
- config: parsed,
392
- status: "CONFIG_LOADED"
393
- };
394
- }
395
- createDefaultResult(status) {
396
- return {
397
- config: defaultBusinessRules,
398
- status
399
- };
400
- }
401
- isBusinessRulesConfig(value) {
402
- return typeof value === "object" && value !== null && "version" in value && "valueAssessment" in value && "sensitivePatterns" in value && "cleaningRules" in value && "compliance" in value && "packaging" in value;
403
- }
404
- isVersionCompatible(version) {
405
- if (typeof version !== "string") return false;
406
- return version.startsWith("1.");
407
- }
408
- hasRequiredNodes(config) {
409
- return config.valueAssessment !== void 0 && config.sensitivePatterns !== void 0 && config.maskingLevels !== void 0 && config.cleaningRules !== void 0 && config.compliance !== void 0 && config.packaging !== void 0;
410
- }
411
- clearCache() {
412
- this.cachedResult = null;
413
- }
414
- };
415
- //#endregion
416
- //#region lib/types/fileFormatAdapter.js
417
- var FileFormatAdapter = class {
418
- detectFormat(filePath) {
419
- const ext = path.extname(filePath).toLowerCase().slice(1);
420
- if (ext === "csv" || ext === "json" || ext === "txt" || ext === "xlsx") return ext;
421
- return "unknown";
422
- }
423
- async read(filePath) {
424
- const format = this.detectFormat(filePath);
425
- const fullPath = path.resolve(filePath);
426
- if (!fs.existsSync(fullPath)) throw new Error(`错误:文件不存在 - ${fullPath}`);
427
- const isLargeFile = fs.statSync(fullPath).size >= 10 * 1024 * 1024;
428
- switch (format) {
429
- case "csv": return this.readCsv(fullPath);
430
- case "json": return this.readJson(fullPath);
431
- case "xlsx": return this.readXlsx(fullPath);
432
- default: return this.readTxt(fullPath, format, isLargeFile);
433
- }
434
- }
435
- async write(filePath, lines, format) {
436
- const fullPath = path.resolve(filePath);
437
- const dir = path.dirname(fullPath);
438
- if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
439
- switch (format) {
440
- case "csv":
441
- this.writeCsv(fullPath, lines);
442
- break;
443
- case "json":
444
- this.writeJson(fullPath, lines);
445
- break;
446
- case "xlsx":
447
- this.writeXlsx(fullPath, lines);
448
- break;
449
- default:
450
- fs.writeFileSync(fullPath, lines.join("\n"), "utf-8");
451
- break;
452
- }
453
- }
454
- readTxt(filePath, format, _isLargeFile) {
455
- const content = fs.readFileSync(filePath, "utf-8");
456
- return {
457
- lines: content.split("\n").filter((line) => line.trim()),
458
- format,
459
- raw: content
460
- };
461
- }
462
- readCsv(filePath) {
463
- const content = fs.readFileSync(filePath, "utf-8");
464
- return {
465
- lines: content.split("\n").filter((line) => line.trim()),
466
- format: "csv",
467
- raw: content
468
- };
469
- }
470
- readJson(filePath) {
471
- const content = fs.readFileSync(filePath, "utf-8");
472
- const parsed = JSON.parse(content);
473
- if (Array.isArray(parsed)) return {
474
- lines: parsed.map((item) => JSON.stringify(item)),
475
- format: "json",
476
- raw: parsed
477
- };
478
- return {
479
- lines: [JSON.stringify(parsed)],
480
- format: "json",
481
- raw: parsed
482
- };
483
- }
484
- readXlsx(filePath) {
485
- const content = fs.readFileSync(filePath, "utf-8");
486
- return {
487
- lines: content.split("\n").filter((line) => line.trim()),
488
- format: "xlsx",
489
- raw: content
490
- };
491
- }
492
- writeCsv(filePath, lines) {
493
- fs.writeFileSync(filePath, lines.join("\n"), "utf-8");
494
- }
495
- writeJson(filePath, lines) {
496
- const objects = lines.map((line) => {
497
- try {
498
- return JSON.parse(line);
499
- } catch {
500
- return line;
501
- }
502
- });
503
- fs.writeFileSync(filePath, JSON.stringify(objects, null, 2), "utf-8");
504
- }
505
- writeXlsx(filePath, lines) {
506
- fs.writeFileSync(filePath, lines.join("\n"), "utf-8");
507
- }
508
- };
509
- //#endregion
510
- //#region lib/types/reportGenerator.js
511
- var ReportGenerator = class {
512
- generateMaskingReport(params) {
513
- const findingsText = Object.keys(params.fieldTypeCounts).length > 0 ? Object.entries(params.fieldTypeCounts).map(([type, count]) => `发现 ${count} 个${type}类型的敏感数据`).join(";") : "无";
514
- return `【脱敏报告】
515
- - 处理文件:${params.inputPath}
516
- - 发现敏感字段:${findingsText}
517
- - 脱敏策略:${params.strategy}
518
- - 输出文件:${params.outputPath}
519
- - 状态:✅ 完成`;
520
- }
521
- generateCleaningReport(params) {
522
- let report = `【数据清洗报告】
523
- - 文件:${params.inputPath}
524
- - 原始行数:${params.originalLineCount}
525
- - 去重后行数:${params.originalLineCount - params.duplicateRemoved}(删除 ${params.duplicateRemoved} 行重复)
526
- `;
527
- if (params.standardizeApplied) report += "- 格式标准化:✅ 完成\n";
528
- report += `- 异常值检测:发现 ${params.anomalyTotalCount} 个异常\n`;
529
- if (params.anomalies.length > 0) {
530
- const anomalyText = params.anomalies.slice(0, 5).map((a) => `行${a.line}: ${a.reason}`).join(";");
531
- report += ` 异常详情:${anomalyText}`;
532
- if (params.anomalyTotalCount > 5) report += `...等${params.anomalyTotalCount}项`;
533
- report += "\n";
534
- }
535
- report += `- 输出文件:${params.outputPath}\n- 状态:✅ 完成`;
536
- return report;
537
- }
538
- generateInventoryReport(params) {
539
- let report = `【数据资产盘点报告】\n- 目录:${params.directory}\n- 发现数据文件:${params.fileCount} 个\n\n`;
540
- report += "| 文件名 | 大小 | 类型 | 价值评估 | 说明 |\n";
541
- report += "|--------|------|------|----------|------|\n";
542
- for (const item of params.assetItems) report += `| ${item.fileName} | ${item.size} | ${item.type} | ${item.valueAssessment} | ${item.description} |\n`;
543
- report += "\n---\n【建议】\n";
544
- report += "1. 高价值数据(★★★★★)建议优先进行清洗和脱敏后交易\n";
545
- report += "2. 低价值数据建议评估是否值得投入处理成本\n";
546
- report += "3. 所有数据交易前必须完成合规审查\n";
547
- return report;
548
- }
549
- generatePackagingManual(params) {
550
- const manual = params.productManual;
551
- const complianceText = manual.complianceStatements.map((stmt) => `✅ ${stmt}`).join("\n");
552
- return `========================================
553
- 数据产品说明书
554
- ========================================
555
-
556
- 【产品名称】${manual.productName}
557
- 【版本】${manual.version}
558
- 【生成日期】${manual.generatedDate}
559
-
560
- 【数据概览】
561
- - 源文件:${manual.dataOverview.sourceFile}
562
- - 数据量:${manual.dataOverview.recordCount} 条记录
563
- - 文件大小:${manual.dataOverview.fileSizeKb.toFixed(1)} KB
564
- - 格式:${manual.dataOverview.format}
565
-
566
- 【数据样例】(前5行)
567
- ${manual.sampleLines.join("\n")}
568
-
569
- 【使用场景】
570
- ${manual.usageScenario}
571
-
572
- 【合规声明】
573
- ${complianceText}
574
-
575
- 【定价建议】
576
- ${manual.pricingSuggestion}
577
-
578
- --- 本产品由数据资产化工具包自动生成 ---`;
579
- }
580
- };
581
- //#endregion
582
- //#region lib/types/pathValidator.js
583
- var PathValidator = class {
584
- validate(filePath, workingDir) {
585
- const resolved = path.resolve(filePath);
586
- if (filePath.includes("..")) throw new Error(`错误:路径不合法 - ${filePath}`);
587
- const relative = path.relative(workingDir, resolved);
588
- if (relative.startsWith("..") || path.isAbsolute(relative)) throw new Error(`错误:路径不合法 - ${filePath}`);
589
- return resolved;
590
- }
591
- validateDirectory(dirPath, workingDir) {
592
- return this.validate(dirPath, workingDir);
593
- }
594
- };
595
- //#endregion
596
- //#region lib/types/auditLogger.js
597
- var AuditLogger = class {
598
- logDir;
599
- constructor(logDir) {
600
- this.logDir = logDir ?? path.resolve(process.cwd(), "logs");
601
- }
602
- log(params) {
603
- const timestamp = (/* @__PURE__ */ new Date()).toISOString();
604
- const operator = params.operator ?? "system";
605
- const logLine = `[${params.pluginName}] ${timestamp} | 操作: ${params.operation} | 操作人: ${operator} | 输入: ${params.inputPath} | 输出: ${params.outputPath} | 结果: ${params.result}`;
606
- console.log(logLine);
607
- if (params.writeToFile) this.writeToFile(logLine);
608
- }
609
- writeToFile(logLine) {
610
- if (!fs.existsSync(this.logDir)) fs.mkdirSync(this.logDir, { recursive: true });
611
- const date = (/* @__PURE__ */ new Date()).toISOString().split("T")[0];
612
- const logFile = path.join(this.logDir, `audit-${date}.log`);
613
- fs.appendFileSync(logFile, logLine + "\n", "utf-8");
614
- }
615
- };
616
- //#endregion
617
- export { AuditLogger, BusinessRulesLoader, FileFormatAdapter, PathValidator, ReportGenerator, defaultAdvancedMaskingConfig, defaultBusinessRules, defaultIncrementalSchedulingConfig, defaultLineageConfig, defaultQualityScoringConfig, defaultSensitivityClassificationConfig, defaultVisualizationConfig };
1
+ export * from './types.js';
2
+ export { defaultBusinessRules } from './defaultBusinessRules.js';
3
+ export { defaultQualityScoringConfig, defaultLineageConfig, defaultAdvancedMaskingConfig, defaultVisualizationConfig, defaultSensitivityClassificationConfig, defaultIncrementalSchedulingConfig, } from './defaultBusinessRules.js';
4
+ export { BusinessRulesLoader } from './businessRulesLoader.js';
5
+ export { FileFormatAdapter } from './fileFormatAdapter.js';
6
+ export { ReportGenerator } from './reportGenerator.js';
7
+ export { PathValidator } from './pathValidator.js';
8
+ export { AuditLogger } from './auditLogger.js';
9
+ //# sourceMappingURL=index.js.map
package/lib/invariant.js CHANGED
@@ -1,5 +1,4 @@
1
- //#region lib/types/invariant.js
2
- const invariant = "data-asset-shared";
3
- function install() {}
4
- //#endregion
5
- export { install, invariant };
1
+ export const invariant = 'data-asset-shared';
2
+ export function install() {
3
+ }
4
+ //# sourceMappingURL=invariant.js.map
@@ -0,0 +1,18 @@
1
+ import * as path from 'path';
2
+ export class PathValidator {
3
+ validate(filePath, workingDir) {
4
+ const resolved = path.resolve(filePath);
5
+ if (filePath.includes('..')) {
6
+ throw new Error(`错误:路径不合法 - ${filePath}`);
7
+ }
8
+ const relative = path.relative(workingDir, resolved);
9
+ if (relative.startsWith('..') || path.isAbsolute(relative)) {
10
+ throw new Error(`错误:路径不合法 - ${filePath}`);
11
+ }
12
+ return resolved;
13
+ }
14
+ validateDirectory(dirPath, workingDir) {
15
+ return this.validate(dirPath, workingDir);
16
+ }
17
+ }
18
+ //# sourceMappingURL=pathValidator.js.map
@@ -0,0 +1,84 @@
1
+ export class ReportGenerator {
2
+ generateMaskingReport(params) {
3
+ const findingsText = Object.keys(params.fieldTypeCounts).length > 0
4
+ ? Object.entries(params.fieldTypeCounts)
5
+ .map(([type, count]) => `发现 ${count} 个${type}类型的敏感数据`)
6
+ .join(';')
7
+ : '无';
8
+ return `【脱敏报告】
9
+ - 处理文件:${params.inputPath}
10
+ - 发现敏感字段:${findingsText}
11
+ - 脱敏策略:${params.strategy}
12
+ - 输出文件:${params.outputPath}
13
+ - 状态:✅ 完成`;
14
+ }
15
+ generateCleaningReport(params) {
16
+ let report = `【数据清洗报告】
17
+ - 文件:${params.inputPath}
18
+ - 原始行数:${params.originalLineCount}
19
+ - 去重后行数:${params.originalLineCount - params.duplicateRemoved}(删除 ${params.duplicateRemoved} 行重复)
20
+ `;
21
+ if (params.standardizeApplied) {
22
+ report += '- 格式标准化:✅ 完成\n';
23
+ }
24
+ report += `- 异常值检测:发现 ${params.anomalyTotalCount} 个异常\n`;
25
+ if (params.anomalies.length > 0) {
26
+ const displayAnomalies = params.anomalies.slice(0, 5);
27
+ const anomalyText = displayAnomalies.map(a => `行${a.line}: ${a.reason}`).join(';');
28
+ report += ` 异常详情:${anomalyText}`;
29
+ if (params.anomalyTotalCount > 5) {
30
+ report += `...等${params.anomalyTotalCount}项`;
31
+ }
32
+ report += '\n';
33
+ }
34
+ report += `- 输出文件:${params.outputPath}\n- 状态:✅ 完成`;
35
+ return report;
36
+ }
37
+ generateInventoryReport(params) {
38
+ let report = `【数据资产盘点报告】\n- 目录:${params.directory}\n- 发现数据文件:${params.fileCount} 个\n\n`;
39
+ report += '| 文件名 | 大小 | 类型 | 价值评估 | 说明 |\n';
40
+ report += '|--------|------|------|----------|------|\n';
41
+ for (const item of params.assetItems) {
42
+ report += `| ${item.fileName} | ${item.size} | ${item.type} | ${item.valueAssessment} | ${item.description} |\n`;
43
+ }
44
+ report += '\n---\n【建议】\n';
45
+ report += '1. 高价值数据(★★★★★)建议优先进行清洗和脱敏后交易\n';
46
+ report += '2. 低价值数据建议评估是否值得投入处理成本\n';
47
+ report += '3. 所有数据交易前必须完成合规审查\n';
48
+ return report;
49
+ }
50
+ generatePackagingManual(params) {
51
+ const manual = params.productManual;
52
+ const complianceText = manual.complianceStatements
53
+ .map(stmt => `✅ ${stmt}`)
54
+ .join('\n');
55
+ return `========================================
56
+ 数据产品说明书
57
+ ========================================
58
+
59
+ 【产品名称】${manual.productName}
60
+ 【版本】${manual.version}
61
+ 【生成日期】${manual.generatedDate}
62
+
63
+ 【数据概览】
64
+ - 源文件:${manual.dataOverview.sourceFile}
65
+ - 数据量:${manual.dataOverview.recordCount} 条记录
66
+ - 文件大小:${manual.dataOverview.fileSizeKb.toFixed(1)} KB
67
+ - 格式:${manual.dataOverview.format}
68
+
69
+ 【数据样例】(前5行)
70
+ ${manual.sampleLines.join('\n')}
71
+
72
+ 【使用场景】
73
+ ${manual.usageScenario}
74
+
75
+ 【合规声明】
76
+ ${complianceText}
77
+
78
+ 【定价建议】
79
+ ${manual.pricingSuggestion}
80
+
81
+ --- 本产品由数据资产化工具包自动生成 ---`;
82
+ }
83
+ }
84
+ //# sourceMappingURL=reportGenerator.js.map
package/lib/types.js ADDED
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=types.js.map
package/package.json CHANGED
@@ -1,13 +1,13 @@
1
1
  {
2
2
  "name": "@liuhange/dsh-data-asset-shared",
3
3
  "description": "Shared infrastructure for data assetization plugins: business rules loader, file format adapter, report generator, path validator, audit logger",
4
- "version": "2.0.1",
4
+ "version": "2.0.3",
5
5
  "publishConfig": {
6
6
  "access": "public"
7
7
  },
8
8
  "repository": {
9
9
  "type": "git",
10
- "url": "git+https://github.com/deepseek-ai/deepseek-harness.git",
10
+ "url": "git+https://github.com/liuhange789/data-asset-inspector.git",
11
11
  "directory": "packages/data-asset/data-asset-shared"
12
12
  },
13
13
  "type": "module",
@@ -26,17 +26,20 @@
26
26
  "./package.json": "./package.json"
27
27
  },
28
28
  "files": [
29
- "lib/index.js",
30
- "lib/invariant.js",
31
- "lib/types/**/*.d.ts"
29
+ "lib/**/*.js", "lib/types/**/*.d.ts"
32
30
  ],
33
31
  "license": "MIT",
32
+ "scripts": {
33
+ "typecheck": "tsc --noEmit",
34
+ "test": "vitest run",
35
+ "build": "tsc"
36
+ },
34
37
  "dependencies": {
35
38
  "csv-parse": "^5.5.0"
36
39
  },
37
40
  "peerDependencies": {
38
- "@deepseek-ai/cordis": "^0.1.0",
39
- "@deepseek-ai/dsh-tools": "^0.1.0",
40
- "@deepseek-ai/schemastery": "^0.1.0"
41
+ "@deepseek-ai/cordis": "^4.0.0",
42
+ "@deepseek-ai/dsh-tools": "0.0.1-rc.1",
43
+ "@deepseek-ai/schemastery": "^3.18.0"
41
44
  }
42
45
  }