@liuhange/dsh-data-asset-shared 1.0.0 → 2.0.1

package/lib/index.js

@@ -1,6 +1,208 @@
 import * as fs from "fs";
 import * as path from "path";
 //#region lib/types/defaultBusinessRules.js
+const defaultQualityScoringConfig = {
+	weights: {
+		completeness: .25,
+		accuracy: .25,
+		consistency: .25,
+		timeliness: .25
+	},
+	thresholds: {
+		completeness: 60,
+		accuracy: 60,
+		consistency: 60,
+		timeliness: 60
+	},
+	completeness: { missingMarkers: [
+		"null",
+		"undefined",
+		"N/A",
+		"",
+		"NULL",
+		"NaN"
+	] },
+	accuracy: {
+		formatRules: [{
+			fieldName: "phone",
+			pattern: "1[3-9]\\d{9}",
+			weight: .5
+		}, {
+			fieldName: "email",
+			pattern: "[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}",
+			weight: .5
+		}],
+		domainRules: [{
+			fieldName: "age",
+			min: 0,
+			max: 150,
+			weight: .5
+		}, {
+			fieldName: "score",
+			min: 0,
+			max: 100,
+			weight: .5
+		}]
+	},
+	consistency: { crossFieldRules: [{
+		name: "endDateAfterStartDate",
+		fields: ["startDate", "endDate"],
+		constraint: "after"
+	}] },
+	timeliness: {
+		timestampField: "updatedAt",
+		freshnessThresholdHours: 720
+	}
+};
+const defaultLineageConfig = {
+	enabled: false,
+	storagePath: "lineage",
+	hashAlgorithm: "SHA-256"
+};
+const defaultAdvancedMaskingConfig = {
+	defaultAlgorithm: "hash",
+	fieldAlgorithms: {},
+	fpe: {
+		key: "",
+		radix: 10
+	},
+	kAnonymity: {
+		kValue: 2,
+		quasiIdentifiers: [
+			"zipCode",
+			"age",
+			"gender"
+		]
+	},
+	differentialPrivacy: {
+		epsilon: 1,
+		totalBudget: 5,
+		sensitivity: 1
+	},
+	hash: {
+		algorithm: "SHA-256",
+		salt: ""
+	}
+};
+const defaultVisualizationConfig = {
+	charts: {
+		bar: {
+			enabled: true,
+			title: "脱敏统计"
+		},
+		comparison: {
+			enabled: true,
+			title: "清洗前后对比"
+		},
+		pie: {
+			enabled: true,
+			title: "数据资产分布"
+		},
+		radar: {
+			enabled: true,
+			title: "质量评分"
+		},
+		flowchart: {
+			enabled: true,
+			title: "血缘追踪"
+		}
+	},
+	layout: {
+		columns: 2,
+		responsive: true
+	},
+	colorScheme: {
+		primary: "#4A90D9",
+		secondary: "#F5A623",
+		success: "#7ED321",
+		warning: "#F8E71C",
+		danger: "#D0021B",
+		background: "#F5F5F5",
+		text: "#333333"
+	},
+	interactions: {
+		expandCollapse: true,
+		chartSwitch: true,
+		tooltip: true
+	},
+	templateVersion: "1.0"
+};
+const defaultSensitivityClassificationConfig = {
+	fieldNameRules: [
+		{
+			name: "idCard",
+			pattern: "id_?card|身份证|identity",
+			level: "Secret"
+		},
+		{
+			name: "phone",
+			pattern: "phone|手机|mobile|tel",
+			level: "Confidential"
+		},
+		{
+			name: "email",
+			pattern: "email|邮箱|mail",
+			level: "Confidential"
+		},
+		{
+			name: "bankCard",
+			pattern: "bank_?card|银行卡|account",
+			level: "Secret"
+		},
+		{
+			name: "address",
+			pattern: "address|地址|addr",
+			level: "Internal"
+		},
+		{
+			name: "name",
+			pattern: "^name$|姓名|username",
+			level: "Internal"
+		},
+		{
+			name: "public",
+			pattern: "status|type|category|label|tag|level",
+			level: "Public"
+		}
+	],
+	fieldValueRules: [
+		{
+			name: "idCardValue",
+			pattern: "[1-9]\\d{5}(18|19|20)?\\d{2}(0[1-9]|1[0-2])(0[1-9]|[12]\\d|3[01])\\d{3}[\\dXx]",
+			level: "Secret"
+		},
+		{
+			name: "phoneValue",
+			pattern: "1[3-9]\\d{9}",
+			level: "Confidential"
+		},
+		{
+			name: "emailValue",
+			pattern: "[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}",
+			level: "Confidential"
+		},
+		{
+			name: "bankCardValue",
+			pattern: "[1-9]\\d{15,18}",
+			level: "Secret"
+		}
+	],
+	levelMapping: {
+		Public: "none",
+		Internal: "partial",
+		Confidential: "full",
+		Secret: "encrypt"
+	},
+	defaultLevel: "Internal"
+};
+const defaultIncrementalSchedulingConfig = {
+	incrementalMode: false,
+	cronExpression: "0 2 * * *",
+	hashAlgorithm: "SHA-256",
+	watchDirectory: "input",
+	stateFilePath: "state/incremental-state.json",
+	lockTimeout: 3600
+};
 const defaultBusinessRules = {
 	version: "1.0",
 	lastUpdated: "2026-08-17",
@@ -322,26 +524,26 @@ var ReportGenerator = class {
 - 原始行数：${params.originalLineCount}
 - 去重后行数：${params.originalLineCount - params.duplicateRemoved}（删除 ${params.duplicateRemoved} 行重复）
 `;
-		if (params.standardizeApplied) report += `- 格式标准化：✅ 完成\n`;
+		if (params.standardizeApplied) report += "- 格式标准化：✅ 完成\n";
 		report += `- 异常值检测：发现 ${params.anomalyTotalCount} 个异常\n`;
 		if (params.anomalies.length > 0) {
 			const anomalyText = params.anomalies.slice(0, 5).map((a) => `行${a.line}: ${a.reason}`).join("；");
 			report += `  异常详情：${anomalyText}`;
 			if (params.anomalyTotalCount > 5) report += `...等${params.anomalyTotalCount}项`;
-			report += `\n`;
+			report += "\n";
 		}
 		report += `- 输出文件：${params.outputPath}\n- 状态：✅ 完成`;
 		return report;
 	}
 	generateInventoryReport(params) {
 		let report = `【数据资产盘点报告】\n- 目录：${params.directory}\n- 发现数据文件：${params.fileCount} 个\n\n`;
-		report += `| 文件名 | 大小 | 类型 | 价值评估 | 说明 |\n`;
-		report += `|--------|------|------|----------|------|\n`;
+		report += "| 文件名 | 大小 | 类型 | 价值评估 | 说明 |\n";
+		report += "|--------|------|------|----------|------|\n";
 		for (const item of params.assetItems) report += `| ${item.fileName} | ${item.size} | ${item.type} | ${item.valueAssessment} | ${item.description} |\n`;
-		report += `\n---\n【建议】\n`;
-		report += `1. 高价值数据（★★★★★）建议优先进行清洗和脱敏后交易\n`;
-		report += `2. 低价值数据建议评估是否值得投入处理成本\n`;
-		report += `3. 所有数据交易前必须完成合规审查\n`;
+		report += "\n---\n【建议】\n";
+		report += "1. 高价值数据（★★★★★）建议优先进行清洗和脱敏后交易\n";
+		report += "2. 低价值数据建议评估是否值得投入处理成本\n";
+		report += "3. 所有数据交易前必须完成合规审查\n";
 		return report;
 	}
 	generatePackagingManual(params) {
@@ -412,4 +614,4 @@ var AuditLogger = class {
 	}
 };
 //#endregion
-export { AuditLogger, BusinessRulesLoader, FileFormatAdapter, PathValidator, ReportGenerator, defaultBusinessRules };
+export { AuditLogger, BusinessRulesLoader, FileFormatAdapter, PathValidator, ReportGenerator, defaultAdvancedMaskingConfig, defaultBusinessRules, defaultIncrementalSchedulingConfig, defaultLineageConfig, defaultQualityScoringConfig, defaultSensitivityClassificationConfig, defaultVisualizationConfig };

package/lib/types/defaultBusinessRules.d.ts

@@ -1,3 +1,140 @@
 import type { BusinessRulesConfig } from './types.js';
+export declare const defaultQualityScoringConfig: {
+    weights: {
+        completeness: number;
+        accuracy: number;
+        consistency: number;
+        timeliness: number;
+    };
+    thresholds: {
+        completeness: number;
+        accuracy: number;
+        consistency: number;
+        timeliness: number;
+    };
+    completeness: {
+        missingMarkers: string[];
+    };
+    accuracy: {
+        formatRules: {
+            fieldName: string;
+            pattern: string;
+            weight: number;
+        }[];
+        domainRules: {
+            fieldName: string;
+            min: number;
+            max: number;
+            weight: number;
+        }[];
+    };
+    consistency: {
+        crossFieldRules: {
+            name: string;
+            fields: string[];
+            constraint: string;
+        }[];
+    };
+    timeliness: {
+        timestampField: string;
+        freshnessThresholdHours: number;
+    };
+};
+export declare const defaultLineageConfig: {
+    enabled: boolean;
+    storagePath: string;
+    hashAlgorithm: string;
+};
+export declare const defaultAdvancedMaskingConfig: {
+    defaultAlgorithm: string;
+    fieldAlgorithms: {};
+    fpe: {
+        key: string;
+        radix: number;
+    };
+    kAnonymity: {
+        kValue: number;
+        quasiIdentifiers: string[];
+    };
+    differentialPrivacy: {
+        epsilon: number;
+        totalBudget: number;
+        sensitivity: number;
+    };
+    hash: {
+        algorithm: string;
+        salt: string;
+    };
+};
+export declare const defaultVisualizationConfig: {
+    charts: {
+        bar: {
+            enabled: boolean;
+            title: string;
+        };
+        comparison: {
+            enabled: boolean;
+            title: string;
+        };
+        pie: {
+            enabled: boolean;
+            title: string;
+        };
+        radar: {
+            enabled: boolean;
+            title: string;
+        };
+        flowchart: {
+            enabled: boolean;
+            title: string;
+        };
+    };
+    layout: {
+        columns: number;
+        responsive: boolean;
+    };
+    colorScheme: {
+        primary: string;
+        secondary: string;
+        success: string;
+        warning: string;
+        danger: string;
+        background: string;
+        text: string;
+    };
+    interactions: {
+        expandCollapse: boolean;
+        chartSwitch: boolean;
+        tooltip: boolean;
+    };
+    templateVersion: string;
+};
+export declare const defaultSensitivityClassificationConfig: {
+    fieldNameRules: {
+        name: string;
+        pattern: string;
+        level: string;
+    }[];
+    fieldValueRules: {
+        name: string;
+        pattern: string;
+        level: string;
+    }[];
+    levelMapping: {
+        Public: string;
+        Internal: string;
+        Confidential: string;
+        Secret: string;
+    };
+    defaultLevel: string;
+};
+export declare const defaultIncrementalSchedulingConfig: {
+    incrementalMode: boolean;
+    cronExpression: string;
+    hashAlgorithm: string;
+    watchDirectory: string;
+    stateFilePath: string;
+    lockTimeout: number;
+};
 export declare const defaultBusinessRules: BusinessRulesConfig;
 //# sourceMappingURL=defaultBusinessRules.d.ts.map

package/lib/types/index.d.ts

@@ -1,5 +1,6 @@
 export * from './types.js';
 export { defaultBusinessRules } from './defaultBusinessRules.js';
+export { defaultQualityScoringConfig, defaultLineageConfig, defaultAdvancedMaskingConfig, defaultVisualizationConfig, defaultSensitivityClassificationConfig, defaultIncrementalSchedulingConfig, } from './defaultBusinessRules.js';
 export { BusinessRulesLoader } from './businessRulesLoader.js';
 export { FileFormatAdapter } from './fileFormatAdapter.js';
 export type { FileReadResult } from './fileFormatAdapter.js';

package/lib/types/types.d.ts

@@ -76,6 +76,12 @@ export interface BusinessRulesConfig {
     cleaningRules: CleaningRules;
     compliance: ComplianceConfig;
     packaging: PackagingRules;
+    qualityScoring?: QualityScoringConfig;
+    lineage?: LineageConfig;
+    advancedMasking?: AdvancedMaskingConfig;
+    visualization?: VisualizationConfig;
+    sensitivityClassification?: SensitivityClassificationConfig;
+    incrementalScheduling?: IncrementalSchedulingConfig;
 }
 export interface SensitiveField {
     type: SensitiveFieldType;
@@ -132,4 +138,141 @@ export interface BusinessRulesLoadResult {
     config: BusinessRulesConfig;
     status: ConfigStatus;
 }
+export interface QualityScoringWeights {
+    completeness: number;
+    accuracy: number;
+    consistency: number;
+    timeliness: number;
+}
+export interface QualityScoringThresholds {
+    completeness: number;
+    accuracy: number;
+    consistency: number;
+    timeliness: number;
+}
+export interface CompletenessConfig {
+    missingMarkers: string[];
+}
+export interface FormatRule {
+    fieldName: string;
+    pattern: string;
+    weight: number;
+}
+export interface DomainRule {
+    fieldName: string;
+    min?: number;
+    max?: number;
+    allowedValues?: unknown[];
+    weight: number;
+}
+export interface AccuracyConfig {
+    formatRules: FormatRule[];
+    domainRules: DomainRule[];
+}
+export interface CrossFieldRule {
+    name: string;
+    fields: string[];
+    constraint: string;
+    value?: unknown;
+}
+export interface ConsistencyConfig {
+    crossFieldRules: CrossFieldRule[];
+}
+export interface TimelinessConfig {
+    timestampField: string;
+    freshnessThresholdHours: number;
+}
+export interface QualityScoringConfig {
+    weights: QualityScoringWeights;
+    thresholds: QualityScoringThresholds;
+    completeness: CompletenessConfig;
+    accuracy: AccuracyConfig;
+    consistency: ConsistencyConfig;
+    timeliness: TimelinessConfig;
+}
+export interface LineageConfig {
+    enabled: boolean;
+    storagePath: string;
+    hashAlgorithm: string;
+}
+export interface FpeConfig {
+    key: string;
+    radix: number;
+}
+export interface KAnonymityConfig {
+    kValue: number;
+    quasiIdentifiers: string[];
+}
+export interface DifferentialPrivacyConfig {
+    epsilon: number;
+    totalBudget: number;
+    sensitivity: number;
+}
+export interface HashMaskingConfig {
+    algorithm: string;
+    salt: string;
+}
+export interface AdvancedMaskingConfig {
+    defaultAlgorithm: string;
+    fieldAlgorithms: Record<string, {
+        algorithm: string;
+    }>;
+    fpe: FpeConfig;
+    kAnonymity: KAnonymityConfig;
+    differentialPrivacy: DifferentialPrivacyConfig;
+    hash: HashMaskingConfig;
+}
+export interface ChartConfig {
+    enabled: boolean;
+    title: string;
+}
+export interface VisualizationLayoutConfig {
+    columns: number;
+    responsive: boolean;
+}
+export interface ColorSchemeConfig {
+    primary: string;
+    secondary: string;
+    success: string;
+    warning: string;
+    danger: string;
+    background: string;
+    text: string;
+}
+export interface InteractionsConfig {
+    expandCollapse: boolean;
+    chartSwitch: boolean;
+    tooltip: boolean;
+}
+export interface VisualizationConfig {
+    charts: Record<string, ChartConfig>;
+    layout: VisualizationLayoutConfig;
+    colorScheme: ColorSchemeConfig;
+    interactions: InteractionsConfig;
+    templateVersion: string;
+}
+export interface FieldNameRule {
+    name: string;
+    pattern: string;
+    level: string;
+}
+export interface FieldValueRule {
+    name: string;
+    pattern: string;
+    level: string;
+}
+export interface SensitivityClassificationConfig {
+    fieldNameRules: FieldNameRule[];
+    fieldValueRules: FieldValueRule[];
+    levelMapping: Record<string, string>;
+    defaultLevel: string;
+}
+export interface IncrementalSchedulingConfig {
+    incrementalMode: boolean;
+    cronExpression: string;
+    hashAlgorithm: string;
+    watchDirectory: string;
+    stateFilePath: string;
+    lockTimeout: number;
+}
 //# sourceMappingURL=types.d.ts.map

package/package.json

@@ -1,42 +1,42 @@
 {
-  "name": "@liuhange/dsh-data-asset-shared",
-  "description": "Shared infrastructure for data assetization plugins: business rules loader, file format adapter, report generator, path validator, audit logger",
-  "version": "1.0.0",
-  "publishConfig": {
-    "access": "public"
-  },
-  "repository": {
-    "type": "git",
-    "url": "git+https://github.com/deepseek-ai/deepseek-harness.git",
-    "directory": "packages/data-asset/data-asset-shared"
-  },
-  "type": "module",
-  "main": "lib/index.js",
-  "types": "lib/types/index.d.ts",
-  "exports": {
-    ".": {
-      "types": "./lib/types/index.d.ts",
-      "default": "./lib/index.js"
-    },
-    "./invariant": {
-      "types": "./lib/types/invariant.d.ts",
-      "default": "./lib/invariant.js"
-    },
-    "./src/*": "./src/*",
-    "./package.json": "./package.json"
-  },
-  "files": [
-    "lib/index.js",
-    "lib/invariant.js",
-    "lib/types/**/*.d.ts"
-  ],
-  "license": "MIT",
-  "dependencies": {
-    "csv-parse": "^5.5.0"
-  },
-  "peerDependencies": {
-    "@deepseek-ai/cordis": "^0.1.0",
-    "@deepseek-ai/dsh-tools": "^0.1.0",
-    "@deepseek-ai/schemastery": "^0.1.0"
-  }
+    "name":  "@liuhange/dsh-data-asset-shared",
+    "description":  "Shared infrastructure for data assetization plugins: business rules loader, file format adapter, report generator, path validator, audit logger",
+    "version": "2.0.1",
+    "publishConfig":  {
+                          "access":  "public"
+                      },
+    "repository":  {
+                       "type":  "git",
+                       "url":  "git+https://github.com/deepseek-ai/deepseek-harness.git",
+                       "directory":  "packages/data-asset/data-asset-shared"
+                   },
+    "type":  "module",
+    "main":  "lib/index.js",
+    "types":  "lib/types/index.d.ts",
+    "exports":  {
+                    ".":  {
+                              "types":  "./lib/types/index.d.ts",
+                              "default":  "./lib/index.js"
+                          },
+                    "./invariant":  {
+                                        "types":  "./lib/types/invariant.d.ts",
+                                        "default":  "./lib/invariant.js"
+                                    },
+                    "./src/*":  "./src/*",
+                    "./package.json":  "./package.json"
+                },
+    "files":  [
+                  "lib/index.js",
+                  "lib/invariant.js",
+                  "lib/types/**/*.d.ts"
+              ],
+    "license":  "MIT",
+    "dependencies":  {
+                         "csv-parse":  "^5.5.0"
+                     },
+    "peerDependencies":  {
+                             "@deepseek-ai/cordis":  "^0.1.0",
+                             "@deepseek-ai/dsh-tools":  "^0.1.0",
+                             "@deepseek-ai/schemastery":  "^0.1.0"
+                         }
 }