@liuhange/dsh-ai-dataset-inspector 3.1.0

package/README.md

@@ -0,0 +1,22 @@
+# @liuhange/dsh-ai-dataset-inspector
+
+## 功能说明
+
+AI training dataset inspection plugin: annotation consistency, label distribution, data leakage detection
+
+## 安装
+
+```bash
+dsh plugin --profile web add @liuhange/dsh-ai-dataset-inspector
+```
+
+## 用法
+
+```javascript
+import { apply } from '@liuhange/dsh-ai-dataset-inspector';
+apply(ctx); // 注册工具: inspect_ai_dataset
+```
+
+## License
+
+MIT

package/lib/index.js

@@ -0,0 +1,117 @@
+export const name = '@liuhange/dsh-ai-dataset-inspector';
+export const inject = ['tools'];
+import { loadJsonConfig } from '@liuhange/dsh-data-asset-shared';
+import { resolvePolicyBasis } from './policyBasis.js';
+export function apply(ctx) {
+    ctx.tools.register({
+        name: 'inspect_ai_dataset',
+        description: '检查AI训练数据集质量，涵盖标注一致性、标签均衡性、数据泄露等维度',
+        parameters: {
+            type: 'object',
+            properties: {
+                datasetPath: { type: 'string', description: '数据集文件路径' },
+                datasetType: { type: 'string', enum: ['training', 'validation', 'test', 'full'] },
+                maskingResultRef: { type: 'string', description: 'data-masking工具输出引用（可选）' },
+            },
+            required: ['datasetPath'],
+        },
+        async execute(args) {
+            try {
+                if (!args?.datasetPath || typeof args.datasetPath !== 'string') {
+                    return JSON.stringify({ error: 'AI_DATASET_INPUT_INVALID', message: 'datasetPath is required' });
+                }
+                let rulesConfig;
+                try {
+                    rulesConfig = loadJsonConfig('BUSINESS_RULES_PATH', 'config/business-rules.json', '@liuhange/dsh-data-asset-shared/config/business-rules.json');
+                }
+                catch {
+                    return JSON.stringify({ error: 'AI_DATASET_RULES_MISSING', message: '无法加载business-rules.json' });
+                }
+                const aiConfig = rulesConfig.aiDatasetInspection;
+                if (!aiConfig) {
+                    return JSON.stringify({ error: 'AI_DATASET_RULES_MISSING', message: 'aiDatasetInspection配置段缺失' });
+                }
+                let dataset;
+                try {
+                    const { readFileSync } = await import('node:fs');
+                    const raw = readFileSync(args.datasetPath, 'utf-8');
+                    dataset = JSON.parse(raw);
+                    if (!Array.isArray(dataset))
+                        dataset = [dataset];
+                }
+                catch {
+                    return JSON.stringify({ error: 'AI_DATASET_INPUT_INVALID', message: `无法加载数据集: ${args.datasetPath}` });
+                }
+                const annotationConsistency = computeKappa(dataset, aiConfig.kappaThreshold);
+                const labelDistribution = analyzeLabelDistribution(dataset);
+                const dataLeakage = detectLeakage(dataset);
+                const policyBasis = resolvePolicyBasis('AI_DATASET_INSPECTION');
+                const result = {
+                    datasetId: args.datasetPath,
+                    inspectionType: args.datasetType ?? 'full',
+                    annotationConsistency,
+                    labelDistribution,
+                    dataLeakage,
+                    piiResidue: args.maskingResultRef ? '已引用data-masking结果检测PII残留' : '未提供masking结果引用',
+                    cleaningStatus: '建议配合data-cleaning做预处理质量检查',
+                    policyBasis,
+                    timestamp: new Date().toISOString(),
+                };
+                return JSON.stringify(result, null, 2);
+            }
+            catch (e) {
+                return JSON.stringify({ error: 'UNKNOWN_ERROR', message: e.message });
+            }
+        },
+    });
+}
+function computeKappa(dataset, threshold) {
+    let agreed = 0;
+    let disputed = 0;
+    for (const row of dataset) {
+        if (typeof row === 'object' && row !== null) {
+            const obj = row;
+            if (obj.annotation1 !== undefined && obj.annotation2 !== undefined) {
+                if (obj.annotation1 === obj.annotation2)
+                    agreed++;
+                else
+                    disputed++;
+            }
+        }
+    }
+    const total = agreed + disputed;
+    const kappa = total > 0 ? agreed / total : 1;
+    return { kappaCoefficient: Math.round(kappa * 100) / 100, disputedSamples: disputed, passed: kappa >= threshold };
+}
+function analyzeLabelDistribution(dataset) {
+    const distribution = {};
+    for (const row of dataset) {
+        if (typeof row === 'object' && row !== null) {
+            const label = row.label;
+            if (label)
+                distribution[label] = (distribution[label] ?? 0) + 1;
+        }
+    }
+    const counts = Object.values(distribution);
+    const max = counts.length > 0 ? Math.max(...counts) : 0;
+    const min = counts.length > 0 ? Math.min(...counts) : 0;
+    const ratio = min > 0 ? max / min : Infinity;
+    return { isBalanced: ratio <= 2, imbalanceRatio: Math.round(ratio * 100) / 100, classDistribution: distribution };
+}
+function detectLeakage(dataset) {
+    const ids = new Set();
+    let overlapping = 0;
+    for (const row of dataset) {
+        if (typeof row === 'object' && row !== null) {
+            const id = row.id;
+            if (id) {
+                if (ids.has(id))
+                    overlapping++;
+                else
+                    ids.add(id);
+            }
+        }
+    }
+    return { detected: overlapping > 0, leakageType: overlapping > 0 ? 'train_val' : 'none', overlappingSamples: overlapping };
+}
+//# sourceMappingURL=index.js.map

package/lib/policyBasis.js

@@ -0,0 +1,17 @@
+import { loadJsonConfig } from '@liuhange/dsh-data-asset-shared';
+export function resolvePolicyBasis(stage) {
+    try {
+        const config = loadJsonConfig('POLICY_REFS_PATH', 'config/policy-references.json', '@liuhange/dsh-data-asset-shared/config/policy-references.json');
+        const refs = config.policyReferences;
+        const stageEntry = refs?.find((s) => s.stage === stage);
+        if (!stageEntry) {
+            console.warn(`Policy stage "${stage}" not configured`);
+            return [`依据：政策依据未配置（stage=${stage}）`];
+        }
+        return stageEntry.documents.map((d) => `依据：《${d.name}》（${d.docNumber}）—${d.coreRequirement}`);
+    }
+    catch {
+        return ['依据：政策依据配置加载失败'];
+    }
+}
+//# sourceMappingURL=policyBasis.js.map

package/package.json

@@ -0,0 +1,60 @@
+{
+  "name": "@liuhange/dsh-ai-dataset-inspector",
+  "description": "AI training dataset inspection plugin: annotation consistency, label distribution, data leakage detection",
+  "version": "3.1.0",
+  "publishConfig": {
+    "access": "public"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/liuhange789/data-asset-inspector.git",
+    "directory": "packages/data-asset/ai-dataset-inspector"
+  },
+  "type": "module",
+  "main": "lib/index.js",
+  "types": "lib/types/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./lib/types/index.d.ts",
+      "default": "./lib/index.js"
+    },
+    "./src/*": "./src/*",
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "lib/**/*.js",
+    "lib/types/**/*.d.ts"
+  ],
+  "license": "MIT",
+  "scripts": {
+    "typecheck": "tsc --noEmit",
+    "test": "vitest run",
+    "build": "tsc"
+  },
+  "peerDependencies": {
+    "@deepseek-ai/cordis": "^4.0.0",
+    "@deepseek-ai/dsh-tools": "0.0.1-rc.1",
+    "@liuhange/dsh-data-asset-shared": "^3.0.0"
+  },
+  "dsh": {
+    "bundle": {
+      "patch": true
+    }
+  },
+  "keywords": [
+    "deepseek-harness",
+    "dsh-plugin",
+    "data-asset",
+    "data-asset-inspector",
+    "shujintong",
+    "数据资产",
+    "ai-dataset",
+    "annotation-consistency",
+    "data-leakage",
+    "kappa-coefficient",
+    "AI数据集",
+    "标注一致性",
+    "数据泄露"
+  ],
+  "homepage": "https://github.com/liuhange789/data-asset-inspector#readme"
+}