@skillsmith/core 0.1.2 → 2.0.0

package/dist/src/scripts/validation/deduplication.js

@@ -0,0 +1,127 @@
+/**
+ * SMI-863: Deduplication functions for skill validation
+ */
+import { EmbeddingService } from '../../embeddings/index.js';
+import { CONFIG } from './types.js';
+import { hashRepoUrl } from './normalizers.js';
+/**
+ * Compare two skills and determine which to keep based on source priority and quality
+ */
+export function compareSkillsForDedup(a, b) {
+    const priorityA = CONFIG.SOURCE_PRIORITY[a.source] ?? CONFIG.SOURCE_PRIORITY.unknown;
+    const priorityB = CONFIG.SOURCE_PRIORITY[b.source] ?? CONFIG.SOURCE_PRIORITY.unknown;
+    // Higher source priority wins
+    if (priorityA !== priorityB) {
+        return priorityA > priorityB ? 'a' : 'b';
+    }
+    // Same source priority - higher quality score wins
+    return a.quality_score >= b.quality_score ? 'a' : 'b';
+}
+/**
+ * Deduplicate skills by repo_url
+ */
+export function deduplicateByRepoUrl(skills) {
+    const seen = new Map();
+    const duplicates = [];
+    for (const skill of skills) {
+        if (!skill.repo_url) {
+            // Skills without repo_url are kept (will be checked for semantic similarity)
+            // Use ID as key for skills without repo_url
+            const key = `no-url:${skill.id}`;
+            if (!seen.has(key)) {
+                seen.set(key, skill);
+            }
+            continue;
+        }
+        const key = hashRepoUrl(skill.repo_url);
+        const existing = seen.get(key);
+        if (existing) {
+            const winner = compareSkillsForDedup(existing, skill);
+            if (winner === 'b') {
+                duplicates.push({
+                    kept: skill,
+                    discarded: existing,
+                    reason: 'repo_url',
+                });
+                seen.set(key, skill);
+            }
+            else {
+                duplicates.push({
+                    kept: existing,
+                    discarded: skill,
+                    reason: 'repo_url',
+                });
+            }
+        }
+        else {
+            seen.set(key, skill);
+        }
+    }
+    return {
+        unique: Array.from(seen.values()),
+        duplicates,
+    };
+}
+/**
+ * Detect semantic duplicates using embedding similarity
+ */
+export async function detectSemanticDuplicates(skills, threshold = CONFIG.SIMILARITY_THRESHOLD) {
+    if (skills.length === 0) {
+        return { unique: [], duplicates: [] };
+    }
+    // Use fallback mode for faster processing
+    const embeddingService = new EmbeddingService({ useFallback: true });
+    const duplicates = [];
+    const unique = [];
+    const embeddings = new Map();
+    try {
+        // Generate embeddings for all skills
+        for (const skill of skills) {
+            const text = `${skill.name} ${skill.description}`;
+            const embedding = await embeddingService.embed(text);
+            embeddings.set(skill.id, embedding);
+        }
+        // Check each skill against accepted unique skills
+        for (const skill of skills) {
+            const skillEmbedding = embeddings.get(skill.id);
+            let isDuplicate = false;
+            for (const uniqueSkill of unique) {
+                const uniqueEmbedding = embeddings.get(uniqueSkill.id);
+                const similarity = embeddingService.cosineSimilarity(skillEmbedding, uniqueEmbedding);
+                if (similarity >= threshold) {
+                    // Found semantic duplicate
+                    const winner = compareSkillsForDedup(uniqueSkill, skill);
+                    if (winner === 'b') {
+                        // New skill is better - swap
+                        const index = unique.indexOf(uniqueSkill);
+                        unique[index] = skill;
+                        duplicates.push({
+                            kept: skill,
+                            discarded: uniqueSkill,
+                            reason: 'semantic_similarity',
+                            similarity,
+                        });
+                    }
+                    else {
+                        duplicates.push({
+                            kept: uniqueSkill,
+                            discarded: skill,
+                            reason: 'semantic_similarity',
+                            similarity,
+                        });
+                    }
+                    isDuplicate = true;
+                    break;
+                }
+            }
+            if (!isDuplicate) {
+                unique.push(skill);
+            }
+        }
+    }
+    finally {
+        embeddingService.close();
+    }
+    return { unique, duplicates };
+}
+//# sourceMappingURL=deduplication.js.map

package/dist/src/scripts/validation/deduplication.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"deduplication.js","sourceRoot":"","sources":["../../../../src/scripts/validation/deduplication.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAA;AAC5D,OAAO,EAAE,MAAM,EAAkC,MAAM,YAAY,CAAA;AACnE,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAA;AAE9C;;GAEG;AACH,MAAM,UAAU,qBAAqB,CAAC,CAAiB,EAAE,CAAiB;IACxE,MAAM,SAAS,GAAG,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,eAAe,CAAC,OAAO,CAAA;IACpF,MAAM,SAAS,GAAG,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,eAAe,CAAC,OAAO,CAAA;IAEpF,8BAA8B;IAC9B,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;QAC5B,OAAO,SAAS,GAAG,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAA;IAC1C,CAAC;IAED,mDAAmD;IACnD,OAAO,CAAC,CAAC,aAAa,IAAI,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAA;AACvD,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAAC,MAAwB;IAI3D,MAAM,IAAI,GAAG,IAAI,GAAG,EAA0B,CAAA;IAC9C,MAAM,UAAU,GAAqB,EAAE,CAAA;IAEvC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;YACpB,6EAA6E;YAC7E,4CAA4C;YAC5C,MAAM,GAAG,GAAG,UAAU,KAAK,CAAC,EAAE,EAAE,CAAA;YAChC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;gBACnB,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;YACtB,CAAC;YACD,SAAQ;QACV,CAAC;QAED,MAAM,GAAG,GAAG,WAAW,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAA;QACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QAE9B,IAAI,QAAQ,EAAE,CAAC;YACb,MAAM,MAAM,GAAG,qBAAqB,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAA;YACrD,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;gBACnB,UAAU,CAAC,IAAI,CAAC;oBACd,IAAI,EAAE,KAAK;oBACX,SAAS,EAAE,QAAQ;oBACnB,MAAM,EAAE,UAAU;iBACnB,CAAC,CAAA;gBACF,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;YACtB,CAAC;iBAAM,CAAC;gBACN,UAAU,CAAC,IAAI,CAAC;oBACd,IAAI,EAAE,QAAQ;oBACd,SAAS,EAAE,KAAK;oBAChB,MAAM,EAAE,UAAU;iBACnB,CAAC,CAAA;YACJ,CAAC;QACH,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;QACtB,CAAC;IACH,CAAC;IAED,OAAO;QACL,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;QACjC,UAAU;KACX,CAAA;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC5C,MAAwB,EACxB,YAAoB,MAAM,CAAC,oBAAoB;IAK/C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,CAAA;IACvC,CAAC;IAED,0CAA0C;IAC1C,MAAM,gBAAgB,GAAG,IAAI,gBAAgB,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAA;IACpE,MAAM,UAAU,GAAqB,EAAE,CAAA;IACvC,MAAM,MAAM,GAAqB,EAAE,CAAA;IACnC,MAAM,UAAU,GAAG,IAAI,GAAG,EAAwB,CAAA;IAElD,IAAI,CAAC;QACH,qCAAqC;QACrC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,IAAI,GAAG,GAAG,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,WAAW,EAAE,CAAA;YACjD,MAAM,SAAS,GAAG,MAAM,gBAAgB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;YACpD,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,SAAS,CAAC,CAAA;QACrC,CAAC;QAED,kDAAkD;QAClD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,cAAc,GAAG,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAE,CAAA;YAChD,IAAI,WAAW,GAAG,KAAK,CAAA;YAEvB,KAAK,MAAM,WAAW,IAAI,MAAM,EAAE,CAAC;gBACjC,MAAM,eAAe,GAAG,UAAU,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAE,CAAA;gBACvD,MAAM,UAAU,GAAG,gBAAgB,CAAC,gBAAgB,CAAC,cAAc,EAAE,eAAe,CAAC,CAAA;gBAErF,IAAI,UAAU,IAAI,SAAS,EAAE,CAAC;oBAC5B,2BAA2B;oBAC3B,MAAM,MAAM,GAAG,qBAAqB,CAAC,WAAW,EAAE,KAAK,CAAC,CAAA;oBACxD,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;wBACnB,6BAA6B;wBAC7B,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAA;wBACzC,MAAM,CAAC,KAAK,CAAC,GAAG,KAAK,CAAA;wBACrB,UAAU,CAAC,IAAI,CAAC;4BACd,IAAI,EAAE,KAAK;4BACX,SAAS,EAAE,WAAW;4BACtB,MAAM,EAAE,qBAAqB;4BAC7B,UAAU;yBACX,CAAC,CAAA;oBACJ,CAAC;yBAAM,CAAC;wBACN,UAAU,CAAC,IAAI,CAAC;4BACd,IAAI,EAAE,WAAW;4BACjB,SAAS,EAAE,KAAK;4BAChB,MAAM,EAAE,qBAAqB;4BAC7B,UAAU;yBACX,CAAC,CAAA;oBACJ,CAAC;oBACD,WAAW,GAAG,IAAI,CAAA;oBAClB,MAAK;gBACP,CAAC;YACH,CAAC;YAED,IAAI,CAAC,WAAW,EAAE,CAAC;gBACjB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YACpB,CAAC;QACH,CAAC;IACH,CAAC;YAAS,CAAC;QACT,gBAAgB,CAAC,KAAK,EAAE,CAAA;IAC1B,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,CAAA;AAC/B,CAAC"}

package/dist/src/scripts/validation/field-validators.d.ts

@@ -0,0 +1,9 @@
+/**
+ * SMI-863: Field validation functions for individual skill fields
+ */
+import { RawSkillInput, SkillValidationResult } from './types.js';
+/**
+ * Validate a single skill and apply auto-fixes where possible
+ */
+export declare function validateSkill(raw: RawSkillInput): SkillValidationResult;
+//# sourceMappingURL=field-validators.d.ts.map

package/dist/src/scripts/validation/field-validators.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"field-validators.d.ts","sourceRoot":"","sources":["../../../../src/scripts/validation/field-validators.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAGL,aAAa,EAGb,qBAAqB,EACtB,MAAM,YAAY,CAAA;AASnB;;GAEG;AACH,wBAAgB,aAAa,CAAC,GAAG,EAAE,aAAa,GAAG,qBAAqB,CAuIvE"}

package/dist/src/scripts/validation/field-validators.js

@@ -0,0 +1,135 @@
+/**
+ * SMI-863: Field validation functions for individual skill fields
+ */
+import { CONFIG, } from './types.js';
+import { extractOwnerFromRepoUrl, generateSkillId, normalizeQualityScore, normalizeTrustTier, normalizeSource, } from './normalizers.js';
+/**
+ * Validate a single skill and apply auto-fixes where possible
+ */
+export function validateSkill(raw) {
+    const errors = [];
+    const warnings = [];
+    const fixes = [];
+    // Normalize field names (handle both snake_case and camelCase)
+    const repoUrl = raw.repo_url ?? raw.repoUrl ?? null;
+    const qualityScore = raw.quality_score ?? raw.qualityScore ?? null;
+    const trustTier = raw.trust_tier ?? raw.trustTier ?? null;
+    // 1. Validate name (required)
+    const name = raw.name?.trim() || '';
+    if (!name) {
+        errors.push({ field: 'name', message: 'Name is required and cannot be empty' });
+    }
+    // 2. Validate author (use repo owner if missing)
+    let author = raw.author?.trim() || '';
+    if (!author) {
+        const ownerFromUrl = extractOwnerFromRepoUrl(repoUrl);
+        if (ownerFromUrl) {
+            author = ownerFromUrl;
+            fixes.push(`Auto-filled author from repo URL: "${author}"`);
+        }
+        else {
+            errors.push({
+                field: 'author',
+                message: 'Author is required (could not extract from repo URL)',
+            });
+        }
+    }
+    // 3. Validate description (use first 100 chars of name if missing)
+    let description = raw.description?.trim() || '';
+    if (!description) {
+        if (name) {
+            description = name.length > 100 ? name.substring(0, 100) + '...' : name;
+            fixes.push(`Auto-filled description from name: "${description}"`);
+        }
+        else {
+            errors.push({ field: 'description', message: 'Description is required' });
+        }
+    }
+    // 4. Validate ID format (author/name)
+    let id = raw.id?.trim() || '';
+    if (!id && author && name) {
+        id = generateSkillId(author, name);
+        fixes.push(`Auto-generated ID: "${id}"`);
+    }
+    if (id && !id.match(/^[^/]+\/[^/]+$/)) {
+        if (author && name) {
+            const newId = generateSkillId(author, name);
+            fixes.push(`Fixed invalid ID format: "${id}" -> "${newId}"`);
+            id = newId;
+        }
+        else {
+            errors.push({
+                field: 'id',
+                message: 'ID must be in format author/name',
+                value: id,
+            });
+        }
+    }
+    // 5. Validate quality score (0-100)
+    const normalizedScore = normalizeQualityScore(qualityScore);
+    if (qualityScore !== null && qualityScore !== undefined) {
+        if (qualityScore < 0 || qualityScore > 100) {
+            if (qualityScore >= 0 && qualityScore <= 1) {
+                fixes.push(`Normalized quality score from 0-1 to 0-100: ${qualityScore} -> ${normalizedScore}`);
+            }
+            else {
+                fixes.push(`Clamped quality score to 0-100: ${qualityScore} -> ${normalizedScore}`);
+            }
+        }
+    }
+    else {
+        warnings.push('Quality score missing, defaulted to 50');
+    }
+    // 6. Validate trust tier (enum)
+    const normalizedTrustTier = normalizeTrustTier(trustTier);
+    if (trustTier && !CONFIG.VALID_TRUST_TIERS.includes(trustTier)) {
+        fixes.push(`Normalized trust tier: "${trustTier}" -> "${normalizedTrustTier}"`);
+    }
+    // Validate source
+    const source = normalizeSource(raw.source);
+    if (!raw.source) {
+        warnings.push('Source missing, defaulted to "unknown"');
+    }
+    // Validate tags
+    const tags = Array.isArray(raw.tags) ? raw.tags.filter((t) => typeof t === 'string') : [];
+    // Check repo_url format if present
+    if (repoUrl) {
+        try {
+            new URL(repoUrl);
+        }
+        catch {
+            warnings.push(`Invalid repo URL format: "${repoUrl}"`);
+        }
+    }
+    // Build validated skill if no critical errors
+    if (errors.length === 0 && name && author && description && id) {
+        const validatedSkill = {
+            id,
+            name,
+            description,
+            author,
+            repo_url: repoUrl,
+            quality_score: normalizedScore,
+            trust_tier: normalizedTrustTier,
+            tags,
+            source,
+        };
+        return {
+            valid: true,
+            skill: validatedSkill,
+            original: raw,
+            errors: [],
+            warnings,
+            fixes,
+        };
+    }
+    return {
+        valid: false,
+        skill: null,
+        original: raw,
+        errors,
+        warnings,
+        fixes,
+    };
+}
+//# sourceMappingURL=field-validators.js.map

package/dist/src/scripts/validation/field-validators.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"field-validators.js","sourceRoot":"","sources":["../../../../src/scripts/validation/field-validators.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EACL,MAAM,GAMP,MAAM,YAAY,CAAA;AACnB,OAAO,EACL,uBAAuB,EACvB,eAAe,EACf,qBAAqB,EACrB,kBAAkB,EAClB,eAAe,GAChB,MAAM,kBAAkB,CAAA;AAEzB;;GAEG;AACH,MAAM,UAAU,aAAa,CAAC,GAAkB;IAC9C,MAAM,MAAM,GAA2B,EAAE,CAAA;IACzC,MAAM,QAAQ,GAAa,EAAE,CAAA;IAC7B,MAAM,KAAK,GAAa,EAAE,CAAA;IAE1B,+DAA+D;IAC/D,MAAM,OAAO,GAAG,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,OAAO,IAAI,IAAI,CAAA;IACnD,MAAM,YAAY,GAAG,GAAG,CAAC,aAAa,IAAI,GAAG,CAAC,YAAY,IAAI,IAAI,CAAA;IAClE,MAAM,SAAS,GAAG,GAAG,CAAC,UAAU,IAAI,GAAG,CAAC,SAAS,IAAI,IAAI,CAAA;IAEzD,8BAA8B;IAC9B,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAA;IACnC,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,sCAAsC,EAAE,CAAC,CAAA;IACjF,CAAC;IAED,iDAAiD;IACjD,IAAI,MAAM,GAAG,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAA;IACrC,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,YAAY,GAAG,uBAAuB,CAAC,OAAO,CAAC,CAAA;QACrD,IAAI,YAAY,EAAE,CAAC;YACjB,MAAM,GAAG,YAAY,CAAA;YACrB,KAAK,CAAC,IAAI,CAAC,sCAAsC,MAAM,GAAG,CAAC,CAAA;QAC7D,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC;gBACV,KAAK,EAAE,QAAQ;gBACf,OAAO,EAAE,sDAAsD;aAChE,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IAED,mEAAmE;IACnE,IAAI,WAAW,GAAG,GAAG,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAA;IAC/C,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,IAAI,IAAI,EAAE,CAAC;YACT,WAAW,GAAG,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAA;YACvE,KAAK,CAAC,IAAI,CAAC,uCAAuC,WAAW,GAAG,CAAC,CAAA;QACnE,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,aAAa,EAAE,OAAO,EAAE,yBAAyB,EAAE,CAAC,CAAA;QAC3E,CAAC;IACH,CAAC;IAED,sCAAsC;IACtC,IAAI,EAAE,GAAG,GAAG,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAA;IAC7B,IAAI,CAAC,EAAE,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;QAC1B,EAAE,GAAG,eAAe,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;QAClC,KAAK,CAAC,IAAI,CAAC,uBAAuB,EAAE,GAAG,CAAC,CAAA;IAC1C,CAAC;IAED,IAAI,EAAE,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,gBAAgB,CAAC,EAAE,CAAC;QACtC,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;YACnB,MAAM,KAAK,GAAG,eAAe,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;YAC3C,KAAK,CAAC,IAAI,CAAC,6BAA6B,EAAE,SAAS,KAAK,GAAG,CAAC,CAAA;YAC5D,EAAE,GAAG,KAAK,CAAA;QACZ,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC;gBACV,KAAK,EAAE,IAAI;gBACX,OAAO,EAAE,kCAAkC;gBAC3C,KAAK,EAAE,EAAE;aACV,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IAED,oCAAoC;IACpC,MAAM,eAAe,GAAG,qBAAqB,CAAC,YAAY,CAAC,CAAA;IAC3D,IAAI,YAAY,KAAK,IAAI,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;QACxD,IAAI,YAAY,GAAG,CAAC,IAAI,YAAY,GAAG,GAAG,EAAE,CAAC;YAC3C,IAAI,YAAY,IAAI,CAAC,IAAI,YAAY,IAAI,CAAC,EAAE,CAAC;gBAC3C,KAAK,CAAC,IAAI,CACR,+CAA+C,YAAY,OAAO,eAAe,EAAE,CACpF,CAAA;YACH,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,IAAI,CAAC,mCAAmC,YAAY,OAAO,eAAe,EAAE,CAAC,CAAA;YACrF,CAAC;QACH,CAAC;IACH,CAAC;SAAM,CAAC;QACN,QAAQ,CAAC,IAAI,CAAC,wCAAwC,CAAC,CAAA;IACzD,CAAC;IAED,gCAAgC;IAChC,MAAM,mBAAmB,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAA;IACzD,IAAI,SAAS,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,QAAQ,CAAC,SAAsB,CAAC,EAAE,CAAC;QAC5E,KAAK,CAAC,IAAI,CAAC,2BAA2B,SAAS,SAAS,mBAAmB,GAAG,CAAC,CAAA;IACjF,CAAC;IAED,kBAAkB;IAClB,MAAM,MAAM,GAAG,eAAe,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;IAC1C,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC;QAChB,QAAQ,CAAC,IAAI,CAAC,wCAAwC,CAAC,CAAA;IACzD,CAAC;IAED,gBAAgB;IAChB,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;IAEzF,mCAAmC;IACnC,IAAI,OAAO,EAAE,CAAC;QACZ,IAAI,CAAC;YACH,IAAI,GAAG,CAAC,OAAO,CAAC,CAAA;QAClB,CAAC;QAAC,MAAM,CAAC;YACP,QAAQ,CAAC,IAAI,CAAC,6BAA6B,OAAO,GAAG,CAAC,CAAA;QACxD,CAAC;IACH,CAAC;IAED,8CAA8C;IAC9C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,IAAI,MAAM,IAAI,WAAW,IAAI,EAAE,EAAE,CAAC;QAC/D,MAAM,cAAc,GAAmB;YACrC,EAAE;YACF,IAAI;YACJ,WAAW;YACX,MAAM;YACN,QAAQ,EAAE,OAAO;YACjB,aAAa,EAAE,eAAe;YAC9B,UAAU,EAAE,mBAAmB;YAC/B,IAAI;YACJ,MAAM;SACP,CAAA;QAED,OAAO;YACL,KAAK,EAAE,IAAI;YACX,KAAK,EAAE,cAAc;YACrB,QAAQ,EAAE,GAAG;YACb,MAAM,EAAE,EAAE;YACV,QAAQ;YACR,KAAK;SACN,CAAA;IACH,CAAC;IAED,OAAO;QACL,KAAK,EAAE,KAAK;QACZ,KAAK,EAAE,IAAI;QACX,QAAQ,EAAE,GAAG;QACb,MAAM;QACN,QAAQ;QACR,KAAK;KACN,CAAA;AACH,CAAC"}

package/dist/src/scripts/validation/index.d.ts

@@ -0,0 +1,29 @@
+/**
+ * SMI-863: Skill Validation and Deduplication Pipeline
+ *
+ * Validates skill data against schema rules and deduplicates
+ * entries using repository URL as primary key with source priority.
+ *
+ * Usage:
+ *   npx tsx packages/core/src/scripts/validate-skills.ts [--input skills.json] [--output-dir data]
+ *
+ * Validation Rules:
+ * 1. Name present (non-empty)
+ * 2. Author present (use repo owner if missing)
+ * 3. Description present (use first 100 chars of name if missing)
+ * 4. Valid ID format (author/name)
+ * 5. Quality score 0-100
+ * 6. Valid trust tier enum
+ *
+ * Deduplication:
+ * - Primary key: repo_url
+ * - Source priority: anthropic-official (100) > github (80) > claude-plugins (40)
+ * - Keep higher quality score on conflict
+ * - Semantic similarity detection (threshold 0.85)
+ */
+export * from './types.js';
+export * from './normalizers.js';
+export * from './field-validators.js';
+export * from './deduplication.js';
+export { runValidationPipeline } from './pipeline.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/src/scripts/validation/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/scripts/validation/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAOH,cAAc,YAAY,CAAA;AAC1B,cAAc,kBAAkB,CAAA;AAChC,cAAc,uBAAuB,CAAA;AACrC,cAAc,oBAAoB,CAAA;AAClC,OAAO,EAAE,qBAAqB,EAAE,MAAM,eAAe,CAAA"}

package/dist/src/scripts/validation/index.js

@@ -0,0 +1,83 @@
+/**
+ * SMI-863: Skill Validation and Deduplication Pipeline
+ *
+ * Validates skill data against schema rules and deduplicates
+ * entries using repository URL as primary key with source priority.
+ *
+ * Usage:
+ *   npx tsx packages/core/src/scripts/validate-skills.ts [--input skills.json] [--output-dir data]
+ *
+ * Validation Rules:
+ * 1. Name present (non-empty)
+ * 2. Author present (use repo owner if missing)
+ * 3. Description present (use first 100 chars of name if missing)
+ * 4. Valid ID format (author/name)
+ * 5. Quality score 0-100
+ * 6. Valid trust tier enum
+ *
+ * Deduplication:
+ * - Primary key: repo_url
+ * - Source priority: anthropic-official (100) > github (80) > claude-plugins (40)
+ * - Keep higher quality score on conflict
+ * - Semantic similarity detection (threshold 0.85)
+ */
+import * as fs from 'fs';
+import { CONFIG } from './types.js';
+import { runValidationPipeline } from './pipeline.js';
+// Re-export all types and functions for external use
+export * from './types.js';
+export * from './normalizers.js';
+export * from './field-validators.js';
+export * from './deduplication.js';
+export { runValidationPipeline } from './pipeline.js';
+// ============================================================================
+// CLI Entry Point
+// ============================================================================
+async function main() {
+    const args = process.argv.slice(2);
+    // Parse arguments
+    let inputPath = CONFIG.DEFAULT_INPUT;
+    let outputDir = CONFIG.DEFAULT_OUTPUT_DIR;
+    for (let i = 0; i < args.length; i++) {
+        if (args[i] === '--input' && args[i + 1]) {
+            inputPath = args[++i];
+        }
+        else if (args[i] === '--output-dir' && args[i + 1]) {
+            outputDir = args[++i];
+        }
+        else if (args[i] === '--help') {
+            console.log(`
+Usage: npx tsx packages/core/src/scripts/validate-skills.ts [options]
+
+Options:
+  --input <path>       Path to input JSON file (default: ${CONFIG.DEFAULT_INPUT})
+  --output-dir <path>  Output directory for results (default: ${CONFIG.DEFAULT_OUTPUT_DIR})
+  --help               Show this help message
+
+Output files:
+  validated-skills.json   Clean, deduplicated skill data
+  validation-report.json  Validation statistics and errors
+  duplicates-report.json  Detected duplicate entries
+`);
+            process.exit(0);
+        }
+    }
+    // Validate input file exists
+    if (!fs.existsSync(inputPath)) {
+        console.error(`Error: Input file not found: ${inputPath}`);
+        process.exit(1);
+    }
+    try {
+        await runValidationPipeline(inputPath, outputDir);
+    }
+    catch (error) {
+        console.error('Pipeline failed:', error);
+        process.exit(1);
+    }
+}
+// Run if executed directly
+const isMainModule = process.argv[1]?.includes('validate-skills') || process.argv[1]?.includes('validation/index');
+if (isMainModule) {
+    main().catch(console.error);
+}
+//# sourceMappingURL=index.js.map

package/dist/src/scripts/validation/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/scripts/validation/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,KAAK,EAAE,MAAM,IAAI,CAAA;AACxB,OAAO,EAAE,MAAM,EAAE,MAAM,YAAY,CAAA;AACnC,OAAO,EAAE,qBAAqB,EAAE,MAAM,eAAe,CAAA;AAErD,qDAAqD;AACrD,cAAc,YAAY,CAAA;AAC1B,cAAc,kBAAkB,CAAA;AAChC,cAAc,uBAAuB,CAAA;AACrC,cAAc,oBAAoB,CAAA;AAClC,OAAO,EAAE,qBAAqB,EAAE,MAAM,eAAe,CAAA;AAErD,+EAA+E;AAC/E,kBAAkB;AAClB,+EAA+E;AAE/E,KAAK,UAAU,IAAI;IACjB,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;IAElC,kBAAkB;IAClB,IAAI,SAAS,GAAW,MAAM,CAAC,aAAa,CAAA;IAC5C,IAAI,SAAS,GAAW,MAAM,CAAC,kBAAkB,CAAA;IAEjD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,SAAS,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;YACzC,SAAS,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC,CAAA;QACvB,CAAC;aAAM,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,cAAc,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;YACrD,SAAS,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC,CAAA;QACvB,CAAC;aAAM,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,QAAQ,EAAE,CAAC;YAChC,OAAO,CAAC,GAAG,CAAC;;;;2DAIyC,MAAM,CAAC,aAAa;gEACf,MAAM,CAAC,kBAAkB;;;;;;;CAOxF,CAAC,CAAA;YACI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACjB,CAAC;IACH,CAAC;IAED,6BAA6B;IAC7B,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC9B,OAAO,CAAC,KAAK,CAAC,gCAAgC,SAAS,EAAE,CAAC,CAAA;QAC1D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACjB,CAAC;IAED,IAAI,CAAC;QACH,MAAM,qBAAqB,CAAC,SAAS,EAAE,SAAS,CAAC,CAAA;IACnD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,kBAAkB,EAAE,KAAK,CAAC,CAAA;QACxC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACjB,CAAC;AACH,CAAC;AAED,2BAA2B;AAC3B,MAAM,YAAY,GAChB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,iBAAiB,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,kBAAkB,CAAC,CAAA;AAC/F,IAAI,YAAY,EAAE,CAAC;IACjB,IAAI,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;AAC7B,CAAC"}

package/dist/src/scripts/validation/normalizers.d.ts

@@ -0,0 +1,29 @@
+/**
+ * SMI-863: Normalization utilities for skill validation
+ */
+import { TrustTier } from './types.js';
+/**
+ * Extract owner from repo URL
+ */
+export declare function extractOwnerFromRepoUrl(repoUrl: string | null | undefined): string | null;
+/**
+ * Generate skill ID from author and name
+ */
+export declare function generateSkillId(author: string, name: string): string;
+/**
+ * Normalize quality score to 0-100 range
+ */
+export declare function normalizeQualityScore(score: number | null | undefined): number;
+/**
+ * Normalize trust tier to valid enum value
+ */
+export declare function normalizeTrustTier(tier: string | null | undefined): TrustTier;
+/**
+ * Normalize source name
+ */
+export declare function normalizeSource(source: string | null | undefined): string;
+/**
+ * Generate hash for repo URL deduplication
+ */
+export declare function hashRepoUrl(repoUrl: string): string;
+//# sourceMappingURL=normalizers.d.ts.map

package/dist/src/scripts/validation/normalizers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"normalizers.d.ts","sourceRoot":"","sources":["../../../../src/scripts/validation/normalizers.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AAEtC;;GAEG;AACH,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,MAAM,GAAG,IAAI,CAUzF;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAQpE;AAED;;GAEG;AACH,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,MAAM,CAU9E;AAED;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,SAAS,CAmB7E;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,MAAM,CAGzE;AAED;;GAEG;AACH,wBAAgB,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAEnD"}

package/dist/src/scripts/validation/normalizers.js

@@ -0,0 +1,79 @@
+/**
+ * SMI-863: Normalization utilities for skill validation
+ */
+import { createHash } from 'crypto';
+/**
+ * Extract owner from repo URL
+ */
+export function extractOwnerFromRepoUrl(repoUrl) {
+    if (!repoUrl)
+        return null;
+    try {
+        const url = new URL(repoUrl);
+        const parts = url.pathname.split('/').filter(Boolean);
+        return parts[0] || null;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Generate skill ID from author and name
+ */
+export function generateSkillId(author, name) {
+    const sanitize = (s) => s
+        .toLowerCase()
+        .replace(/[^a-z0-9-_]/g, '-')
+        .replace(/-+/g, '-')
+        .replace(/^-|-$/g, '');
+    return `${sanitize(author)}/${sanitize(name)}`;
+}
+/**
+ * Normalize quality score to 0-100 range
+ */
+export function normalizeQualityScore(score) {
+    if (score === null || score === undefined)
+        return 50; // Default score
+    // If score is 0-1, convert to 0-100
+    if (score >= 0 && score <= 1) {
+        return Math.round(score * 100);
+    }
+    // Clamp to 0-100
+    return Math.max(0, Math.min(100, Math.round(score)));
+}
+/**
+ * Normalize trust tier to valid enum value
+ */
+export function normalizeTrustTier(tier) {
+    if (!tier)
+        return 'unknown';
+    const normalized = tier.toLowerCase().trim();
+    // Map common variations
+    const mappings = {
+        verified: 'verified',
+        official: 'verified',
+        'anthropic-official': 'verified',
+        community: 'community',
+        experimental: 'experimental',
+        beta: 'experimental',
+        unknown: 'unknown',
+        unverified: 'unknown',
+        standard: 'community',
+    };
+    return mappings[normalized] || 'unknown';
+}
+/**
+ * Normalize source name
+ */
+export function normalizeSource(source) {
+    if (!source)
+        return 'unknown';
+    return source.toLowerCase().trim();
+}
+/**
+ * Generate hash for repo URL deduplication
+ */
+export function hashRepoUrl(repoUrl) {
+    return createHash('md5').update(repoUrl.toLowerCase().trim()).digest('hex');
+}
+//# sourceMappingURL=normalizers.js.map

package/dist/src/scripts/validation/normalizers.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"normalizers.js","sourceRoot":"","sources":["../../../../src/scripts/validation/normalizers.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAA;AAGnC;;GAEG;AACH,MAAM,UAAU,uBAAuB,CAAC,OAAkC;IACxE,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAA;IAEzB,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAA;QAC5B,MAAM,KAAK,GAAG,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;QACrD,OAAO,KAAK,CAAC,CAAC,CAAC,IAAI,IAAI,CAAA;IACzB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAA;IACb,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,MAAc,EAAE,IAAY;IAC1D,MAAM,QAAQ,GAAG,CAAC,CAAS,EAAE,EAAE,CAC7B,CAAC;SACE,WAAW,EAAE;SACb,OAAO,CAAC,cAAc,EAAE,GAAG,CAAC;SAC5B,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC;SACnB,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAA;IAC1B,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAA;AAChD,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,qBAAqB,CAAC,KAAgC;IACpE,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,EAAE,CAAA,CAAC,gBAAgB;IAErE,oCAAoC;IACpC,IAAI,KAAK,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,GAAG,CAAC,CAAA;IAChC,CAAC;IAED,iBAAiB;IACjB,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;AACtD,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAA+B;IAChE,IAAI,CAAC,IAAI;QAAE,OAAO,SAAS,CAAA;IAE3B,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAA;IAE5C,wBAAwB;IACxB,MAAM,QAAQ,GAA8B;QAC1C,QAAQ,EAAE,UAAU;QACpB,QAAQ,EAAE,UAAU;QACpB,oBAAoB,EAAE,UAAU;QAChC,SAAS,EAAE,WAAW;QACtB,YAAY,EAAE,cAAc;QAC5B,IAAI,EAAE,cAAc;QACpB,OAAO,EAAE,SAAS;QAClB,UAAU,EAAE,SAAS;QACrB,QAAQ,EAAE,WAAW;KACtB,CAAA;IAED,OAAO,QAAQ,CAAC,UAAU,CAAC,IAAI,SAAS,CAAA;AAC1C,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,MAAiC;IAC/D,IAAI,CAAC,MAAM;QAAE,OAAO,SAAS,CAAA;IAC7B,OAAO,MAAM,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAA;AACpC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,OAAe;IACzC,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;AAC7E,CAAC"}

package/dist/src/scripts/validation/pipeline.d.ts

@@ -0,0 +1,13 @@
+/**
+ * SMI-863: Validation pipeline for skill processing
+ */
+import { ValidatedSkill, ValidationReport, DuplicatesReport } from './types.js';
+/**
+ * Run the complete validation and deduplication pipeline
+ */
+export declare function runValidationPipeline(inputPath: string, outputDir: string): Promise<{
+    validatedSkills: ValidatedSkill[];
+    validationReport: ValidationReport;
+    duplicatesReport: DuplicatesReport;
+}>;
+//# sourceMappingURL=pipeline.d.ts.map

package/dist/src/scripts/validation/pipeline.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../../../src/scripts/validation/pipeline.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,EAAiB,cAAc,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAA;AAI9F;;GAEG;AACH,wBAAsB,qBAAqB,CACzC,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC;IACT,eAAe,EAAE,cAAc,EAAE,CAAA;IACjC,gBAAgB,EAAE,gBAAgB,CAAA;IAClC,gBAAgB,EAAE,gBAAgB,CAAA;CACnC,CAAC,CAqID"}