@sporesec/arcana 3.0.0 → 3.0.1

package/dist/cli.js

@@ -127,6 +127,9 @@ export function createCli() {
         .option("-a, --all", "Validate all installed skills")
         .option("-f, --fix", "Auto-fix common issues")
         .option("-j, --json", "Output as JSON")
+        .option("--source <dir>", "Validate from source directory instead of install dir")
+        .option("--cross", "Run cross-validation (marketplace sync, companions, orphans)")
+        .option("--min-score <n>", "Minimum quality score (0-100), fail if any skill scores below", parseInt)
         .action(async (skill, opts) => {
         const { validateCommand } = await import("./commands/validate.js");
         return validateCommand(skill, opts);
@@ -213,6 +216,7 @@ export function createCli() {
         .description("Audit skill quality (code examples, BAD/GOOD pairs, structure)")
         .option("-a, --all", "Audit all installed skills")
         .option("-j, --json", "Output as JSON")
+        .option("--source <dir>", "Audit from source directory instead of install dir")
         .action(async (skill, opts) => {
         const { auditCommand } = await import("./commands/audit.js");
         return auditCommand(skill, opts);

package/dist/commands/audit.d.ts

@@ -1,4 +1,4 @@
-interface AuditResult {
+export interface AuditResult {
     skill: string;
     rating: "PERFECT" | "STRONG" | "ADEQUATE" | "WEAK";
     score: number;
@@ -12,5 +12,5 @@ export declare function auditSkill(skillDir: string, skillName: string): AuditRe
 export declare function auditCommand(skill: string | undefined, opts: {
     all?: boolean;
     json?: boolean;
+    source?: string;
 }): Promise<void>;
-export {};

package/dist/commands/audit.js

@@ -1,5 +1,5 @@
 import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
-import { join } from "node:path";
+import { join, resolve } from "node:path";
 import { getInstallDir } from "../utils/fs.js";
 import { extractFrontmatter, parseFrontmatter } from "../utils/frontmatter.js";
 import { ui, banner } from "../utils/ui.js";
@@ -76,9 +76,25 @@ export function auditSkill(skillDir, skillName) {
     });
     if (hasExtras)
         score += 10;
-    // Rating
+    // 9. Section diversity (3+ unique ## headings)
+    const uniqueHeadings = new Set((body.match(/^## .+$/gm) || []).map((h) => h.toLowerCase()));
+    const goodDiversity = uniqueHeadings.size >= 3;
+    checks.push({
+        name: "Section diversity (3+ unique headings)",
+        passed: goodDiversity,
+        detail: `${uniqueHeadings.size} unique sections`,
+    });
+    if (goodDiversity)
+        score += 5;
+    // 10. Numbered steps (task decomposition signal)
+    const numberedSteps = (body.match(/^\d+\.\s/gm) || []).length;
+    const hasSteps = numberedSteps >= 3;
+    checks.push({ name: "Has numbered steps (3+)", passed: hasSteps, detail: `${numberedSteps} steps` });
+    if (hasSteps)
+        score += 5;
+    // Rating (max possible: 110)
     let rating;
-    if (score >= 85)
+    if (score >= 90)
         rating = "PERFECT";
     else if (score >= 65)
         rating = "STRONG";
@@ -91,8 +107,8 @@ export function auditSkill(skillDir, skillName) {
 export async function auditCommand(skill, opts) {
     if (!opts.json)
         banner();
-    const installDir = getInstallDir();
-    if (!existsSync(installDir)) {
+    const baseDir = opts.source ? resolve(opts.source) : getInstallDir();
+    if (!existsSync(baseDir)) {
         if (opts.json) {
             console.log(JSON.stringify({ results: [] }));
         }
@@ -104,7 +120,14 @@ export async function auditCommand(skill, opts) {
     }
     let skills;
     if (opts.all) {
-        skills = readdirSync(installDir).filter((d) => statSync(join(installDir, d)).isDirectory());
+        skills = readdirSync(baseDir).filter((d) => {
+            try {
+                return statSync(join(baseDir, d)).isDirectory();
+            }
+            catch {
+                return false;
+            }
+        });
     }
     else if (skill) {
         skills = [skill];
@@ -123,7 +146,7 @@ export async function auditCommand(skill, opts) {
     }
     const results = [];
     for (const name of skills.sort()) {
-        const skillDir = join(installDir, name);
+        const skillDir = join(baseDir, name);
         if (!existsSync(skillDir)) {
             results.push({ skill: name, rating: "WEAK", score: 0, checks: [{ name: "Exists", passed: false }] });
             continue;
@@ -144,7 +167,7 @@ export async function auditCommand(skill, opts) {
                 : r.rating === "ADEQUATE"
                     ? ui.warn
                     : ui.error;
-        console.log(`  ${ratingColor(`[${r.rating}]`)} ${ui.bold(r.skill)} ${ui.dim(`(${r.score}/100)`)}`);
+        console.log(`  ${ratingColor(`[${r.rating}]`)} ${ui.bold(r.skill)} ${ui.dim(`(${r.score}/110)`)}`);
         for (const check of r.checks) {
             if (!check.passed) {
                 const detail = check.detail ? ` ${ui.dim(`(${check.detail})`)}` : "";

package/dist/commands/validate.d.ts

@@ -2,4 +2,7 @@ export declare function validateCommand(skill: string | undefined, opts: {
     all?: boolean;
     fix?: boolean;
     json?: boolean;
+    source?: string;
+    cross?: boolean;
+    minScore?: number;
 }): Promise<void>;

package/dist/commands/validate.js

@@ -1,5 +1,5 @@
 import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
-import { join } from "node:path";
+import { join, resolve } from "node:path";
 import { getInstallDir } from "../utils/fs.js";
 import { validateSkillDir, fixSkillFrontmatter } from "../utils/frontmatter.js";
 import { atomicWriteSync } from "../utils/atomic.js";
@@ -8,8 +8,8 @@ import { scanSkillContent } from "../utils/scanner.js";
 export async function validateCommand(skill, opts) {
     if (!opts.json)
         banner();
-    const installDir = getInstallDir();
-    if (!existsSync(installDir)) {
+    const baseDir = opts.source ? resolve(opts.source) : getInstallDir();
+    if (!existsSync(baseDir)) {
         if (opts.json) {
             console.log(JSON.stringify({ results: [] }));
         }
@@ -21,7 +21,14 @@ export async function validateCommand(skill, opts) {
     }
     let skills;
     if (opts.all) {
-        skills = readdirSync(installDir).filter((d) => statSync(join(installDir, d)).isDirectory());
+        skills = readdirSync(baseDir).filter((d) => {
+            try {
+                return statSync(join(baseDir, d)).isDirectory();
+            }
+            catch {
+                return false;
+            }
+        });
     }
     else if (skill) {
         skills = [skill];
@@ -40,7 +47,7 @@ export async function validateCommand(skill, opts) {
     }
     const results = [];
     for (const name of skills) {
-        const skillDir = join(installDir, name);
+        const skillDir = join(baseDir, name);
         if (!existsSync(skillDir)) {
             results.push({ skill: name, valid: false, errors: ["Not installed"], warnings: [], infos: [] });
             continue;
@@ -85,10 +92,42 @@ export async function validateCommand(skill, opts) {
                 /* skip if unreadable */
             }
         }
-        results.push(result);
+        // Quality scoring (when --min-score is set)
+        const entry = { ...result };
+        if (opts.minScore !== undefined) {
+            const { auditSkill } = await import("./audit.js");
+            const audit = auditSkill(skillDir, name);
+            entry.qualityScore = audit.score;
+            entry.qualityRating = audit.rating;
+            if (audit.score < opts.minScore) {
+                entry.valid = false;
+                entry.errors.push(`Quality score ${audit.score} below minimum ${opts.minScore} (${audit.rating})`);
+            }
+        }
+        results.push(entry);
+    }
+    // Cross-validation (when --cross is set)
+    let crossIssues = [];
+    if (opts.cross) {
+        const { crossValidate } = await import("../utils/quality.js");
+        const marketplacePaths = opts.source
+            ? [
+                resolve(opts.source, "..", ".claude-plugin", "marketplace.json"),
+                resolve(opts.source, ".claude-plugin", "marketplace.json"),
+            ]
+            : [resolve(baseDir, "..", ".claude-plugin", "marketplace.json")];
+        const marketplacePath = marketplacePaths.find((p) => existsSync(p));
+        if (marketplacePath) {
+            crossIssues = crossValidate(baseDir, marketplacePath);
+        }
+        else if (!opts.json) {
+            console.log(ui.warn("  Could not find marketplace.json for cross-validation"));
+        }
     }
+    const hasErrors = results.some((r) => !r.valid);
+    const hasCrossErrors = crossIssues.some((i) => i.level === "error");
     if (opts.json) {
-        console.log(JSON.stringify({
+        const output = {
             results: results.map((r) => ({
                 skill: r.skill,
                 valid: r.valid,
@@ -96,12 +135,28 @@ export async function validateCommand(skill, opts) {
                 warnings: r.warnings,
                 infos: r.infos,
                 fixed: r.fixed ?? false,
+                ...(r.qualityScore !== undefined && { qualityScore: r.qualityScore, qualityRating: r.qualityRating }),
             })),
-        }, null, 2));
-        if (results.some((r) => !r.valid))
+        };
+        if (opts.cross && crossIssues.length > 0) {
+            output.crossValidation = crossIssues;
+        }
+        const scores = results.filter((r) => r.qualityScore !== undefined).map((r) => r.qualityScore);
+        if (scores.length > 0) {
+            output.summary = {
+                total: results.length,
+                passed: results.filter((r) => r.valid).length,
+                failed: results.filter((r) => !r.valid).length,
+                averageScore: Math.round(scores.reduce((a, b) => a + b, 0) / scores.length),
+                belowThreshold: opts.minScore ? results.filter((r) => (r.qualityScore ?? 0) < opts.minScore).length : 0,
+            };
+        }
+        console.log(JSON.stringify(output, null, 2));
+        if (hasErrors || hasCrossErrors)
             process.exit(1);
         return;
     }
+    // Human-readable output
     let passed = 0;
     let warned = 0;
     let failed = 0;
@@ -109,7 +164,8 @@ export async function validateCommand(skill, opts) {
     for (const r of results) {
         const icon = r.valid ? (r.warnings.length > 0 ? ui.warn("[!!]") : ui.success("[OK]")) : ui.error("[XX]");
         const fixTag = r.fixed ? ui.cyan(" [fixed]") : "";
-        console.log(`  ${icon} ${ui.bold(r.skill)}${fixTag}`);
+        const scoreTag = r.qualityScore !== undefined ? ui.dim(` (${r.qualityScore}/100 ${r.qualityRating})`) : "";
+        console.log(`  ${icon} ${ui.bold(r.skill)}${fixTag}${scoreTag}`);
         for (const err of r.errors) {
             console.log(ui.error(`    Error: ${err}`));
         }
@@ -131,6 +187,15 @@ export async function validateCommand(skill, opts) {
         if (r.fixed)
             fixed++;
     }
+    // Cross-validation output
+    if (opts.cross && crossIssues.length > 0) {
+        console.log();
+        console.log(ui.bold("  Cross-validation:"));
+        for (const issue of crossIssues) {
+            const icon = issue.level === "error" ? ui.error("[XX]") : ui.warn("[!!]");
+            console.log(`  ${icon} ${issue.skill}: ${issue.detail}`);
+        }
+    }
     console.log();
     const parts = [];
     if (passed > 0)
@@ -141,8 +206,10 @@ export async function validateCommand(skill, opts) {
         parts.push(ui.error(`${failed} failed`));
     if (fixed > 0)
         parts.push(ui.cyan(`${fixed} fixed`));
+    if (hasCrossErrors)
+        parts.push(ui.error(`${crossIssues.filter((i) => i.level === "error").length} cross-validation errors`));
     console.log(`  ${parts.join(ui.dim(" | "))}`);
     console.log();
-    if (failed > 0)
+    if (failed > 0 || hasCrossErrors)
         process.exit(1);
 }

package/dist/utils/frontmatter.js

@@ -138,13 +138,16 @@ export function validateSkillDir(skillDir, skillName) {
         return result;
     }
     if (!parsed.description) {
-        result.warnings.push("Missing description in frontmatter");
+        result.valid = false;
+        result.errors.push("Missing description in frontmatter");
     }
     else if (parsed.description.length < MIN_DESC_LENGTH) {
-        result.warnings.push(`Description too short (${parsed.description.length} chars, recommend ${MIN_DESC_LENGTH}+)`);
+        result.valid = false;
+        result.errors.push(`Description too short (${parsed.description.length} chars, minimum ${MIN_DESC_LENGTH})`);
     }
     else if (parsed.description.length > MAX_DESC_LENGTH) {
-        result.warnings.push(`Description too long (${parsed.description.length} chars, max ${MAX_DESC_LENGTH})`);
+        result.valid = false;
+        result.errors.push(`Description too long (${parsed.description.length} chars, max ${MAX_DESC_LENGTH})`);
     }
     // Check for non-standard fields (metadata is invalid per spec)
     const standardFields = ["name", "description"];
@@ -181,11 +184,11 @@ export function validateSkillDir(skillDir, skillName) {
         result.warnings.push("SKILL.md body is very short (less than 50 chars)");
     }
     if (extracted.body.trim().length >= 50 && !extracted.body.includes("##")) {
-        result.infos.push("Body has no ## headings (recommended for structure)");
+        result.warnings.push("Body has no ## headings (required for structure)");
     }
     // Check for code blocks (quality signal)
     if (extracted.body.trim().length >= 50 && !extracted.body.includes("```")) {
-        result.infos.push("No code blocks found (procedural skills should include code examples)");
+        result.warnings.push("No code blocks found (skills must include code examples)");
     }
     // Check for BAD/GOOD pattern examples
     const hasPattern = /(?:BAD|GOOD|WRONG|RIGHT|AVOID|PREFER|DO NOT|INSTEAD)/i.test(extracted.body) ||

package/dist/utils/quality.d.ts

@@ -0,0 +1,27 @@
+import type { MarketplacePlugin } from "../types.js";
+export interface CrossValidationIssue {
+    level: "error" | "warning";
+    category: "marketplace-drift" | "orphan" | "companion" | "duplicate-desc";
+    skill: string;
+    detail: string;
+}
+/**
+ * Jaccard word-level similarity between two strings.
+ * Returns 0.0 (completely different) to 1.0 (identical).
+ */
+export declare function jaccardSimilarity(a: string, b: string): number;
+/**
+ * Validate companion references in marketplace plugins.
+ * Every companion must reference an existing plugin name.
+ */
+export declare function validateCompanions(plugins: MarketplacePlugin[]): CrossValidationIssue[];
+/**
+ * Check description sync between SKILL.md frontmatter and marketplace.json.
+ * Returns an issue if similarity is below 0.5.
+ */
+export declare function validateDescriptionSync(skillName: string, frontmatterDesc: string, marketplaceDesc: string): CrossValidationIssue | null;
+/**
+ * Cross-validate skill directories against marketplace.json.
+ * Checks: orphans, companions, description drift, near-duplicates.
+ */
+export declare function crossValidate(skillsDir: string, marketplacePath: string): CrossValidationIssue[];

package/dist/utils/quality.js

@@ -0,0 +1,174 @@
+import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
+import { join } from "node:path";
+import { extractFrontmatter, parseFrontmatter } from "./frontmatter.js";
+/**
+ * Jaccard word-level similarity between two strings.
+ * Returns 0.0 (completely different) to 1.0 (identical).
+ */
+export function jaccardSimilarity(a, b) {
+    const wordsA = new Set(a.toLowerCase().split(/\s+/).filter(Boolean));
+    const wordsB = new Set(b.toLowerCase().split(/\s+/).filter(Boolean));
+    if (wordsA.size === 0 && wordsB.size === 0)
+        return 1.0;
+    if (wordsA.size === 0 || wordsB.size === 0)
+        return 0.0;
+    let intersection = 0;
+    for (const w of wordsA) {
+        if (wordsB.has(w))
+            intersection++;
+    }
+    const union = wordsA.size + wordsB.size - intersection;
+    return union === 0 ? 0 : intersection / union;
+}
+/**
+ * Validate companion references in marketplace plugins.
+ * Every companion must reference an existing plugin name.
+ */
+export function validateCompanions(plugins) {
+    const issues = [];
+    const names = new Set(plugins.map((p) => p.name));
+    for (const plugin of plugins) {
+        if (!plugin.companions)
+            continue;
+        for (const companion of plugin.companions) {
+            if (!names.has(companion)) {
+                issues.push({
+                    level: "error",
+                    category: "companion",
+                    skill: plugin.name,
+                    detail: `Companion "${companion}" does not exist in marketplace`,
+                });
+            }
+        }
+    }
+    return issues;
+}
+/**
+ * Check description sync between SKILL.md frontmatter and marketplace.json.
+ * Returns an issue if similarity is below 0.5.
+ */
+export function validateDescriptionSync(skillName, frontmatterDesc, marketplaceDesc) {
+    if (!frontmatterDesc || !marketplaceDesc)
+        return null;
+    const similarity = jaccardSimilarity(frontmatterDesc, marketplaceDesc);
+    if (similarity < 0.5) {
+        return {
+            level: "warning",
+            category: "marketplace-drift",
+            skill: skillName,
+            detail: `Description drift (${Math.round(similarity * 100)}% similarity) between SKILL.md and marketplace.json`,
+        };
+    }
+    return null;
+}
+/**
+ * Cross-validate skill directories against marketplace.json.
+ * Checks: orphans, companions, description drift, near-duplicates.
+ */
+export function crossValidate(skillsDir, marketplacePath) {
+    const issues = [];
+    // Load marketplace
+    let marketplace;
+    try {
+        marketplace = JSON.parse(readFileSync(marketplacePath, "utf-8"));
+    }
+    catch {
+        issues.push({
+            level: "error",
+            category: "orphan",
+            skill: "marketplace.json",
+            detail: "Cannot read or parse marketplace.json",
+        });
+        return issues;
+    }
+    const pluginNames = new Set(marketplace.plugins.map((p) => p.name));
+    const pluginMap = new Map(marketplace.plugins.map((p) => [p.name, p]));
+    // Get skill directories
+    let skillDirs;
+    try {
+        skillDirs = readdirSync(skillsDir).filter((d) => {
+            try {
+                return statSync(join(skillsDir, d)).isDirectory();
+            }
+            catch {
+                return false;
+            }
+        });
+    }
+    catch {
+        issues.push({
+            level: "error",
+            category: "orphan",
+            skill: "skills/",
+            detail: "Cannot read skills directory",
+        });
+        return issues;
+    }
+    const dirNames = new Set(skillDirs);
+    // 1. Orphan directories (dir exists, no marketplace entry)
+    for (const dir of skillDirs) {
+        if (!pluginNames.has(dir)) {
+            issues.push({
+                level: "error",
+                category: "orphan",
+                skill: dir,
+                detail: "Skill directory exists but no marketplace.json entry",
+            });
+        }
+    }
+    // 2. Orphan entries (marketplace entry, no dir)
+    for (const name of pluginNames) {
+        if (!dirNames.has(name)) {
+            issues.push({
+                level: "error",
+                category: "orphan",
+                skill: name,
+                detail: "Marketplace entry exists but no skill directory",
+            });
+        }
+    }
+    // 3. Companion validation
+    issues.push(...validateCompanions(marketplace.plugins));
+    // 4. Description drift (SKILL.md frontmatter vs marketplace.json)
+    for (const dir of skillDirs) {
+        const plugin = pluginMap.get(dir);
+        if (!plugin)
+            continue;
+        const skillMd = join(skillsDir, dir, "SKILL.md");
+        if (!existsSync(skillMd))
+            continue;
+        try {
+            const content = readFileSync(skillMd, "utf-8");
+            const extracted = extractFrontmatter(content);
+            if (!extracted)
+                continue;
+            const parsed = parseFrontmatter(extracted.raw);
+            if (!parsed?.description)
+                continue;
+            const driftIssue = validateDescriptionSync(dir, parsed.description, plugin.description);
+            if (driftIssue)
+                issues.push(driftIssue);
+        }
+        catch {
+            /* skip unreadable */
+        }
+    }
+    // 5. Near-duplicate descriptions across skills
+    const descriptions = marketplace.plugins.map((p) => ({ name: p.name, desc: p.description }));
+    for (let i = 0; i < descriptions.length; i++) {
+        for (let j = i + 1; j < descriptions.length; j++) {
+            const a = descriptions[i];
+            const b = descriptions[j];
+            const sim = jaccardSimilarity(a.desc, b.desc);
+            if (sim > 0.85) {
+                issues.push({
+                    level: "warning",
+                    category: "duplicate-desc",
+                    skill: a.name,
+                    detail: `Near-duplicate description with ${b.name} (${Math.round(sim * 100)}% similarity)`,
+                });
+            }
+        }
+    }
+    return issues;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sporesec/arcana",
-  "version": "3.0.0",
+  "version": "3.0.1",
   "description": "Universal AI development CLI. Skills, scaffolding, diagnostics, and analytics for every agent.",
   "bin": {
     "arcana": "dist/index.js"