@dzhechkov/harness-core 0.3.2 → 0.3.4

package/dist/benchmark.d.ts

@@ -0,0 +1,54 @@
+/**
+ * Skill benchmarking — Layer 0 deterministic checks + scoring.
+ *
+ * Runs structural validation checks against a skill directory,
+ * producing a scored report. Checks are based on BTO Layer 0
+ * (universal U1-U5 + skill-specific S1-S10).
+ *
+ * @packageDocumentation
+ */
+/** A single benchmark check result. */
+export interface BenchmarkCheck {
+    readonly id: string;
+    readonly name: string;
+    readonly passed: boolean;
+    readonly detail?: string | undefined;
+}
+/** Score for a single skill benchmark. */
+export interface BenchmarkScore {
+    readonly skillId: string;
+    readonly skillDir: string;
+    readonly checks: readonly BenchmarkCheck[];
+    readonly passed: number;
+    readonly total: number;
+    readonly passRate: number;
+    readonly grade: 'A' | 'B' | 'C' | 'D' | 'F';
+}
+/** Result of benchmarking one or more skills. */
+export interface BenchmarkReport {
+    readonly skills: readonly BenchmarkScore[];
+    readonly totalPassed: number;
+    readonly totalChecks: number;
+    readonly overallPassRate: number;
+}
+/** Result of A/B comparison between two skills. */
+export interface CompareResult {
+    readonly skillA: BenchmarkScore;
+    readonly skillB: BenchmarkScore;
+    readonly winner: string;
+    readonly deltaChecks: readonly {
+        readonly id: string;
+        readonly aPass: boolean;
+        readonly bPass: boolean;
+    }[];
+}
+/** Run Layer 0 benchmark checks on a single skill directory. */
+export declare function benchmarkSkill(skillDir: string, skillId: string): BenchmarkScore;
+/** Benchmark multiple skills. */
+export declare function benchmarkSkills(skillDirs: {
+    id: string;
+    dir: string;
+}[]): BenchmarkReport;
+/** Compare two skills head-to-head. */
+export declare function compareSkills(skillADir: string, skillAId: string, skillBDir: string, skillBId: string): CompareResult;
+//# sourceMappingURL=benchmark.d.ts.map

package/dist/benchmark.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"benchmark.d.ts","sourceRoot":"","sources":["../src/benchmark.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAKH,uCAAuC;AACvC,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACtC;AAED,0CAA0C;AAC1C,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,MAAM,EAAE,SAAS,cAAc,EAAE,CAAC;IAC3C,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,KAAK,EAAE,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,CAAC;CAC7C;AAED,iDAAiD;AACjD,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,MAAM,EAAE,SAAS,cAAc,EAAE,CAAC;IAC3C,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;CAClC;AAED,mDAAmD;AACnD,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;IAChC,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;IAChC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,WAAW,EAAE,SAAS;QAAE,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC;QAAC,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAA;KAAE,EAAE,CAAC;CAC5G;AAUD,gEAAgE;AAChE,wBAAgB,cAAc,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,cAAc,CAsGhF;AAED,iCAAiC;AACjC,wBAAgB,eAAe,CAAC,SAAS,EAAE;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,EAAE,GAAG,eAAe,CAUzF;AAED,uCAAuC;AACvC,wBAAgB,aAAa,CAC3B,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EACnC,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAClC,aAAa,CAiBf"}

package/dist/benchmark.js

@@ -0,0 +1,132 @@
+/**
+ * Skill benchmarking — Layer 0 deterministic checks + scoring.
+ *
+ * Runs structural validation checks against a skill directory,
+ * producing a scored report. Checks are based on BTO Layer 0
+ * (universal U1-U5 + skill-specific S1-S10).
+ *
+ * @packageDocumentation
+ */
+import { existsSync, readFileSync, statSync } from 'node:fs';
+import { join } from 'node:path';
+function gradeFromRate(rate) {
+    if (rate >= 90)
+        return 'A';
+    if (rate >= 80)
+        return 'B';
+    if (rate >= 70)
+        return 'C';
+    if (rate >= 60)
+        return 'D';
+    return 'F';
+}
+/** Run Layer 0 benchmark checks on a single skill directory. */
+export function benchmarkSkill(skillDir, skillId) {
+    const checks = [];
+    const skillMdPath = join(skillDir, 'SKILL.md');
+    // U1: SKILL.md exists
+    const skillMdExists = existsSync(skillMdPath);
+    checks.push({ id: 'U1', name: 'SKILL.md exists', passed: skillMdExists });
+    if (!skillMdExists) {
+        return {
+            skillId, skillDir, checks, passed: 0, total: checks.length,
+            passRate: 0, grade: 'F',
+        };
+    }
+    const content = readFileSync(skillMdPath, 'utf-8');
+    const stat = statSync(skillMdPath);
+    // U2: Valid UTF-8 (if we can read it, it's valid)
+    checks.push({ id: 'U2', name: 'valid UTF-8', passed: true });
+    // U3: Has headings
+    const hasHeadings = /^#{1,6}\s/m.test(content);
+    checks.push({ id: 'U3', name: 'has headings', passed: hasHeadings });
+    // U4: No excessive blanks (>3 consecutive blank lines)
+    const excessiveBlanks = /\n{5,}/.test(content);
+    checks.push({ id: 'U4', name: 'no excessive blanks', passed: !excessiveBlanks });
+    // U5: Size bounds (100B - 50KB)
+    const sizeOk = stat.size >= 100 && stat.size <= 50000;
+    checks.push({
+        id: 'U5', name: 'size bounds (100B-50KB)', passed: sizeOk,
+        detail: `${stat.size} bytes`,
+    });
+    // S1: Frontmatter present
+    const hasFrontmatter = content.startsWith('---');
+    checks.push({ id: 'S1', name: 'frontmatter present', passed: hasFrontmatter });
+    // S2: name field
+    const hasName = /^name:\s/m.test(content);
+    checks.push({ id: 'S2', name: 'name field', passed: hasName });
+    // S3: description field
+    const hasDescription = /^description:\s/m.test(content);
+    checks.push({ id: 'S3', name: 'description field', passed: hasDescription });
+    // S4: trust_tier field
+    const hasTrustTier = /^trust_tier:\s/m.test(content);
+    checks.push({ id: 'S4', name: 'trust_tier field', passed: hasTrustTier });
+    // S5: Protocol/procedure section
+    const hasProtocol = /^##\s+(Protocol|Procedure|Steps|When to use)/mi.test(content);
+    checks.push({ id: 'S5', name: 'protocol/procedure section', passed: hasProtocol });
+    // S6: Output section or schema reference
+    const hasOutput = /^##\s+Output/mi.test(content) || /schema_path/m.test(content);
+    checks.push({ id: 'S6', name: 'output section or schema ref', passed: hasOutput });
+    // S7: schemas/output.json exists
+    const hasSchema = existsSync(join(skillDir, 'schemas', 'output.json'));
+    checks.push({ id: 'S7', name: 'schemas/output.json exists', passed: hasSchema });
+    // S8: scripts/validate-config.json exists
+    const hasValidator = existsSync(join(skillDir, 'scripts', 'validate-config.json'));
+    checks.push({ id: 'S8', name: 'scripts/validate-config.json exists', passed: hasValidator });
+    // S9: When to use section
+    const hasWhenToUse = /^##\s+(When to use|When To Activate|Use this skill when)/mi.test(content);
+    checks.push({ id: 'S9', name: 'when-to-use section', passed: hasWhenToUse });
+    // S10: No TODO/PLACEHOLDER markers
+    const hasTodos = /\[TODO\]|\[PLACEHOLDER\]|\[TBD\]/i.test(content);
+    checks.push({ id: 'S10', name: 'no TODO/PLACEHOLDER markers', passed: !hasTodos });
+    // S11: Evals exist
+    const hasEvals = existsSync(join(skillDir, 'evals'));
+    checks.push({ id: 'S11', name: 'evals directory exists', passed: hasEvals });
+    // S12: Content depth (>20 lines)
+    const lineCount = content.split('\n').length;
+    checks.push({
+        id: 'S12', name: 'content depth (>20 lines)', passed: lineCount > 20,
+        detail: `${lineCount} lines`,
+    });
+    // S13: Examples section
+    const hasExamples = /^##\s+(Examples|Example|Quick Start)/mi.test(content);
+    checks.push({ id: 'S13', name: 'examples section', passed: hasExamples });
+    // S14: Anti-patterns or self-check section
+    const hasAntiPatterns = /^##\s+(Anti.?[Pp]atterns|Self.?check|Self-check)/mi.test(content);
+    checks.push({ id: 'S14', name: 'anti-patterns or self-check', passed: hasAntiPatterns });
+    const passed = checks.filter((c) => c.passed).length;
+    const passRate = Math.round((passed / checks.length) * 100);
+    return {
+        skillId, skillDir, checks, passed, total: checks.length,
+        passRate, grade: gradeFromRate(passRate),
+    };
+}
+/** Benchmark multiple skills. */
+export function benchmarkSkills(skillDirs) {
+    const skills = skillDirs.map((s) => benchmarkSkill(s.dir, s.id));
+    const totalPassed = skills.reduce((sum, s) => sum + s.passed, 0);
+    const totalChecks = skills.reduce((sum, s) => sum + s.total, 0);
+    return {
+        skills,
+        totalPassed,
+        totalChecks,
+        overallPassRate: totalChecks > 0 ? Math.round((totalPassed / totalChecks) * 100) : 0,
+    };
+}
+/** Compare two skills head-to-head. */
+export function compareSkills(skillADir, skillAId, skillBDir, skillBId) {
+    const skillA = benchmarkSkill(skillADir, skillAId);
+    const skillB = benchmarkSkill(skillBDir, skillBId);
+    const deltaChecks = [];
+    for (const checkA of skillA.checks) {
+        const checkB = skillB.checks.find((c) => c.id === checkA.id);
+        if (checkB && checkA.passed !== checkB.passed) {
+            deltaChecks.push({ id: checkA.id, aPass: checkA.passed, bPass: checkB.passed });
+        }
+    }
+    const winner = skillA.passRate > skillB.passRate ? skillAId
+        : skillB.passRate > skillA.passRate ? skillBId
+            : 'tie';
+    return { skillA, skillB, winner, deltaChecks };
+}
+//# sourceMappingURL=benchmark.js.map

package/dist/benchmark.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"benchmark.js","sourceRoot":"","sources":["../src/benchmark.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAC7D,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAqCjC,SAAS,aAAa,CAAC,IAAY;IACjC,IAAI,IAAI,IAAI,EAAE;QAAE,OAAO,GAAG,CAAC;IAC3B,IAAI,IAAI,IAAI,EAAE;QAAE,OAAO,GAAG,CAAC;IAC3B,IAAI,IAAI,IAAI,EAAE;QAAE,OAAO,GAAG,CAAC;IAC3B,IAAI,IAAI,IAAI,EAAE;QAAE,OAAO,GAAG,CAAC;IAC3B,OAAO,GAAG,CAAC;AACb,CAAC;AAED,gEAAgE;AAChE,MAAM,UAAU,cAAc,CAAC,QAAgB,EAAE,OAAe;IAC9D,MAAM,MAAM,GAAqB,EAAE,CAAC;IACpC,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IAE/C,sBAAsB;IACtB,MAAM,aAAa,GAAG,UAAU,CAAC,WAAW,CAAC,CAAC;IAC9C,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,iBAAiB,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;IAE1E,IAAI,CAAC,aAAa,EAAE,CAAC;QACnB,OAAO;YACL,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM;YAC1D,QAAQ,EAAE,CAAC,EAAE,KAAK,EAAE,GAAG;SACxB,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAG,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;IACnD,MAAM,IAAI,GAAG,QAAQ,CAAC,WAAW,CAAC,CAAC;IAEnC,kDAAkD;IAClD,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;IAE7D,mBAAmB;IACnB,MAAM,WAAW,GAAG,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC;IAErE,uDAAuD;IACvD,MAAM,eAAe,GAAG,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,qBAAqB,EAAE,MAAM,EAAE,CAAC,eAAe,EAAE,CAAC,CAAC;IAEjF,gCAAgC;IAChC,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,IAAI,GAAG,IAAI,IAAI,CAAC,IAAI,IAAI,KAAK,CAAC;IACtD,MAAM,CAAC,IAAI,CAAC;QACV,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,yBAAyB,EAAE,MAAM,EAAE,MAAM;QACzD,MAAM,EAAE,GAAG,IAAI,CAAC,IAAI,QAAQ;KAC7B,CAAC,CAAC;IAEH,0BAA0B;IAC1B,MAAM,cAAc,GAAG,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;IACjD,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,qBAAqB,EAAE,MAAM,EAAE,cAAc,EAAE,CAAC,CAAC;IAE/E,iBAAiB;IACjB,MAAM,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC1C,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;IAE/D,wBAAwB;IACxB,MAAM,cAAc,GAAG,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACxD,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,mBAAmB,EAAE,MAAM,EAAE,cAAc,EAAE,CAAC,CAAC;IAE7E,uBAAuB;IACvB,MAAM,YAAY,GAAG,iBAAiB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACrD,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,kBAAkB,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,CAAC;IAE1E,iCAAiC;IACjC,MAAM,WAAW,GAAG,gDAAgD,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACnF,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,4BAA4B,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC;IAEnF,yCAAyC;IACzC,MAAM,SAAS,GAAG,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACjF,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,8BAA8B,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;IAEnF,iCAAiC;IACjC,MAAM,SAAS,GAAG,UAAU,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC,CAAC;IACvE,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,4BAA4B,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;IAEjF,0CAA0C;IAC1C,MAAM,YAAY,GAAG,UAAU,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,EAAE,sBAAsB,CAAC,CAAC,CAAC;IACnF,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,qCAAqC,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,CAAC;IAE7F,0BAA0B;IAC1B,MAAM,YAAY,GAAG,4DAA4D,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAChG,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,qBAAqB,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,CAAC;IAE7E,mCAAmC;IACnC,MAAM,QAAQ,GAAG,mCAAmC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACnE,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,6BAA6B,EAAE,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,CAAC;IAEnF,mBAAmB;IACnB,MAAM,QAAQ,GAAG,UAAU,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;IACrD,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,wBAAwB,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;IAE7E,iCAAiC;IACjC,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;IAC7C,MAAM,CAAC,IAAI,CAAC;QACV,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,2BAA2B,EAAE,MAAM,EAAE,SAAS,GAAG,EAAE;QACpE,MAAM,EAAE,GAAG,SAAS,QAAQ;KAC7B,CAAC,CAAC;IAEH,wBAAwB;IACxB,MAAM,WAAW,GAAG,wCAAwC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC3E,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,kBAAkB,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC;IAE1E,2CAA2C;IAC3C,MAAM,eAAe,GAAG,oDAAoD,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC3F,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,6BAA6B,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC,CAAC;IAEzF,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IACrD,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC;IAE5D,OAAO;QACL,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM;QACvD,QAAQ,EAAE,KAAK,EAAE,aAAa,CAAC,QAAQ,CAAC;KACzC,CAAC;AACJ,CAAC;AAED,iCAAiC;AACjC,MAAM,UAAU,eAAe,CAAC,SAAwC;IACtE,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACjE,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACjE,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IAChE,OAAO;QACL,MAAM;QACN,WAAW;QACX,WAAW;QACX,eAAe,EAAE,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,WAAW,GAAG,WAAW,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;KACrF,CAAC;AACJ,CAAC;AAED,uCAAuC;AACvC,MAAM,UAAU,aAAa,CAC3B,SAAiB,EAAE,QAAgB,EACnC,SAAiB,EAAE,QAAgB;IAEnC,MAAM,MAAM,GAAG,cAAc,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IACnD,MAAM,MAAM,GAAG,cAAc,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IAEnD,MAAM,WAAW,GAAqD,EAAE,CAAC;IACzE,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QACnC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,MAAM,CAAC,EAAE,CAAC,CAAC;QAC7D,IAAI,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC;YAC9C,WAAW,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QAClF,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;QACzD,CAAC,CAAC,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;YAC9C,CAAC,CAAC,KAAK,CAAC;IAEV,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;AACjD,CAAC"}

package/dist/index.d.ts

@@ -12,4 +12,8 @@ export * from './operations.js';
 export * from './workflows.js';
 export { createSkill } from './create-skill.js';
 export type { CreateSkillOptions, CreateSkillResult } from './create-skill.js';
+export { checkUpstream, loadSourcesManifest } from './sync-upstream.js';
+export type { SyncUpstreamReport, UpstreamCheckResult, SourcesManifest } from './sync-upstream.js';
+export { benchmarkSkill, benchmarkSkills, compareSkills } from './benchmark.js';
+export type { BenchmarkCheck, BenchmarkScore, BenchmarkReport, CompareResult } from './benchmark.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,yDAAyD;AACzD,eAAO,MAAM,oBAAoB,UAAU,CAAC;AAE5C,cAAc,aAAa,CAAC;AAC5B,cAAc,YAAY,CAAC;AAC3B,cAAc,cAAc,CAAC;AAC7B,cAAc,iBAAiB,CAAC;AAChC,cAAc,gBAAgB,CAAC;AAC/B,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,YAAY,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,yDAAyD;AACzD,eAAO,MAAM,oBAAoB,UAAU,CAAC;AAE5C,cAAc,aAAa,CAAC;AAC5B,cAAc,YAAY,CAAC;AAC3B,cAAc,cAAc,CAAC;AAC7B,cAAc,iBAAiB,CAAC;AAChC,cAAc,gBAAgB,CAAC;AAC/B,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,YAAY,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAC/E,OAAO,EAAE,aAAa,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACxE,YAAY,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACnG,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAChF,YAAY,EAAE,cAAc,EAAE,cAAc,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC"}

package/dist/index.js

@@ -11,4 +11,6 @@ export * from './targets.js';
 export * from './operations.js';
 export * from './workflows.js';
 export { createSkill } from './create-skill.js';
+export { checkUpstream, loadSourcesManifest } from './sync-upstream.js';
+export { benchmarkSkill, benchmarkSkills, compareSkills } from './benchmark.js';
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,yDAAyD;AACzD,MAAM,CAAC,MAAM,oBAAoB,GAAG,OAAO,CAAC;AAE5C,cAAc,aAAa,CAAC;AAC5B,cAAc,YAAY,CAAC;AAC3B,cAAc,cAAc,CAAC;AAC7B,cAAc,iBAAiB,CAAC;AAChC,cAAc,gBAAgB,CAAC;AAC/B,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,yDAAyD;AACzD,MAAM,CAAC,MAAM,oBAAoB,GAAG,OAAO,CAAC;AAE5C,cAAc,aAAa,CAAC;AAC5B,cAAc,YAAY,CAAC;AAC3B,cAAc,cAAc,CAAC;AAC7B,cAAc,iBAAiB,CAAC;AAChC,cAAc,gBAAgB,CAAC;AAC/B,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,OAAO,EAAE,aAAa,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAExE,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC"}

package/dist/sync-upstream.d.ts

@@ -0,0 +1,53 @@
+/**
+ * Upstream sync — fetches SKILL.md from origin repos defined in sources.json
+ * and reports which skills have upstream changes.
+ *
+ * @packageDocumentation
+ */
+/** A single skill's source mapping. */
+export interface SkillSource {
+    readonly path: string;
+    readonly version?: string | undefined;
+    readonly trust?: string | undefined;
+    readonly note?: string | undefined;
+}
+/** The origin repo metadata. */
+export interface OriginMeta {
+    readonly repo: string;
+    readonly url: string;
+    readonly branch: string;
+    readonly license: string;
+    readonly author: string;
+    readonly canonicalized: string;
+}
+/** The sources.json schema. */
+export interface SourcesManifest {
+    readonly origin: OriginMeta;
+    readonly skills: Record<string, SkillSource>;
+}
+/** Result of checking one skill against upstream. */
+export interface UpstreamCheckResult {
+    readonly skillId: string;
+    readonly originPath: string;
+    readonly status: 'up-to-date' | 'changed' | 'fetch-error' | 'local-missing';
+    readonly localLines?: number | undefined;
+    readonly upstreamLines?: number | undefined;
+    readonly error?: string | undefined;
+}
+/** Result of a full upstream sync check. */
+export interface SyncUpstreamReport {
+    readonly origin: OriginMeta;
+    readonly checked: number;
+    readonly changed: number;
+    readonly upToDate: number;
+    readonly errors: number;
+    readonly skills: readonly UpstreamCheckResult[];
+}
+/** Load sources.json from a skills package directory. */
+export declare function loadSourcesManifest(packageDir: string): SourcesManifest | undefined;
+/**
+ * Check all skills in a sources.json manifest against their upstream.
+ * Compares line counts as a quick diff indicator (full diff would be too verbose).
+ */
+export declare function checkUpstream(packageDir: string): Promise<SyncUpstreamReport | undefined>;
+//# sourceMappingURL=sync-upstream.d.ts.map

package/dist/sync-upstream.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sync-upstream.d.ts","sourceRoot":"","sources":["../src/sync-upstream.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH,uCAAuC;AACvC,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACtC,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACpC,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACpC;AAED,gCAAgC;AAChC,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;CAChC;AAED,+BAA+B;AAC/B,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;CAC9C;AAED,qDAAqD;AACrD,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,YAAY,GAAG,SAAS,GAAG,aAAa,GAAG,eAAe,CAAC;IAC5E,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACzC,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5C,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACrC;AAED,4CAA4C;AAC5C,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAC;IAC5B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,MAAM,EAAE,SAAS,mBAAmB,EAAE,CAAC;CACjD;AAED,yDAAyD;AACzD,wBAAgB,mBAAmB,CAAC,UAAU,EAAE,MAAM,GAAG,eAAe,GAAG,SAAS,CAKnF;AA6BD;;;GAGG;AACH,wBAAsB,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,GAAG,SAAS,CAAC,CAiD/F"}

package/dist/sync-upstream.js

@@ -0,0 +1,89 @@
+/**
+ * Upstream sync — fetches SKILL.md from origin repos defined in sources.json
+ * and reports which skills have upstream changes.
+ *
+ * @packageDocumentation
+ */
+import { existsSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+/** Load sources.json from a skills package directory. */
+export function loadSourcesManifest(packageDir) {
+    const sourcesPath = join(packageDir, 'sources.json');
+    if (!existsSync(sourcesPath))
+        return undefined;
+    const raw = readFileSync(sourcesPath, 'utf-8');
+    return JSON.parse(raw);
+}
+/** Build the raw GitHub URL for a skill's SKILL.md. */
+function rawUrl(origin, skillPath) {
+    return `https://raw.githubusercontent.com/${origin.repo}/${origin.branch}/${skillPath}`;
+}
+/** Fetch text content from a URL with timeout. */
+async function fetchText(url, timeoutMs = 10000) {
+    try {
+        const controller = new AbortController();
+        const timer = setTimeout(() => { controller.abort(); }, timeoutMs);
+        const response = await fetch(url, { signal: controller.signal });
+        clearTimeout(timer);
+        if (!response.ok) {
+            return { ok: false, text: '', error: `HTTP ${response.status}` };
+        }
+        const text = await response.text();
+        return { ok: true, text };
+    }
+    catch (err) {
+        return { ok: false, text: '', error: err instanceof Error ? err.message : String(err) };
+    }
+}
+/** Count non-empty lines in text. */
+function lineCount(text) {
+    return text.split('\n').length;
+}
+/**
+ * Check all skills in a sources.json manifest against their upstream.
+ * Compares line counts as a quick diff indicator (full diff would be too verbose).
+ */
+export async function checkUpstream(packageDir) {
+    const manifest = loadSourcesManifest(packageDir);
+    if (!manifest)
+        return undefined;
+    const results = [];
+    for (const [skillId, source] of Object.entries(manifest.skills)) {
+        const localSkillMd = join(packageDir, skillId, 'SKILL.md');
+        if (!existsSync(localSkillMd)) {
+            results.push({ skillId, originPath: source.path, status: 'local-missing' });
+            continue;
+        }
+        const localText = readFileSync(localSkillMd, 'utf-8');
+        const url = rawUrl(manifest.origin, source.path);
+        const upstream = await fetchText(url);
+        if (!upstream.ok) {
+            results.push({
+                skillId,
+                originPath: source.path,
+                status: 'fetch-error',
+                error: upstream.error,
+            });
+            continue;
+        }
+        const localLines = lineCount(localText);
+        const upstreamLines = lineCount(upstream.text);
+        const changed = localLines !== upstreamLines;
+        results.push({
+            skillId,
+            originPath: source.path,
+            status: changed ? 'changed' : 'up-to-date',
+            localLines,
+            upstreamLines,
+        });
+    }
+    return {
+        origin: manifest.origin,
+        checked: results.length,
+        changed: results.filter((r) => r.status === 'changed').length,
+        upToDate: results.filter((r) => r.status === 'up-to-date').length,
+        errors: results.filter((r) => r.status === 'fetch-error' || r.status === 'local-missing').length,
+        skills: results,
+    };
+}
+//# sourceMappingURL=sync-upstream.js.map

package/dist/sync-upstream.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sync-upstream.js","sourceRoot":"","sources":["../src/sync-upstream.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AA8CjC,yDAAyD;AACzD,MAAM,UAAU,mBAAmB,CAAC,UAAkB;IACpD,MAAM,WAAW,GAAG,IAAI,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;IACrD,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC;QAAE,OAAO,SAAS,CAAC;IAC/C,MAAM,GAAG,GAAG,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;IAC/C,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAoB,CAAC;AAC5C,CAAC;AAED,uDAAuD;AACvD,SAAS,MAAM,CAAC,MAAkB,EAAE,SAAiB;IACnD,OAAO,qCAAqC,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;AAC1F,CAAC;AAED,kDAAkD;AAClD,KAAK,UAAU,SAAS,CAAC,GAAW,EAAE,YAAoB,KAAK;IAC7D,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,GAAG,UAAU,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;QACnE,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;QACjE,YAAY,CAAC,KAAK,CAAC,CAAC;QACpB,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,EAAE,KAAK,EAAE,QAAQ,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;QACnE,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnC,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IAC5B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;IAC1F,CAAC;AACH,CAAC;AAED,qCAAqC;AACrC,SAAS,SAAS,CAAC,IAAY;IAC7B,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;AACjC,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,UAAkB;IACpD,MAAM,QAAQ,GAAG,mBAAmB,CAAC,UAAU,CAAC,CAAC;IACjD,IAAI,CAAC,QAAQ;QAAE,OAAO,SAAS,CAAC;IAEhC,MAAM,OAAO,GAA0B,EAAE,CAAC;IAE1C,KAAK,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QAChE,MAAM,YAAY,GAAG,IAAI,CAAC,UAAU,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;QAE3D,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;YAC9B,OAAO,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC,CAAC;YAC5E,SAAS;QACX,CAAC;QAED,MAAM,SAAS,GAAG,YAAY,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;QACtD,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;QACjD,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,CAAC;QAEtC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,OAAO,CAAC,IAAI,CAAC;gBACX,OAAO;gBACP,UAAU,EAAE,MAAM,CAAC,IAAI;gBACvB,MAAM,EAAE,aAAa;gBACrB,KAAK,EAAE,QAAQ,CAAC,KAAK;aACtB,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QAED,MAAM,UAAU,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC;QACxC,MAAM,aAAa,GAAG,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC/C,MAAM,OAAO,GAAG,UAAU,KAAK,aAAa,CAAC;QAE7C,OAAO,CAAC,IAAI,CAAC;YACX,OAAO;YACP,UAAU,EAAE,MAAM,CAAC,IAAI;YACvB,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY;YAC1C,UAAU;YACV,aAAa;SACd,CAAC,CAAC;IACL,CAAC;IAED,OAAO;QACL,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,OAAO,EAAE,OAAO,CAAC,MAAM;QACvB,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,MAAM;QAC7D,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,YAAY,CAAC,CAAC,MAAM;QACjE,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,aAAa,IAAI,CAAC,CAAC,MAAM,KAAK,eAAe,CAAC,CAAC,MAAM;QAChG,MAAM,EAAE,OAAO;KAChB,CAAC;AACJ,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dzhechkov/harness-core",
-  "version": "0.3.2",
+  "version": "0.3.4",
   "description": "Shared harness logic - skill loading, additive apply, and the init/sync/verify/doctor operations.",
   "type": "module",
   "license": "MIT",
@@ -25,11 +25,11 @@
   ],
   "dependencies": {
     "yaml": "^2.0.0",
-    "@dzhechkov/adapter-claude": "0.2.0",
     "@dzhechkov/adapter-hermes": "0.2.0",
     "@dzhechkov/adapter-opencode": "0.2.0",
-    "@dzhechkov/adapter-codex": "0.2.0",
+    "@dzhechkov/adapter-claude": "0.2.0",
     "@dzhechkov/core": "0.2.0",
+    "@dzhechkov/adapter-codex": "0.2.0",
     "@dzhechkov/adapter-openclaude": "0.1.0"
   },
   "devDependencies": {

package/src/benchmark.ts

@@ -0,0 +1,196 @@
+/**
+ * Skill benchmarking — Layer 0 deterministic checks + scoring.
+ *
+ * Runs structural validation checks against a skill directory,
+ * producing a scored report. Checks are based on BTO Layer 0
+ * (universal U1-U5 + skill-specific S1-S10).
+ *
+ * @packageDocumentation
+ */
+
+import { existsSync, readFileSync, statSync } from 'node:fs';
+import { join } from 'node:path';
+
+/** A single benchmark check result. */
+export interface BenchmarkCheck {
+  readonly id: string;
+  readonly name: string;
+  readonly passed: boolean;
+  readonly detail?: string | undefined;
+}
+
+/** Score for a single skill benchmark. */
+export interface BenchmarkScore {
+  readonly skillId: string;
+  readonly skillDir: string;
+  readonly checks: readonly BenchmarkCheck[];
+  readonly passed: number;
+  readonly total: number;
+  readonly passRate: number;
+  readonly grade: 'A' | 'B' | 'C' | 'D' | 'F';
+}
+
+/** Result of benchmarking one or more skills. */
+export interface BenchmarkReport {
+  readonly skills: readonly BenchmarkScore[];
+  readonly totalPassed: number;
+  readonly totalChecks: number;
+  readonly overallPassRate: number;
+}
+
+/** Result of A/B comparison between two skills. */
+export interface CompareResult {
+  readonly skillA: BenchmarkScore;
+  readonly skillB: BenchmarkScore;
+  readonly winner: string;
+  readonly deltaChecks: readonly { readonly id: string; readonly aPass: boolean; readonly bPass: boolean }[];
+}
+
+function gradeFromRate(rate: number): 'A' | 'B' | 'C' | 'D' | 'F' {
+  if (rate >= 90) return 'A';
+  if (rate >= 80) return 'B';
+  if (rate >= 70) return 'C';
+  if (rate >= 60) return 'D';
+  return 'F';
+}
+
+/** Run Layer 0 benchmark checks on a single skill directory. */
+export function benchmarkSkill(skillDir: string, skillId: string): BenchmarkScore {
+  const checks: BenchmarkCheck[] = [];
+  const skillMdPath = join(skillDir, 'SKILL.md');
+
+  // U1: SKILL.md exists
+  const skillMdExists = existsSync(skillMdPath);
+  checks.push({ id: 'U1', name: 'SKILL.md exists', passed: skillMdExists });
+
+  if (!skillMdExists) {
+    return {
+      skillId, skillDir, checks, passed: 0, total: checks.length,
+      passRate: 0, grade: 'F',
+    };
+  }
+
+  const content = readFileSync(skillMdPath, 'utf-8');
+  const stat = statSync(skillMdPath);
+
+  // U2: Valid UTF-8 (if we can read it, it's valid)
+  checks.push({ id: 'U2', name: 'valid UTF-8', passed: true });
+
+  // U3: Has headings
+  const hasHeadings = /^#{1,6}\s/m.test(content);
+  checks.push({ id: 'U3', name: 'has headings', passed: hasHeadings });
+
+  // U4: No excessive blanks (>3 consecutive blank lines)
+  const excessiveBlanks = /\n{5,}/.test(content);
+  checks.push({ id: 'U4', name: 'no excessive blanks', passed: !excessiveBlanks });
+
+  // U5: Size bounds (100B - 50KB)
+  const sizeOk = stat.size >= 100 && stat.size <= 50000;
+  checks.push({
+    id: 'U5', name: 'size bounds (100B-50KB)', passed: sizeOk,
+    detail: `${stat.size} bytes`,
+  });
+
+  // S1: Frontmatter present
+  const hasFrontmatter = content.startsWith('---');
+  checks.push({ id: 'S1', name: 'frontmatter present', passed: hasFrontmatter });
+
+  // S2: name field
+  const hasName = /^name:\s/m.test(content);
+  checks.push({ id: 'S2', name: 'name field', passed: hasName });
+
+  // S3: description field
+  const hasDescription = /^description:\s/m.test(content);
+  checks.push({ id: 'S3', name: 'description field', passed: hasDescription });
+
+  // S4: trust_tier field
+  const hasTrustTier = /^trust_tier:\s/m.test(content);
+  checks.push({ id: 'S4', name: 'trust_tier field', passed: hasTrustTier });
+
+  // S5: Protocol/procedure section
+  const hasProtocol = /^##\s+(Protocol|Procedure|Steps|When to use)/mi.test(content);
+  checks.push({ id: 'S5', name: 'protocol/procedure section', passed: hasProtocol });
+
+  // S6: Output section or schema reference
+  const hasOutput = /^##\s+Output/mi.test(content) || /schema_path/m.test(content);
+  checks.push({ id: 'S6', name: 'output section or schema ref', passed: hasOutput });
+
+  // S7: schemas/output.json exists
+  const hasSchema = existsSync(join(skillDir, 'schemas', 'output.json'));
+  checks.push({ id: 'S7', name: 'schemas/output.json exists', passed: hasSchema });
+
+  // S8: scripts/validate-config.json exists
+  const hasValidator = existsSync(join(skillDir, 'scripts', 'validate-config.json'));
+  checks.push({ id: 'S8', name: 'scripts/validate-config.json exists', passed: hasValidator });
+
+  // S9: When to use section
+  const hasWhenToUse = /^##\s+(When to use|When To Activate|Use this skill when)/mi.test(content);
+  checks.push({ id: 'S9', name: 'when-to-use section', passed: hasWhenToUse });
+
+  // S10: No TODO/PLACEHOLDER markers
+  const hasTodos = /\[TODO\]|\[PLACEHOLDER\]|\[TBD\]/i.test(content);
+  checks.push({ id: 'S10', name: 'no TODO/PLACEHOLDER markers', passed: !hasTodos });
+
+  // S11: Evals exist
+  const hasEvals = existsSync(join(skillDir, 'evals'));
+  checks.push({ id: 'S11', name: 'evals directory exists', passed: hasEvals });
+
+  // S12: Content depth (>20 lines)
+  const lineCount = content.split('\n').length;
+  checks.push({
+    id: 'S12', name: 'content depth (>20 lines)', passed: lineCount > 20,
+    detail: `${lineCount} lines`,
+  });
+
+  // S13: Examples section
+  const hasExamples = /^##\s+(Examples|Example|Quick Start)/mi.test(content);
+  checks.push({ id: 'S13', name: 'examples section', passed: hasExamples });
+
+  // S14: Anti-patterns or self-check section
+  const hasAntiPatterns = /^##\s+(Anti.?[Pp]atterns|Self.?check|Self-check)/mi.test(content);
+  checks.push({ id: 'S14', name: 'anti-patterns or self-check', passed: hasAntiPatterns });
+
+  const passed = checks.filter((c) => c.passed).length;
+  const passRate = Math.round((passed / checks.length) * 100);
+
+  return {
+    skillId, skillDir, checks, passed, total: checks.length,
+    passRate, grade: gradeFromRate(passRate),
+  };
+}
+
+/** Benchmark multiple skills. */
+export function benchmarkSkills(skillDirs: { id: string; dir: string }[]): BenchmarkReport {
+  const skills = skillDirs.map((s) => benchmarkSkill(s.dir, s.id));
+  const totalPassed = skills.reduce((sum, s) => sum + s.passed, 0);
+  const totalChecks = skills.reduce((sum, s) => sum + s.total, 0);
+  return {
+    skills,
+    totalPassed,
+    totalChecks,
+    overallPassRate: totalChecks > 0 ? Math.round((totalPassed / totalChecks) * 100) : 0,
+  };
+}
+
+/** Compare two skills head-to-head. */
+export function compareSkills(
+  skillADir: string, skillAId: string,
+  skillBDir: string, skillBId: string,
+): CompareResult {
+  const skillA = benchmarkSkill(skillADir, skillAId);
+  const skillB = benchmarkSkill(skillBDir, skillBId);
+
+  const deltaChecks: { id: string; aPass: boolean; bPass: boolean }[] = [];
+  for (const checkA of skillA.checks) {
+    const checkB = skillB.checks.find((c) => c.id === checkA.id);
+    if (checkB && checkA.passed !== checkB.passed) {
+      deltaChecks.push({ id: checkA.id, aPass: checkA.passed, bPass: checkB.passed });
+    }
+  }
+
+  const winner = skillA.passRate > skillB.passRate ? skillAId
+    : skillB.passRate > skillA.passRate ? skillBId
+    : 'tie';
+
+  return { skillA, skillB, winner, deltaChecks };
+}

package/src/index.ts

@@ -14,3 +14,7 @@ export * from './operations.js';
 export * from './workflows.js';
 export { createSkill } from './create-skill.js';
 export type { CreateSkillOptions, CreateSkillResult } from './create-skill.js';
+export { checkUpstream, loadSourcesManifest } from './sync-upstream.js';
+export type { SyncUpstreamReport, UpstreamCheckResult, SourcesManifest } from './sync-upstream.js';
+export { benchmarkSkill, benchmarkSkills, compareSkills } from './benchmark.js';
+export type { BenchmarkCheck, BenchmarkScore, BenchmarkReport, CompareResult } from './benchmark.js';

package/src/sync-upstream.ts

@@ -0,0 +1,143 @@
+/**
+ * Upstream sync — fetches SKILL.md from origin repos defined in sources.json
+ * and reports which skills have upstream changes.
+ *
+ * @packageDocumentation
+ */
+
+import { existsSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+
+/** A single skill's source mapping. */
+export interface SkillSource {
+  readonly path: string;
+  readonly version?: string | undefined;
+  readonly trust?: string | undefined;
+  readonly note?: string | undefined;
+}
+
+/** The origin repo metadata. */
+export interface OriginMeta {
+  readonly repo: string;
+  readonly url: string;
+  readonly branch: string;
+  readonly license: string;
+  readonly author: string;
+  readonly canonicalized: string;
+}
+
+/** The sources.json schema. */
+export interface SourcesManifest {
+  readonly origin: OriginMeta;
+  readonly skills: Record<string, SkillSource>;
+}
+
+/** Result of checking one skill against upstream. */
+export interface UpstreamCheckResult {
+  readonly skillId: string;
+  readonly originPath: string;
+  readonly status: 'up-to-date' | 'changed' | 'fetch-error' | 'local-missing';
+  readonly localLines?: number | undefined;
+  readonly upstreamLines?: number | undefined;
+  readonly error?: string | undefined;
+}
+
+/** Result of a full upstream sync check. */
+export interface SyncUpstreamReport {
+  readonly origin: OriginMeta;
+  readonly checked: number;
+  readonly changed: number;
+  readonly upToDate: number;
+  readonly errors: number;
+  readonly skills: readonly UpstreamCheckResult[];
+}
+
+/** Load sources.json from a skills package directory. */
+export function loadSourcesManifest(packageDir: string): SourcesManifest | undefined {
+  const sourcesPath = join(packageDir, 'sources.json');
+  if (!existsSync(sourcesPath)) return undefined;
+  const raw = readFileSync(sourcesPath, 'utf-8');
+  return JSON.parse(raw) as SourcesManifest;
+}
+
+/** Build the raw GitHub URL for a skill's SKILL.md. */
+function rawUrl(origin: OriginMeta, skillPath: string): string {
+  return `https://raw.githubusercontent.com/${origin.repo}/${origin.branch}/${skillPath}`;
+}
+
+/** Fetch text content from a URL with timeout. */
+async function fetchText(url: string, timeoutMs: number = 10000): Promise<{ ok: boolean; text: string; error?: string }> {
+  try {
+    const controller = new AbortController();
+    const timer = setTimeout(() => { controller.abort(); }, timeoutMs);
+    const response = await fetch(url, { signal: controller.signal });
+    clearTimeout(timer);
+    if (!response.ok) {
+      return { ok: false, text: '', error: `HTTP ${response.status}` };
+    }
+    const text = await response.text();
+    return { ok: true, text };
+  } catch (err) {
+    return { ok: false, text: '', error: err instanceof Error ? err.message : String(err) };
+  }
+}
+
+/** Count non-empty lines in text. */
+function lineCount(text: string): number {
+  return text.split('\n').length;
+}
+
+/**
+ * Check all skills in a sources.json manifest against their upstream.
+ * Compares line counts as a quick diff indicator (full diff would be too verbose).
+ */
+export async function checkUpstream(packageDir: string): Promise<SyncUpstreamReport | undefined> {
+  const manifest = loadSourcesManifest(packageDir);
+  if (!manifest) return undefined;
+
+  const results: UpstreamCheckResult[] = [];
+
+  for (const [skillId, source] of Object.entries(manifest.skills)) {
+    const localSkillMd = join(packageDir, skillId, 'SKILL.md');
+
+    if (!existsSync(localSkillMd)) {
+      results.push({ skillId, originPath: source.path, status: 'local-missing' });
+      continue;
+    }
+
+    const localText = readFileSync(localSkillMd, 'utf-8');
+    const url = rawUrl(manifest.origin, source.path);
+    const upstream = await fetchText(url);
+
+    if (!upstream.ok) {
+      results.push({
+        skillId,
+        originPath: source.path,
+        status: 'fetch-error',
+        error: upstream.error,
+      });
+      continue;
+    }
+
+    const localLines = lineCount(localText);
+    const upstreamLines = lineCount(upstream.text);
+    const changed = localLines !== upstreamLines;
+
+    results.push({
+      skillId,
+      originPath: source.path,
+      status: changed ? 'changed' : 'up-to-date',
+      localLines,
+      upstreamLines,
+    });
+  }
+
+  return {
+    origin: manifest.origin,
+    checked: results.length,
+    changed: results.filter((r) => r.status === 'changed').length,
+    upToDate: results.filter((r) => r.status === 'up-to-date').length,
+    errors: results.filter((r) => r.status === 'fetch-error' || r.status === 'local-missing').length,
+    skills: results,
+  };
+}