@skillmark/cli 0.1.0

package/dist/types/benchmark-types.d.ts

@@ -0,0 +1,142 @@
+/**
+ * Core types for Skillmark benchmark system
+ */
+/** Test definition parsed from markdown frontmatter */
+export interface TestDefinition {
+    /** Unique test identifier */
+    name: string;
+    /** Type of test: knowledge (Q&A) or task (execution) */
+    type: 'knowledge' | 'task';
+    /** Concepts to check in response */
+    concepts: string[];
+    /** Timeout in seconds */
+    timeout: number;
+    /** The prompt to send to Claude */
+    prompt: string;
+    /** Expected response patterns/criteria */
+    expected: string[];
+    /** Source file path */
+    sourcePath: string;
+}
+/** Metrics collected from a single benchmark run */
+export interface BenchmarkMetrics {
+    /** Accuracy percentage (concepts matched / total) */
+    accuracy: number;
+    /** Total tokens consumed */
+    tokensTotal: number;
+    /** Input tokens */
+    tokensInput: number;
+    /** Output tokens */
+    tokensOutput: number;
+    /** Wall-clock time in milliseconds */
+    durationMs: number;
+    /** Number of tool calls made */
+    toolCount: number;
+    /** Estimated API cost in USD */
+    costUsd: number;
+}
+/** Result of a single test execution */
+export interface TestResult {
+    /** Test definition */
+    test: TestDefinition;
+    /** Collected metrics */
+    metrics: BenchmarkMetrics;
+    /** Concepts that were matched */
+    matchedConcepts: string[];
+    /** Concepts that were missed */
+    missedConcepts: string[];
+    /** Claude's response */
+    response: string;
+    /** Run timestamp */
+    timestamp: string;
+    /** Whether the test passed accuracy threshold */
+    passed: boolean;
+}
+/** Aggregated results across multiple runs */
+export interface BenchmarkResult {
+    /** Skill identifier */
+    skillId: string;
+    /** Skill name */
+    skillName: string;
+    /** Skill source (local path, git URL, or skill.sh path) */
+    skillSource: string;
+    /** Model used for benchmark */
+    model: 'haiku' | 'sonnet' | 'opus';
+    /** Number of runs performed */
+    runs: number;
+    /** Individual test results */
+    testResults: TestResult[];
+    /** Aggregated metrics (averaged across runs) */
+    aggregatedMetrics: BenchmarkMetrics;
+    /** Benchmark timestamp */
+    timestamp: string;
+    /** Skillmark CLI version */
+    version: string;
+    /** Hash for result verification */
+    hash?: string;
+}
+/** Skill source types */
+export type SkillSourceType = 'local' | 'git' | 'skillsh';
+/** Parsed skill source information */
+export interface SkillSource {
+    /** Source type */
+    type: SkillSourceType;
+    /** Original input (path, URL, or skill.sh identifier) */
+    original: string;
+    /** Resolved local path (after clone if needed) */
+    localPath: string;
+    /** Skill name extracted from source */
+    name: string;
+}
+/** CLI run command options */
+export interface RunOptions {
+    /** Path to test suite */
+    tests: string;
+    /** Model to use */
+    model: 'haiku' | 'sonnet' | 'opus';
+    /** Number of iterations */
+    runs: number;
+    /** Output directory */
+    output: string;
+    /** Show verbose progress output */
+    verbose?: boolean;
+}
+/** CLI publish command options */
+export interface PublishOptions {
+    /** API key for authentication */
+    apiKey: string;
+    /** API endpoint URL */
+    endpoint?: string;
+}
+/** JSONL transcript entry from Claude CLI */
+export interface TranscriptEntry {
+    type: 'system' | 'human' | 'assistant' | 'result';
+    message?: {
+        role: string;
+        content: string | Array<{
+            type: string;
+            text?: string;
+            tool_use_id?: string;
+        }>;
+        model?: string;
+    };
+    costUSD?: number;
+    durationMs?: number;
+    inputTokens?: number;
+    outputTokens?: number;
+    cacheCreationInputTokens?: number;
+    cacheReadInputTokens?: number;
+}
+/** Leaderboard entry from API */
+export interface LeaderboardEntry {
+    skillId: string;
+    skillName: string;
+    source: string;
+    bestAccuracy: number;
+    bestModel: string;
+    avgTokens: number;
+    avgCost: number;
+    lastTested: string;
+    totalRuns: number;
+}
+//# sourceMappingURL=benchmark-types.d.ts.map

package/dist/types/benchmark-types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"benchmark-types.d.ts","sourceRoot":"","sources":["../../src/types/benchmark-types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,uDAAuD;AACvD,MAAM,WAAW,cAAc;IAC7B,6BAA6B;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,wDAAwD;IACxD,IAAI,EAAE,WAAW,GAAG,MAAM,CAAC;IAC3B,oCAAoC;IACpC,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,yBAAyB;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,mCAAmC;IACnC,MAAM,EAAE,MAAM,CAAC;IACf,0CAA0C;IAC1C,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,uBAAuB;IACvB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,oDAAoD;AACpD,MAAM,WAAW,gBAAgB;IAC/B,qDAAqD;IACrD,QAAQ,EAAE,MAAM,CAAC;IACjB,4BAA4B;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,mBAAmB;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,oBAAoB;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,sCAAsC;IACtC,UAAU,EAAE,MAAM,CAAC;IACnB,gCAAgC;IAChC,SAAS,EAAE,MAAM,CAAC;IAClB,gCAAgC;IAChC,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,wCAAwC;AACxC,MAAM,WAAW,UAAU;IACzB,sBAAsB;IACtB,IAAI,EAAE,cAAc,CAAC;IACrB,wBAAwB;IACxB,OAAO,EAAE,gBAAgB,CAAC;IAC1B,iCAAiC;IACjC,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,gCAAgC;IAChC,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,wBAAwB;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,oBAAoB;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,iDAAiD;IACjD,MAAM,EAAE,OAAO,CAAC;CACjB;AAED,8CAA8C;AAC9C,MAAM,WAAW,eAAe;IAC9B,uBAAuB;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,iBAAiB;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,2DAA2D;IAC3D,WAAW,EAAE,MAAM,CAAC;IACpB,+BAA+B;IAC/B,KAAK,EAAE,OAAO,GAAG,QAAQ,GAAG,MAAM,CAAC;IACnC,+BAA+B;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,8BAA8B;IAC9B,WAAW,EAAE,UAAU,EAAE,CAAC;IAC1B,gDAAgD;IAChD,iBAAiB,EAAE,gBAAgB,CAAC;IACpC,0BAA0B;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,4BAA4B;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,mCAAmC;IACnC,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,yBAAyB;AACzB,MAAM,MAAM,eAAe,GAAG,OAAO,GAAG,KAAK,GAAG,SAAS,CAAC;AAE1D,sCAAsC;AACtC,MAAM,WAAW,WAAW;IAC1B,kBAAkB;IAClB,IAAI,EAAE,eAAe,CAAC;IACtB,yDAAyD;IACzD,QAAQ,EAAE,MAAM,CAAC;IACjB,kDAAkD;IAClD,SAAS,EAAE,MAAM,CAAC;IAClB,uCAAuC;IACvC,IAAI,EAAE,MAAM,CAAC;CACd;AAED,8BAA8B;AAC9B,MAAM,WAAW,UAAU;IACzB,yBAAyB;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,mBAAmB;IACnB,KAAK,EAAE,OAAO,GAAG,QAAQ,GAAG,MAAM,CAAC;IACnC,2BAA2B;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,uBAAuB;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,mCAAmC;IACnC,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,kCAAkC;AAClC,MAAM,WAAW,cAAc;IAC7B,iCAAiC;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,uBAAuB;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,6CAA6C;AAC7C,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,QAAQ,GAAG,OAAO,GAAG,WAAW,GAAG,QAAQ,CAAC;IAClD,OAAO,CAAC,EAAE;QACR,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,MAAM,GAAG,KAAK,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,IAAI,CAAC,EAAE,MAAM,CAAC;YAAC,WAAW,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;QAC/E,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,CAAC;IACF,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,oBAAoB,CAAC,EAAE,MAAM,CAAC;CAC/B;AAED,iCAAiC;AACjC,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;CACnB"}

package/dist/types/benchmark-types.js

@@ -0,0 +1,5 @@
+/**
+ * Core types for Skillmark benchmark system
+ */
+export {};
+//# sourceMappingURL=benchmark-types.js.map

package/dist/types/benchmark-types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"benchmark-types.js","sourceRoot":"","sources":["../../src/types/benchmark-types.ts"],"names":[],"mappings":"AAAA;;GAEG"}

package/dist/types/index.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Re-export all types for convenient imports
+ */
+export * from './benchmark-types.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/types/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,cAAc,sBAAsB,CAAC"}

package/dist/types/index.js

@@ -0,0 +1,5 @@
+/**
+ * Re-export all types for convenient imports
+ */
+export * from './benchmark-types.js';
+//# sourceMappingURL=index.js.map

package/dist/types/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,cAAc,sBAAsB,CAAC"}

package/package.json

@@ -0,0 +1,38 @@
+{
+  "name": "@skillmark/cli",
+  "version": "0.1.0",
+  "description": "CLI tool for benchmarking Claude agent skills",
+  "type": "module",
+  "bin": {
+    "skillmark": "./dist/cli-entry-point.js"
+  },
+  "main": "./dist/cli-entry-point.js",
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "build": "tsc",
+    "dev": "tsc --watch",
+    "lint": "tsc --noEmit",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "test:coverage": "vitest run --coverage",
+    "clean": "rm -rf dist"
+  },
+  "dependencies": {
+    "chalk": "^5.3.0",
+    "commander": "^12.0.0",
+    "gray-matter": "^4.0.3",
+    "ora": "^8.0.1",
+    "simple-git": "^3.22.0"
+  },
+  "devDependencies": {
+    "@types/node": "^20.11.0",
+    "@vitest/coverage-v8": "^4.0.18",
+    "typescript": "^5.3.3",
+    "vitest": "^4.0.18"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  }
+}