@cristobalme/skill-test 0.1.0

package/dist/index.d.ts

@@ -0,0 +1,461 @@
+/** Severity of a lint finding. Exit code is non-zero iff any `error` exists. */
+type Severity = "error" | "warn" | "info";
+/** A single result produced by a lint rule. */
+interface Finding {
+    ruleId: string;
+    severity: Severity;
+    message: string;
+    /** Absolute or repo-relative path to the file the finding concerns. */
+    file: string;
+    /** 1-based line number within `file`, when known. */
+    line?: number;
+}
+/** Parsed representation of a SKILL.md file. */
+interface ParsedSkill {
+    /** Absolute path to the SKILL.md file. */
+    path: string;
+    /** Absolute path to the skill's root directory (parent of SKILL.md). */
+    dir: string;
+    /** Raw frontmatter object (may be empty if frontmatter was missing/invalid). */
+    frontmatter: Record<string, unknown>;
+    /** Markdown body after the frontmatter. */
+    body: string;
+    /** 1-based line where the body starts in the original file. */
+    bodyStartLine: number;
+    /** Estimated token count of the body. */
+    bodyTokens: number;
+    /** Number of lines in the body. */
+    bodyLines: number;
+    /** True when valid YAML frontmatter delimited by `---` was found. */
+    hasFrontmatter: boolean;
+    /** Parse-level error message, if the file could not be parsed at all. */
+    parseError?: string;
+}
+/** Result of the static lint layer for one skill. */
+interface LintResult {
+    layer: "lint";
+    skillPath: string;
+    findings: Finding[];
+    /** True iff no `error`-severity findings. */
+    ok: boolean;
+}
+/** Outcome of classifying a single prompt against a skill's metadata. */
+interface ClassificationResult {
+    prompt: string;
+    /** Whether the model decided the skill would activate. */
+    activated: boolean;
+    /** Short reason from the model. */
+    reason: string;
+    /** True when this result came from the on-disk cache. */
+    cached: boolean;
+}
+/** Precision/recall/F1 over a labeled corpus of prompts. */
+interface TriggerScore {
+    truePositives: number;
+    falsePositives: number;
+    trueNegatives: number;
+    falseNegatives: number;
+    precision: number;
+    recall: number;
+    f1: number;
+}
+/** Result of the triggering layer for one skill. */
+interface TriggerResult {
+    layer: "trigger";
+    skillPath: string;
+    score: TriggerScore;
+    classifications: ClassificationResult[];
+    /** Prompts that should have activated but didn't. */
+    falseNegatives: string[];
+    /** Prompts that should not have activated but did. */
+    falsePositives: string[];
+    ok: boolean;
+}
+/** A layer that was skipped, with a human-readable reason. */
+interface SkippedLayer {
+    layer: "trigger" | "run";
+    skipped: true;
+    reason: string;
+}
+/** Aggregate report across all layers run for a set of skills. */
+interface Report {
+    results: SkillReport[];
+    /** Overall: true iff every skill passed every layer that ran. */
+    ok: boolean;
+}
+/** Per-skill aggregate across layers. */
+interface SkillReport {
+    skillPath: string;
+    lint?: LintResult;
+    trigger?: TriggerResult | SkippedLayer;
+    ok: boolean;
+}
+/** Process exit codes (documented in the README). */
+declare const EXIT: {
+    readonly OK: 0;
+    readonly FAILURES: 1;
+    readonly USAGE: 2;
+};
+
+/**
+ * Agent Skills specification constants.
+ *
+ * Source: https://agentskills.io/specification (fetched 2026-06-04).
+ * The Agent Skills format was originally developed by Anthropic and released as
+ * an open standard. These constants mirror the live spec verbatim; update them
+ * (and the source date) whenever the spec changes.
+ *
+ * Frontmatter fields:
+ *   - name           (required) Max 64 chars. Lowercase a-z, 0-9 and hyphens.
+ *                               Must not start/end with a hyphen or contain "--".
+ *                               Must match the parent directory name.
+ *   - description    (required) Max 1024 chars. Non-empty. What it does + when to use.
+ *   - license        (optional) License name or reference to a bundled file.
+ *   - compatibility  (optional) Max 500 chars. Environment requirements.
+ *   - metadata       (optional) Map of string keys to string values.
+ *   - allowed-tools  (optional) Space-separated string of pre-approved tools. (Experimental)
+ *
+ * Body: no format restrictions. Recommended < 5000 tokens once activated, and the
+ * main SKILL.md kept under 500 lines; move detail into referenced files.
+ * File references should be relative paths from the skill root, one level deep.
+ */
+/** Allowed characters and shape for the `name` field. */
+declare const NAME_PATTERN: RegExp;
+declare const NAME_MAX_LENGTH = 64;
+declare const DESCRIPTION_MAX_LENGTH = 1024;
+declare const COMPATIBILITY_MAX_LENGTH = 500;
+/** Recommended ceiling for the activated SKILL.md body. */
+declare const BODY_RECOMMENDED_MAX_TOKENS = 5000;
+/** We warn as the body approaches the ceiling (90%). */
+declare const BODY_WARN_TOKENS: number;
+/** Recommended maximum number of lines in the main SKILL.md. */
+declare const BODY_RECOMMENDED_MAX_LINES = 500;
+/** Known frontmatter fields per the spec, used to flag unknown keys (info only). */
+declare const KNOWN_FRONTMATTER_FIELDS: readonly ["name", "description", "license", "compatibility", "metadata", "allowed-tools"];
+declare const SPEC_URL = "https://agentskills.io/specification";
+
+declare const spec_BODY_RECOMMENDED_MAX_LINES: typeof BODY_RECOMMENDED_MAX_LINES;
+declare const spec_BODY_RECOMMENDED_MAX_TOKENS: typeof BODY_RECOMMENDED_MAX_TOKENS;
+declare const spec_BODY_WARN_TOKENS: typeof BODY_WARN_TOKENS;
+declare const spec_COMPATIBILITY_MAX_LENGTH: typeof COMPATIBILITY_MAX_LENGTH;
+declare const spec_DESCRIPTION_MAX_LENGTH: typeof DESCRIPTION_MAX_LENGTH;
+declare const spec_KNOWN_FRONTMATTER_FIELDS: typeof KNOWN_FRONTMATTER_FIELDS;
+declare const spec_NAME_MAX_LENGTH: typeof NAME_MAX_LENGTH;
+declare const spec_NAME_PATTERN: typeof NAME_PATTERN;
+declare const spec_SPEC_URL: typeof SPEC_URL;
+declare namespace spec {
+  export { spec_BODY_RECOMMENDED_MAX_LINES as BODY_RECOMMENDED_MAX_LINES, spec_BODY_RECOMMENDED_MAX_TOKENS as BODY_RECOMMENDED_MAX_TOKENS, spec_BODY_WARN_TOKENS as BODY_WARN_TOKENS, spec_COMPATIBILITY_MAX_LENGTH as COMPATIBILITY_MAX_LENGTH, spec_DESCRIPTION_MAX_LENGTH as DESCRIPTION_MAX_LENGTH, spec_KNOWN_FRONTMATTER_FIELDS as KNOWN_FRONTMATTER_FIELDS, spec_NAME_MAX_LENGTH as NAME_MAX_LENGTH, spec_NAME_PATTERN as NAME_PATTERN, spec_SPEC_URL as SPEC_URL };
+}
+
+/**
+ * Parse a SKILL.md file into frontmatter + body. Never throws on malformed
+ * content: parse-level problems are reported via `parseError` / `hasFrontmatter`
+ * so lint rules can turn them into findings.
+ */
+declare function parseSkillFile(path: string): ParsedSkill;
+/** Parse already-loaded SKILL.md content. Exposed for tests. */
+declare function parseSkillContent(raw: string, path: string): ParsedSkill;
+
+/**
+ * Offline token estimator.
+ *
+ * The static layer must never require a network call or an API key, so we cannot
+ * use a real tokenizer service. This heuristic blends a character-based and a
+ * word-based estimate, which tracks BPE tokenizers (e.g. cl100k) closely enough
+ * for a "is this body near the recommended ceiling?" check. It intentionally
+ * errs slightly high so we warn before a body actually exceeds the limit.
+ */
+declare function estimateTokens(text: string): number;
+
+/** Run every lint rule against a single SKILL.md file. */
+declare function lintSkill(skillPath: string): LintResult;
+
+type Rule = (skill: ParsedSkill) => Finding[];
+/** The ordered set of rules run by the lint orchestrator. */
+declare const rules: Rule[];
+
+/**
+ * Resolve one or more path arguments into a deduplicated list of SKILL.md files.
+ *
+ * Each argument may be:
+ *   - a path to a SKILL.md file directly,
+ *   - a skill directory (containing SKILL.md),
+ *   - a directory of many skills (walked recursively for SKILL.md files).
+ *
+ * Glob patterns are expanded by the shell before reaching us; a directory
+ * argument is walked recursively, so pointing at a repo of skills works.
+ */
+declare function discoverSkills(paths: string[]): string[];
+
+/** Serialize lint results to a stable JSON-friendly object. */
+declare function lintResultsToJson(results: LintResult[]): {
+    tool: string;
+    layer: string;
+    ok: boolean;
+    skills: {
+        skill: string;
+        ok: boolean;
+        findings: Finding[];
+    }[];
+};
+/** Serialize a full multi-layer report. */
+declare function reportToJson(report: Report): {
+    tool: string;
+    ok: boolean;
+    skills: {
+        skill: string;
+        ok: boolean;
+        lint: {
+            ok: boolean;
+            findings: Finding[];
+        } | undefined;
+        trigger: {
+            skipped: boolean;
+            reason: string;
+            ok?: undefined;
+            score?: undefined;
+            falseNegatives?: undefined;
+            falsePositives?: undefined;
+        } | {
+            ok: boolean;
+            score: TriggerScore;
+            falseNegatives: string[];
+            falsePositives: string[];
+            skipped?: undefined;
+            reason?: undefined;
+        } | undefined;
+    }[];
+};
+
+/** Render results of the lint layer for a set of skills. */
+declare function renderLintResults(results: LintResult[], opts?: {
+    quiet?: boolean;
+}): string;
+/** Render an aggregate Report across all layers (used by `lint` and `check`). */
+declare function renderReport(report: Report, opts?: {
+    quiet?: boolean;
+}): string;
+
+/** Render a Report as JUnit XML for CI systems. */
+declare function reportToJUnit(report: Report): string;
+
+declare function setColorEnabled(on: boolean): void;
+
+interface CachedClassification {
+    activated: boolean;
+    reason: string;
+}
+/**
+ * Disk cache for classifications, keyed by a hash of (model, name, description,
+ * prompt). Identical inputs are free to re-run, which keeps CI cheap and
+ * deterministic. The cache file is a single JSON object.
+ */
+declare class TriggerCache {
+    private readonly file;
+    private readonly enabled;
+    private store;
+    private dirty;
+    constructor(file: string, enabled?: boolean);
+    key(model: string, name: string, description: string, prompt: string): string;
+    get(key: string): CachedClassification | undefined;
+    set(key: string, value: CachedClassification): void;
+    /** Persist to disk if anything changed. */
+    flush(): void;
+}
+
+/** Default classifier model. Pinned for reproducibility; override with --model. */
+declare const DEFAULT_MODEL = "claude-haiku-4-5-20251001";
+/** One classification request: skill metadata only — never the skill body. */
+interface ClassifyRequest {
+    name: string;
+    description: string;
+    prompt: string;
+    model: string;
+}
+interface ClassifyResponse {
+    activated: boolean;
+    reason: string;
+}
+/**
+ * Pluggable classification backend. The Anthropic implementation lives in
+ * backend/anthropic.ts; tests provide a mock. Keeping this an interface is what
+ * lets the trigger layer be backend-agnostic later.
+ */
+interface ClassifierBackend {
+    readonly model: string;
+    classify(req: ClassifyRequest): Promise<ClassifyResponse>;
+}
+interface SkillMeta {
+    name: string;
+    description: string;
+}
+/**
+ * Classify every prompt against the skill's metadata, using the on-disk cache
+ * when available so reruns make zero API calls. Returns one result per prompt in
+ * input order.
+ */
+declare function classifyPrompts(meta: SkillMeta, prompts: string[], backend: ClassifierBackend, cache?: TriggerCache): Promise<ClassificationResult[]>;
+
+/** Validated shape of a SKILL.test.yaml file. */
+interface TriggerSpec {
+    specPath: string;
+    /** Optional pointer to the SKILL.md this spec targets (relative to the spec). */
+    skillRef?: string;
+    shouldActivate: string[];
+    shouldNotActivate: string[];
+    /** Behavioral tasks — validated structurally but exercised in Phase 5. */
+    tasks: unknown[];
+}
+declare class SpecValidationError extends Error {
+}
+/** Locate the SKILL.test.yaml that sits beside a SKILL.md, if any. */
+declare function findSpec(skillPath: string): string | undefined;
+/** Load and validate a SKILL.test.yaml file. Throws SpecValidationError. */
+declare function loadSpec(specPath: string): TriggerSpec;
+
+/**
+ * Compute precision/recall/F1 over the labeled corpus.
+ *   - should_activate that activated   ⇒ true positive
+ *   - should_activate that didn't      ⇒ false negative
+ *   - should_not_activate that activated ⇒ false positive
+ *   - should_not_activate that didn't  ⇒ true negative
+ */
+declare function computeTriggerResult(skillPath: string, spec: TriggerSpec, classifications: ClassificationResult[]): TriggerResult;
+
+interface TriggerOptions {
+    /** Backend to classify with. When absent, the layer is skipped (no API key). */
+    backend?: ClassifierBackend;
+    /** Cache instance. When absent, a default disk cache is created. */
+    cache?: TriggerCache;
+    /** Directory for the default disk cache. */
+    cacheDir?: string;
+    /** Disable caching entirely. */
+    noCache?: boolean;
+}
+/**
+ * Run the triggering layer for one skill. Returns a TriggerResult, or a
+ * SkippedLayer with a clear reason when there's no spec, no metadata, or no
+ * backend. Throws SpecValidationError when a spec exists but is invalid.
+ */
+declare function triggerSkill(skillPath: string, opts?: TriggerOptions): Promise<TriggerResult | SkippedLayer>;
+
+/** Extract the first JSON object from a model response. */
+declare function parseDecision(text: string): ClassifyResponse;
+interface AnthropicBackendOptions {
+    apiKey?: string;
+    model?: string;
+}
+/** Create a classifier backed by the Anthropic Messages API. */
+declare function createAnthropicBackend(opts?: AnthropicBackendOptions): ClassifierBackend;
+/** True when an API key is available for the trigger/behavioral layers. */
+declare function hasApiKey(): boolean;
+
+/**
+ * Behavioral graders (Phase 5 — interfaces only).
+ *
+ * Deterministic graders come first and are the default; `llm_judge` is opt-in
+ * and must be validated against hand-labeled cases before it is relied on.
+ */
+/** Declarative grader specs as they appear in SKILL.test.yaml `tasks[].grade`. */
+type GraderSpec = {
+    type: "file_exists";
+    path: string;
+} | {
+    type: "regex_match";
+    file?: string;
+    pattern: string;
+} | {
+    type: "contains";
+    file?: string;
+    text: string;
+} | {
+    type: "exit_code";
+    equals: number;
+} | {
+    type: "llm_judge";
+    rubric: string;
+    threshold: number;
+};
+/** Context handed to a grader after a task run completes. */
+interface GradeContext {
+    /** Working directory the agent ran in (inside the sandbox mount). */
+    workdir: string;
+    /** Captured stdout of the run. */
+    stdout: string;
+    /** Process exit code of the run. */
+    exitCode: number;
+}
+interface GraderResult {
+    type: GraderSpec["type"];
+    passed: boolean;
+    detail?: string;
+}
+interface Grader {
+    grade(ctx: GradeContext): Promise<GraderResult> | GraderResult;
+}
+/** Build a grader from its spec. Implemented in Phase 5. */
+declare function createGrader(_spec: GraderSpec): Grader;
+
+/**
+ * Execution sandbox (Phase 5 — interfaces only).
+ *
+ * Skill-provided code is NEVER executed on the host. Tasks run inside an
+ * isolated container with only the task fixtures mounted.
+ */
+interface SandboxOptions {
+    /** Container image to run in. */
+    image: string;
+    /** Host paths mounted read-only into the sandbox. */
+    mounts: string[];
+    /** Network policy; default is no network. */
+    network?: "none" | "host";
+}
+interface SandboxRun {
+    stdout: string;
+    stderr: string;
+    exitCode: number;
+}
+interface Sandbox {
+    run(command: string): Promise<SandboxRun>;
+    dispose(): Promise<void>;
+}
+/** Create a Docker-backed sandbox. Implemented in Phase 5. */
+declare function createDockerSandbox(_opts: SandboxOptions): Sandbox;
+
+/**
+ * Behavioral task runner (Phase 5 — interfaces only).
+ *
+ * Each task runs N times (default 5) to surface nondeterminism; the result is a
+ * pass *rate*, not a single boolean, and passes on a threshold. Model and
+ * temperature are pinned for reproducibility.
+ */
+
+/** A behavioral task as it appears under SKILL.test.yaml `tasks`. */
+interface TaskSpec {
+    /** Optional setup commands run before the prompt. */
+    setup?: string;
+    /** The user prompt given to the agent. */
+    prompt: string;
+    /** One or more graders; deterministic graders are preferred. */
+    grade: GraderSpec[];
+}
+interface TaskRunResult {
+    prompt: string;
+    /** Fraction of runs that passed all graders (0..1). */
+    passRate: number;
+    runs: number;
+    passed: boolean;
+}
+interface RunOptions {
+    /** How many times to run each task (default 5). */
+    repeats?: number;
+    /** Pass threshold on the pass rate (default 1.0). */
+    threshold?: number;
+    /** Pinned model + temperature for reproducibility. */
+    model?: string;
+    temperature?: number;
+}
+/** Run a behavioral task N times and report a pass rate. Implemented in Phase 5. */
+declare function runTask(_task: TaskSpec, _opts?: RunOptions): Promise<TaskRunResult>;
+
+export { type ClassificationResult, type ClassifierBackend, type ClassifyRequest, type ClassifyResponse, DEFAULT_MODEL, EXIT, type Finding, type GradeContext, type Grader, type GraderResult, type GraderSpec, type LintResult, type ParsedSkill, type Report, type RunOptions, type Sandbox, type SandboxOptions, type Severity, type SkillMeta, type SkillReport, type SkippedLayer, SpecValidationError, type TaskRunResult, type TaskSpec, TriggerCache, type TriggerOptions, type TriggerResult, type TriggerScore, type TriggerSpec, classifyPrompts, computeTriggerResult, createAnthropicBackend, createDockerSandbox, createGrader, discoverSkills, estimateTokens, findSpec, hasApiKey, lintResultsToJson, lintSkill, loadSpec, parseDecision, parseSkillContent, parseSkillFile, renderLintResults, renderReport, reportToJUnit, reportToJson, rules, runTask, setColorEnabled, spec, triggerSkill };