modelmix 5.1.20 → 5.2.1

package/plugins/benchmark/test/benchmark.test.js

@@ -0,0 +1,518 @@
+const { expect } = require('chai');
+const sinon = require('sinon');
+
+const { MixOpenAI, ModelMix } = require('../../..');
+const { benchmark } = require('..');
+
+const criteria = {
+    criteria: [
+        { id: 'correctness', description: 'How completely the response fulfills the task.' },
+        { id: 'clarity', description: 'How clear and coherent the response is.' }
+    ]
+};
+
+function request(overrides = {}) {
+    return {
+        system: 'Original system instructions',
+        messages: [{ role: 'user', content: [{ type: 'text', text: 'Complete the task.' }] }],
+        options: { max_tokens: 123, response_format: { type: 'json_object' } },
+        config: { system: 'Original system instructions' },
+        outputMode: 'json',
+        ...overrides
+    };
+}
+
+function modelKey(input) {
+    return input.model.models[0].key;
+}
+
+function validEvaluation(correctness = 8, clarity = 6) {
+    return JSON.stringify({
+        scores: [
+            { criterionId: 'correctness', score: correctness, justification: 'It fulfills the task.' },
+            { criterionId: 'clarity', score: clarity, justification: 'It is easy to follow.' }
+        ]
+    });
+}
+
+function metricsResult(message, cost = 0.01) {
+    return {
+        message,
+        tokens: { input: 1, output: 2, total: 3, cost }
+    };
+}
+
+function directContext(invoke, overrides = {}) {
+    return {
+        request: request(overrides),
+        execution: { executionId: 'root', parentExecutionId: null, depth: 0 },
+        signal: overrides.signal,
+        invoke
+    };
+}
+
+describe('Empty benchmark responses', () => {
+    it('records a generation failure and never evaluates an empty candidate', async () => {
+        const plugin = benchmark({ criteriaModel: 'gpt5nano', models: ['gpt5', 'gpt5mini'] });
+        const result = await plugin.execute(directContext(async input => {
+            if (input.system.includes('define evaluation criteria')) return metricsResult(JSON.stringify(criteria));
+            if (input.system.includes('evaluate one candidate response')) {
+                expect(input.messages[0].content).to.include('usable response');
+                return metricsResult(validEvaluation());
+            }
+            return metricsResult(modelKey(input) === 'gpt-5' ? '  ' : 'usable response');
+        }));
+        expect(result.benchmark.results[0]).to.include({ response: null, score: null });
+        expect(result.benchmark.results[0].evaluationCount).to.deep.equal({ expected: 0, valid: 0 });
+        expect(result.benchmark.errors[0]).to.include({ stage: 'response', participant: 'gpt5' });
+        expect(result.benchmark.errors[0].error.message).to.include('no text response');
+    });
+});
+
+async function expectRejection(promise, message) {
+    let rejection;
+    try {
+        await promise;
+    } catch (error) {
+        rejection = error;
+    }
+    expect(rejection).to.be.an('error');
+    expect(rejection.message).to.include(message);
+}
+
+describe('benchmark plugin', () => {
+    afterEach(() => sinon.restore());
+
+    it('uses the native DeepSeek provider for criteria, responses and judging when selected', async () => {
+        const { MixDeepSeek, MixOpenRouter } = require('../../..');
+        const originalApiKey = process.env.DEEPSEEK_API_KEY;
+        process.env.DEEPSEEK_API_KEY = 'native-test-key';
+        const stages = [];
+        try {
+            const plugin = benchmark({
+                criteriaModel: 'deepseekV41Flash@20',
+                models: ['deepseekV41Flash@60', 'gpt5'],
+                mix: { deepseek: true, openrouter: false }
+            });
+            const raw = await plugin.execute(directContext(async input => {
+                expect(input.model.models.some(item => item.provider instanceof MixOpenRouter)).to.equal(false);
+                const stage = input.system.includes('define evaluation criteria')
+                    ? 'criteria'
+                    : input.system.includes('evaluate one candidate response') ? 'evaluation' : 'response';
+                if (modelKey(input) === 'deepseek-flash') {
+                    expect(input.model.models).to.have.length(1);
+                    const { provider } = input.model.models[0];
+                    expect(provider).to.be.instanceOf(MixDeepSeek);
+                    expect(provider.config.url).to.equal('https://api.deepseek.com/chat/completions');
+                    expect(provider.config.effort).to.equal(stage === 'criteria' ? 20 : 60);
+                    stages.push(stage);
+                }
+                if (stage === 'criteria') return metricsResult(JSON.stringify(criteria));
+                if (stage === 'evaluation') return metricsResult(validEvaluation());
+                return metricsResult('Candidate answer');
+            }));
+            expect(stages).to.deep.equal(['criteria', 'response', 'evaluation']);
+            expect(raw.benchmark.errors).to.deep.equal([]);
+            expect(raw.benchmark.results[0].canonicalModel).to.equal('deepseek-flash');
+        } finally {
+            if (originalApiKey === undefined) delete process.env.DEEPSEEK_API_KEY;
+            else process.env.DEEPSEEK_API_KEY = originalApiKey;
+        }
+    });
+
+    it('accepts a closing Markdown delimiter on criteria and evaluations without changing the response', async () => {
+        const response = 'Candidate with ```text\ncontent\n```';
+        const plugin = benchmark({ criteriaModel: 'gpt5nano', models: ['gpt5', 'sonnet5'] });
+        const raw = await plugin.execute(directContext(async input => {
+            if (input.system.includes('define evaluation criteria')) {
+                return metricsResult(JSON.stringify(criteria) + '```');
+            }
+            if (input.system.includes('evaluate one candidate response')) {
+                return metricsResult(validEvaluation() + '\n```\n');
+            }
+            return metricsResult(response);
+        }));
+        expect(raw.benchmark.errors).to.deep.equal([]);
+        expect(raw.benchmark.results[0].response).to.equal(response);
+        expect(raw.benchmark.results.map(result => result.score)).to.deep.equal([7, 7]);
+    });
+
+    it('still rejects invalid or incomplete evaluations with closing Markdown delimiters', async () => {
+        const plugin = benchmark({ criteriaModel: 'gpt5nano', models: ['gpt5', 'sonnet5'] });
+        let evaluation = 0;
+        const invalid = [validEvaluation() + ' explanation```', '{"scores":[]}```'];
+        const raw = await plugin.execute(directContext(async input => {
+            if (input.system.includes('define evaluation criteria')) {
+                return metricsResult(JSON.stringify(criteria));
+            }
+            if (input.system.includes('evaluate one candidate response')) {
+                return metricsResult(invalid[evaluation++]);
+            }
+            return metricsResult('Candidate');
+        }));
+        expect(raw.benchmark.results.map(result => result.score)).to.deep.equal([null, null]);
+        expect(raw.benchmark.errors.map(item => item.error.details.outputText)).to.deep.equal(invalid);
+        expect(raw.benchmark.errors[0].error.message).to.include('invalid JSON');
+        expect(raw.benchmark.errors[1].error.message).to.include('every criterion exactly once');
+    });
+
+    for (const failJudge of [false, true]) {
+        it(`runs Gemini through its adapter and ${failJudge ? 'preserves provider failures in logs and the report' : 'keeps JSON mode limited to criteria and judging'}`, async () => {
+            const log = sinon.stub(console, 'log');
+            const bodies = [];
+            const providerError = 'Invalid JSON payload received. Unknown name "content".';
+            sinon.stub(global, 'fetch').callsFake(async (_url, input) => {
+                const body = JSON.parse(input.body);
+                bodies.push(body);
+                const system = body.systemInstruction.parts.map(part => part.text).join('');
+                const judging = system.includes('evaluate one candidate response');
+                if (judging && failJudge) {
+                    return new Response(JSON.stringify({ error: { message: providerError } }), { status: 400 });
+                }
+                const text = system.includes('define evaluation criteria')
+                    ? JSON.stringify(criteria)
+                    : judging ? validEvaluation() : 'Gemini candidate';
+                return new Response(JSON.stringify({
+                    candidates: [{ content: { parts: [{ text }] }, finishReason: 'STOP' }]
+                }), { status: 200 });
+            });
+            sinon.stub(MixOpenAI.prototype, 'create').callsFake(async ({ config }) => (
+                metricsResult(config.system.includes('evaluate one candidate response')
+                    ? validEvaluation() : 'Other candidate')
+            ));
+            const report = await ModelMix.new({ config: { apiKey: 'test-key', debug: 1 } })
+                .use(benchmark({ criteriaModel: 'gemini38flash@20', models: ['gemini38flash@20', 'gpt5mini@20'] }))
+                .addText('Complete the task.')
+                .json();
+
+            expect(bodies).to.have.length(3);
+            expect(bodies.map(body => body.generationConfig.responseMimeType)).to.deep.equal([
+                'application/json', 'text/plain', 'application/json'
+            ]);
+            for (const body of bodies) {
+                expect(body.contents).to.have.length(1);
+                expect(body.contents[0]).to.have.keys('role', 'parts');
+                expect(body.contents[0].parts[0].text).to.be.a('string');
+            }
+            expect(report.results[0].response).to.equal('Gemini candidate');
+            expect(report.results[0].score).to.equal(7);
+            expect(report.metrics.calls.attempted).to.equal(5);
+            expect(report.errors).to.have.length(failJudge ? 1 : 0);
+            expect(report.results[1].score).to.equal(failJudge ? null : 7);
+            if (failJudge) {
+                expect(report.errors[0]).to.include({ stage: 'evaluation', judge: 'gemini38flash@20' });
+                expect(report.errors[0].error.statusCode).to.equal(400);
+                expect(report.errors[0].error.details.error.message).to.equal(providerError);
+                expect(log.args.flat().join('\n')).to.include(providerError).and.include('HTTP 400');
+            }
+        });
+    }
+
+    it('preserves Markdown fences inside valid JSON evaluations and the final JSON report', async () => {
+        sinon.stub(MixOpenAI.prototype, 'create').callsFake(async ({ config }) => {
+            if (config.system.includes('define evaluation criteria')) {
+                return metricsResult(JSON.stringify(criteria));
+            }
+            if (config.system.includes('evaluate one candidate response')) {
+                const value = JSON.parse(validEvaluation());
+                value.scores[0].justification = 'The response contains ```text and ``` markers.';
+                return metricsResult(JSON.stringify(value));
+            }
+            return metricsResult('```text\nA candidate answer\n```');
+        });
+        const report = await ModelMix.new().use(benchmark({
+            criteriaModel: 'gpt5nano', models: ['gpt5', 'gpt5mini']
+        })).addText('Evaluate the answer.').json();
+        expect(report.errors).to.deep.equal([]);
+        expect(report.results[0].response).to.equal('```text\nA candidate answer\n```');
+        expect(report.results[0].evaluationCount.valid).to.equal(1);
+    });
+
+    it('rejects truncated output with its finish reason and preserves the text and metrics', async () => {
+        const plugin = benchmark({ criteriaModel: 'gpt5nano', models: ['gpt5', 'sonnet5'] });
+        const log = sinon.stub(console, 'log');
+        const raw = await plugin.execute(directContext(async input => {
+            if (input.system.includes('define evaluation criteria')) {
+                return metricsResult(JSON.stringify(criteria));
+            }
+            if (input.system.includes('evaluate one candidate response')) {
+                return {
+                    ...metricsResult('{"scores":['),
+                    response: { choices: [{ finish_reason: 'length' }] }
+                };
+            }
+            return metricsResult('Candidate answer');
+        }, { config: { debug: 1 } }));
+        expect(raw.benchmark.errors).to.have.length(2);
+        expect(raw.benchmark.errors[0].error.message).to.include('token limit');
+        expect(raw.benchmark.errors[0].error.details).to.include({
+            finishReason: 'length', outputText: '{"scores":['
+        });
+        expect(raw.benchmark.metrics.calls.attempted).to.equal(5);
+        expect(raw.benchmark.metrics.cost).to.be.closeTo(0.05, 1e-12);
+        expect(raw.benchmark.results[0].score).to.equal(null);
+        expect(log.args.flat().join('\n')).to.include('token limit');
+    });
+
+    it('returns a JSON report with isolated responses, cross-evaluations, averages, and metrics', async () => {
+        const calls = [];
+        const plugin = benchmark({
+            criteriaModel: 'gpt5nano@10',
+            models: ['gpt5@0', 'gpt5mini@25', 'sonnet5@50']
+        });
+        const context = directContext(async input => {
+            calls.push(input);
+            if (input.system.includes('define evaluation criteria')) {
+                return metricsResult(JSON.stringify(criteria));
+            }
+            if (input.system.includes('evaluate one candidate response')) {
+                return metricsResult(validEvaluation());
+            }
+            return metricsResult(`response:${modelKey(input)}`);
+        });
+
+        const raw = await plugin.execute(context);
+        const report = JSON.parse(raw.message);
+
+        expect(raw.benchmark).to.deep.equal(report);
+        expect(report.criteria.items).to.deep.equal(criteria.criteria);
+        expect(report.criteria.model).to.include({ id: 'gpt5nano@10', effort: 10 });
+        expect(report.results).to.have.length(3);
+        expect(report.results.map(result => result.response)).to.deep.equal([
+            'response:gpt-5',
+            'response:gpt-5-mini',
+            'response:claude-sonnet-5'
+        ]);
+        for (const result of report.results) {
+            expect(result.evaluationCount).to.deep.equal({ expected: 2, valid: 2 });
+            expect(result.averages).to.deep.equal([
+                { criterionId: 'correctness', score: 8 },
+                { criterionId: 'clarity', score: 6 }
+            ]);
+            expect(result.score).to.equal(7);
+        }
+        expect(report.errors).to.deep.equal([]);
+        expect(report.metrics.tokens).to.include({ input: 10, output: 20, total: 30 });
+        expect(report.metrics.cost).to.be.closeTo(0.1, 1e-12);
+        expect(report.metrics.calls).to.deep.equal({ attempted: 10, withTokens: 10, withCost: 10 });
+
+        const participantCalls = calls.filter(call => call.system === request().system);
+        expect(participantCalls).to.have.length(3);
+        for (const call of participantCalls) {
+            expect(call.messages).to.deep.equal(request().messages);
+            expect(call.options).to.not.have.property('response_format');
+            expect(call.plugins).to.equal('none');
+            expect(call.history).to.equal(false);
+        }
+        for (const call of calls.filter(call => call.system !== request().system)) {
+            expect(call.options.response_format).to.deep.equal({ type: 'json_object' });
+        }
+    });
+
+    it('treats aliases as one model for duplicates and self-evaluation exclusion', async () => {
+        const plugin = benchmark({
+            criteriaModel: 'gpt5nano',
+            models: ['sonnet5@10', 'sonnet50@20', 'gpt5@30']
+        });
+        const raw = await plugin.execute(directContext(async input => {
+            if (input.system.includes('define evaluation criteria')) {
+                return metricsResult(JSON.stringify(criteria));
+            }
+            if (input.system.includes('evaluate one candidate response')) {
+                return metricsResult(validEvaluation());
+            }
+            return metricsResult(`response:${modelKey(input)}`);
+        }));
+
+        expect(raw.benchmark.results.map(result => result.evaluationCount)).to.deep.equal([
+            { expected: 1, valid: 1 },
+            { expected: 1, valid: 1 },
+            { expected: 2, valid: 2 }
+        ]);
+        expect(raw.benchmark.results[0].evaluations[0].judge.canonicalModel).to.equal('gpt-5');
+        expect(raw.benchmark.results[1].evaluations[0].judge.canonicalModel).to.equal('gpt-5');
+        expect(raw.benchmark.results[2].evaluations.map(item => item.judge.id)).to.deep.equal([
+            'sonnet5@10',
+            'sonnet50@20'
+        ]);
+
+        const duplicate = benchmark({
+            criteriaModel: 'gpt5nano',
+            models: ['sonnet5@100', 'sonnet50@100', 'gpt5']
+        });
+        await expectRejection(duplicate.execute(directContext(async () => {
+            throw new Error('should not be called');
+        })), 'duplicates the same model and effective effort');
+
+        const inheritedDuplicate = benchmark({
+            criteriaModel: 'gpt5nano',
+            models: ['sonnet5', 'sonnet50', 'gpt5']
+        });
+        await expectRejection(inheritedDuplicate.execute(directContext(async () => {
+            throw new Error('should not be called');
+        }, {
+            config: { system: 'Original system instructions', effort: 40 }
+        })), 'duplicates the same model and effective effort');
+
+        const oneDistinctModel = benchmark({
+            criteriaModel: 'gpt5nano',
+            models: ['sonnet5@10', 'sonnet50@20']
+        });
+        await expectRejection(oneDistinctModel.execute(directContext(async () => {
+            throw new Error('should not be called');
+        })), 'at least two distinct models');
+    });
+
+    it('keeps partial results and lets a failed participant continue as a judge', async () => {
+        const evaluationCalls = [];
+        const plugin = benchmark({
+            criteriaModel: 'gpt5nano',
+            models: ['gpt5', 'gpt5mini', 'sonnet5']
+        });
+        const raw = await plugin.execute(directContext(async input => {
+            if (input.system.includes('define evaluation criteria')) {
+                return metricsResult(JSON.stringify(criteria));
+            }
+            if (input.system.includes('evaluate one candidate response')) {
+                const payload = JSON.parse(input.messages[0].content);
+                const judge = modelKey(input);
+                evaluationCalls.push({ judge, response: payload.candidateResponse });
+                if (judge === 'claude-sonnet-5' && payload.candidateResponse === 'response:gpt-5-mini') {
+                    return metricsResult('{invalid');
+                }
+                return metricsResult(validEvaluation(9, 7));
+            }
+            if (modelKey(input) === 'gpt-5') throw new Error('participant unavailable');
+            return metricsResult(`response:${modelKey(input)}`);
+        }));
+
+        const [failed, mini, sonnet] = raw.benchmark.results;
+        expect(failed.response).to.equal(null);
+        expect(failed.responseMetrics).to.include({ tokens: null, cost: null });
+        expect(failed.score).to.equal(null);
+        expect(failed.evaluationCount).to.deep.equal({ expected: 0, valid: 0 });
+        expect(mini.evaluationCount).to.deep.equal({ expected: 2, valid: 1 });
+        expect(sonnet.evaluationCount).to.deep.equal({ expected: 2, valid: 2 });
+        expect(sonnet.score).to.equal(8);
+        expect(evaluationCalls).to.deep.include({
+            judge: 'gpt-5',
+            response: 'response:gpt-5-mini'
+        });
+        expect(evaluationCalls).to.deep.include({
+            judge: 'gpt-5',
+            response: 'response:claude-sonnet-5'
+        });
+        expect(raw.benchmark.errors.map(error => error.stage)).to.deep.equal([
+            'response',
+            'evaluation'
+        ]);
+        expect(raw.benchmark.errors[1].error.message).to.include('invalid JSON');
+        expect(raw.benchmark.metrics.calls).to.deep.equal({ attempted: 8, withTokens: 7, withCost: 7 });
+        expect(raw.benchmark.metrics.cost).to.be.closeTo(0.07, 1e-12);
+    });
+
+    it('rejects incomplete evaluations without using any of their scores', async () => {
+        const plugin = benchmark({
+            criteriaModel: 'gpt5nano',
+            models: ['gpt5', 'sonnet5']
+        });
+        const raw = await plugin.execute(directContext(async input => {
+            if (input.system.includes('define evaluation criteria')) {
+                return metricsResult(JSON.stringify(criteria));
+            }
+            if (input.system.includes('evaluate one candidate response')) {
+                return metricsResult(JSON.stringify({
+                    scores: [{
+                        criterionId: 'correctness',
+                        score: 10,
+                        justification: 'Only one criterion was scored.'
+                    }]
+                }));
+            }
+            return metricsResult(`response:${modelKey(input)}`);
+        }));
+
+        for (const result of raw.benchmark.results) {
+            expect(result.evaluations).to.deep.equal([]);
+            expect(result.averages).to.deep.equal([
+                { criterionId: 'correctness', score: null },
+                { criterionId: 'clarity', score: null }
+            ]);
+            expect(result.score).to.equal(null);
+        }
+        expect(raw.benchmark.errors).to.have.length(2);
+    });
+
+    it('aborts when criteria generation fails and propagates cancellation', async () => {
+        const invalidCriteria = benchmark({
+            criteriaModel: 'gpt5nano',
+            models: ['gpt5', 'sonnet5']
+        });
+        await expectRejection(invalidCriteria.execute(directContext(async () => (
+            metricsResult(JSON.stringify({ criteria: [] }))
+        ))), 'criteria generation failed');
+
+        const controller = new AbortController();
+        controller.abort(new Error('cancel benchmark'));
+        let calls = 0;
+        await expectRejection(invalidCriteria.execute(directContext(async () => {
+            calls += 1;
+            return metricsResult(JSON.stringify(criteria));
+        }, { signal: controller.signal })), 'cancel benchmark');
+        expect(calls).to.equal(0);
+    });
+
+    it('logs intermediate progress when ModelMix debug is enabled', async () => {
+        const plugin = benchmark({
+            criteriaModel: 'gpt5nano',
+            models: ['gpt5', 'sonnet5']
+        });
+        const log = sinon.stub(console, 'log');
+        try {
+            await plugin.execute(directContext(async input => {
+                if (input.system.includes('define evaluation criteria')) {
+                    return metricsResult(JSON.stringify(criteria));
+                }
+                if (input.system.includes('evaluate one candidate response')) {
+                    return metricsResult(validEvaluation());
+                }
+                return metricsResult(`response:${modelKey(input)}`);
+            }, {
+                config: { system: 'Original system instructions', debug: 1 }
+            }));
+        } finally {
+            log.restore();
+        }
+
+        const output = log.args.flat().join('\n');
+        expect(output).to.include('[benchmark] Generating criteria with gpt5nano.');
+        expect(output).to.include('[benchmark] Running response 1/2: gpt5.');
+        expect(output).to.include('[benchmark] Running evaluation 2/2:');
+        expect(output).to.include('[benchmark] Completed benchmark');
+    });
+
+    it('integrates with ModelMix json() and exposes the same report through lastRaw', async () => {
+        sinon.stub(MixOpenAI.prototype, 'create').callsFake(async ({ config, options }) => {
+            if (config.system.includes('define evaluation criteria')) {
+                return metricsResult(JSON.stringify(criteria));
+            }
+            if (config.system.includes('evaluate one candidate response')) {
+                return metricsResult(validEvaluation());
+            }
+            expect(options).to.not.have.property('response_format');
+            return metricsResult(`response:${options.model}`);
+        });
+        const model = ModelMix.new()
+            .use(benchmark({
+                criteriaModel: 'gpt5nano',
+                models: ['gpt5', 'gpt5mini']
+            }))
+            .addText('Complete the task.');
+
+        const report = await model.json();
+
+        expect(report.results).to.have.length(2);
+        expect(model.lastRaw.benchmark).to.deep.equal(report);
+        expect(JSON.parse(model.lastRaw.message)).to.deep.equal(report);
+    });
+});

package/plugins/skills/index.d.ts

@@ -0,0 +1,9 @@
+import type { ModelMixPlugin } from '../..';
+
+export interface SkillsOptions {
+  /** Explicit skill directories or SKILL.md files, resolved relative to process.cwd(). */
+  paths: string[];
+}
+
+/** Load local skill metadata and expose instructions and references through read_skill. */
+export declare function skills(options: SkillsOptions): Promise<ModelMixPlugin>;

package/plugins/skills/index.js

@@ -0,0 +1,107 @@
+const fs = require('node:fs/promises');
+const path = require('node:path');
+const yaml = require('js-yaml');
+
+async function resolveFile(root, relativePath, signal) {
+    signal?.throwIfAborted();
+    if (typeof relativePath !== 'string' || !relativePath || path.isAbsolute(relativePath)) {
+        throw new TypeError('Skill file path must be a non-empty relative path.');
+    }
+    const filename = await fs.realpath(path.resolve(root, relativePath));
+    const relative = path.relative(root, filename);
+    if (relative === '..' || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) {
+        throw new Error('Skill file path must stay inside the skill directory.');
+    }
+    if (!(await fs.stat(filename)).isFile()) {
+        throw new Error('Skill file path must point to a regular file.');
+    }
+    return filename;
+}
+
+async function readText(filename, signal) {
+    const buffer = await fs.readFile(filename, { signal });
+    const text = new TextDecoder('utf-8', { fatal: true }).decode(buffer);
+    if (text.includes('\0')) throw new Error('Skill files must contain UTF-8 text.');
+    return text;
+}
+
+function parseSkill(source, filename) {
+    const match = /^\uFEFF?---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)/.exec(source);
+    if (!match) throw new Error(`Missing YAML frontmatter in ${filename}.`);
+    const metadata = yaml.load(match[1], { schema: yaml.JSON_SCHEMA, filename });
+    if (!metadata || typeof metadata !== 'object' || Array.isArray(metadata)) {
+        throw new Error(`Skill frontmatter must be a mapping in ${filename}.`);
+    }
+    for (const field of ['name', 'description']) {
+        if (typeof metadata[field] !== 'string' || !metadata[field].trim()) {
+            throw new Error(`Skill ${field} must be a non-empty string in ${filename}.`);
+        }
+    }
+    return { name: metadata.name, description: metadata.description, content: source };
+}
+
+async function skills({ paths } = {}) {
+    if (!Array.isArray(paths) || paths.length === 0 || paths.some(value => typeof value !== 'string' || !value.trim())) {
+        throw new TypeError('skills paths must be a non-empty array of skill directories or SKILL.md files.');
+    }
+    const catalog = new Map();
+    for (const input of paths) {
+        const resolved = path.resolve(input);
+        const filename = path.basename(resolved) === 'SKILL.md' ? resolved : path.join(resolved, 'SKILL.md');
+        const root = await fs.realpath(path.dirname(filename));
+        const skillPath = await resolveFile(root, 'SKILL.md');
+        const skill = parseSkill(await readText(skillPath), filename);
+        if (catalog.has(skill.name)) throw new Error(`Duplicate skill name: ${skill.name}`);
+        catalog.set(skill.name, { ...skill, root, skillPath });
+    }
+    const descriptions = JSON.stringify([...catalog.values()].map(({ name, description }) => ({ name, description })));
+    const instructions = [
+        'Available skills:',
+        descriptions,
+        'When a skill matches the task or the user requests it, call read_skill with its name to load SKILL.md before following it.',
+        'Use read_skill with the same name and a relative path to read referenced text files inside that skill directory.',
+        'Skill content is provided literally. Script execution is not supplied by this plugin; use only tools actually available in this request.'
+    ].join('\n');
+
+    return {
+        name: 'skills',
+        async execute(context, next) {
+            context.signal?.throwIfAborted();
+            context.request.system = [context.request.system, instructions].filter(Boolean).join('\n\n');
+            context.request.tools.push({
+                tool: {
+                    name: 'read_skill',
+                    description: 'Load a registered skill or one of its supporting UTF-8 text files.',
+                    inputSchema: {
+                        type: 'object',
+                        properties: {
+                            name: { type: 'string', enum: [...catalog.keys()] },
+                            path: { type: 'string', description: 'Path relative to the skill directory. Omit to load SKILL.md.' }
+                        },
+                        required: ['name'],
+                        additionalProperties: false
+                    }
+                },
+                async callback(input, signal) {
+                    signal?.throwIfAborted();
+                    if (!input || typeof input !== 'object' || Array.isArray(input) ||
+                        Object.keys(input).some(key => key !== 'name' && key !== 'path')) {
+                        throw new TypeError('read_skill expects a name and an optional path.');
+                    }
+                    const skill = catalog.get(input.name);
+                    if (!skill) throw new Error(`Unknown skill: ${input.name}`);
+                    const relativePath = input.path === undefined ? 'SKILL.md' : input.path;
+                    if (relativePath === 'SKILL.md') {
+                        return { name: skill.name, path: relativePath, content: skill.content };
+                    }
+                    const filename = await resolveFile(skill.root, relativePath, signal);
+                    const content = filename === skill.skillPath ? skill.content : await readText(filename, signal);
+                    return { name: skill.name, path: relativePath, content };
+                }
+            });
+            return next();
+        }
+    };
+}
+
+module.exports = { skills };