modelmix 5.0.2 → 5.0.4

package/plugins/rlm/test/planner-template.test.js

@@ -0,0 +1,140 @@
+const { expect } = require('chai');
+const { MixCustom, ModelMix } = require('../../../index.js');
+const { createPlannerInvocation } = require('..');
+
+function createProvider(handler) {
+    const provider = new MixCustom();
+    provider.create = handler;
+    return provider;
+}
+
+describe('RLM planner Markdown template', () => {
+    const limits = {
+        maxQueryBytes: 64,
+        sandboxMemoryBytes: 64 * 1024 * 1024,
+        maxConcurrentQueries: 4,
+        maxCalls: 20,
+        maxOutputBytes: 1024 * 1024,
+        maxGeneratedTokens: 10000,
+        maxWallTimeMs: 30000
+    };
+    const workerManifest = {
+        translator: {
+            intelligence: 3,
+            cost: 2,
+            speed: 4,
+            description: 'Translate external text'
+        }
+    };
+
+    it('renders the planner Markdown through child assign() and its relative include', async () => {
+        const task = 'Translate the external book chapter by chapter.';
+        const variables = {
+            chapters: [{
+                heading: 'Hidden chapter title',
+                content: 'Hidden paragraph one.\n\nHidden paragraph two.'
+            }]
+        };
+        let plannerRequest;
+        const model = ModelMix.new()
+            .attach('planner', createProvider(async request => {
+                plannerRequest = request;
+                return { message: '(async () => "translated")()', toolCalls: [] };
+            }))
+            .use({
+                name: 'rlm',
+                execute(context) {
+                    return context.invoke(createPlannerInvocation({
+                        task,
+                        variables,
+                        limits,
+                        workerManifest
+                    }));
+                }
+            })
+            .addText('Original request intercepted by RLM.');
+
+        const result = await model.raw();
+
+        expect(result.message).to.equal('(async () => "translated")()');
+        expect(plannerRequest.config.system).to.include('# Recursive Language Model Planner');
+        expect(plannerRequest.config.system).to.include('## External variable manifest');
+        expect(plannerRequest.config.system).to.include('"items": 1');
+        expect(plannerRequest.config.system).to.include('"maxQueryBytes": 64');
+        expect(plannerRequest.config.system).to.include('## Worker catalog');
+        expect(plannerRequest.config.system).to.include('"translator"');
+        expect(plannerRequest.config.system).to.include('## Output requirements');
+        expect(plannerRequest.config.system).to.include('"mode": "raw"');
+        expect(plannerRequest.config.system).to.include('## Required processing rules');
+        expect(plannerRequest.config.system).to.include('at most 4 active queries');
+        expect(plannerRequest.options.messages).to.deep.equal([{
+            role: 'user',
+            content: [{ type: 'text', text: task }]
+        }]);
+        expect(plannerRequest.config.system).to.not.include(variables.chapters[0].heading);
+        expect(plannerRequest.config.system).to.not.include(variables.chapters[0].content);
+    });
+
+    it('keeps EJS-looking variable paths as data instead of rendering them twice', async () => {
+        const marker = '<%- leakedTemplateValue %>';
+        let plannerSystem;
+        const model = ModelMix.new()
+            .attach('planner', createProvider(async ({ config }) => {
+                plannerSystem = config.system;
+                return { message: '(async () => null)()', toolCalls: [] };
+            }))
+            .use({
+                name: 'rlm',
+                execute(context) {
+                    return context.invoke(createPlannerInvocation({
+                        task: 'Inspect the external variable.',
+                        variables: { [marker]: 'hidden payload' },
+                        limits,
+                        workerManifest
+                    }));
+                }
+            })
+            .assign({ leakedTemplateValue: 'must not execute' })
+            .addText('parent');
+
+        await model.raw();
+
+        expect(plannerSystem).to.include(marker);
+        expect(plannerSystem).to.not.include('must not execute');
+        expect(plannerSystem).to.not.include('hidden payload');
+    });
+
+    it('fails before provider execution when planner template data is incomplete', async () => {
+        let providerCalls = 0;
+        const model = ModelMix.new()
+            .attach('planner', createProvider(async () => {
+                providerCalls += 1;
+                return { message: 'unexpected', toolCalls: [] };
+            }))
+            .use({
+                name: 'rlm',
+                execute(context) {
+                    const invocation = createPlannerInvocation({
+                        task: 'Plan this task.',
+                        variables: { input: 'hidden' },
+                        limits,
+                        workerManifest
+                    });
+                    delete invocation.assign.planningHints;
+                    return context.invoke(invocation);
+                }
+            })
+            .addText('parent');
+
+        let failure;
+        try {
+            await model.raw();
+        } catch (error) {
+            failure = error;
+        }
+
+        expect(failure).to.be.instanceOf(Error);
+        expect(failure.message).to.include('planningHints');
+        expect(providerCalls).to.equal(0);
+    });
+});

package/plugins/rlm/test/plugin-contract.test.js

@@ -0,0 +1,182 @@
+const { expect } = require('chai');
+const { MixCustom, ModelMix } = require('../../../index.js');
+const { RlmLimitError, rlm } = require('..');
+
+function createProvider(handler) {
+    const provider = new MixCustom();
+    provider.create = handler;
+    return provider;
+}
+
+function baseOptions(overrides = {}) {
+    const workerModel = ModelMix.new().attach('worker', createProvider(async () => ({
+        message: 'worker',
+        toolCalls: []
+    })));
+    return {
+        maxDepth: 0,
+        variables: { input: 'hidden input' },
+        workers: {
+            worker: {
+                model: workerModel,
+                intelligence: 1,
+                cost: 1,
+                speed: 1,
+                description: 'Test worker'
+            }
+        },
+        limits: {
+            maxQueryBytes: 1024,
+            sandboxMemoryBytes: 1024 * 1024,
+            maxConcurrentQueries: 1,
+            maxCalls: 5,
+            maxOutputBytes: 1024,
+            maxGeneratedTokens: 100,
+            maxWallTimeMs: 1000
+        },
+        sandbox: {
+            async execute() {
+                return 'done';
+            }
+        },
+        ...overrides
+    };
+}
+
+function plannerModel(plannerMessage, options = baseOptions()) {
+    return ModelMix.new()
+        .attach('planner', createProvider(async () => ({
+            message: plannerMessage,
+            toolCalls: []
+        })))
+        .use(rlm(options))
+        .addText('Plan this operation.');
+}
+
+describe('RLM plugin contract', () => {
+    it('fails at registration when required policy is absent or invalid', () => {
+        expect(() => rlm()).to.throw('maxDepth');
+        expect(() => rlm(baseOptions({ maxDepth: -1 }))).to.throw('maxDepth');
+        expect(() => rlm(baseOptions({ variables: null }))).to.throw('variables');
+        expect(() => rlm(baseOptions({ sandbox: {} }))).to.throw('sandbox');
+        expect(() => rlm(baseOptions({
+            limits: { ...baseOptions().limits, maxCalls: undefined }
+        }))).to.throw('limits.maxCalls');
+    });
+
+    it('rejects streaming and malformed planner programs explicitly', async () => {
+        let streamFailure;
+        try {
+            await plannerModel('(async () => "ok")()').stream(() => {});
+        } catch (error) {
+            streamFailure = error;
+        }
+        expect(streamFailure).to.be.instanceOf(Error);
+        expect(streamFailure.message).to.include('streaming is not supported');
+
+        let syntaxFailure;
+        try {
+            await plannerModel('```js\n(async () => "bad")()\n```').raw();
+        } catch (error) {
+            syntaxFailure = error;
+        }
+        expect(syntaxFailure).to.be.instanceOf(SyntaxError);
+        expect(syntaxFailure.message).to.include('Markdown fences');
+        expect(syntaxFailure.rlm.terminationReason).to.equal('error');
+    });
+
+    it('enforces wall time and attaches limit diagnostics to the error', async () => {
+        const options = baseOptions({
+            limits: { ...baseOptions().limits, maxWallTimeMs: 20 },
+            sandbox: {
+                execute() {
+                    return new Promise(() => {});
+                }
+            }
+        });
+        let failure;
+        try {
+            await plannerModel('(async () => "slow")()', options).raw();
+        } catch (error) {
+            failure = error;
+        }
+
+        expect(failure).to.be.instanceOf(RlmLimitError);
+        expect(failure.limit).to.equal('maxWallTimeMs');
+        expect(failure.rlm.terminationReason).to.equal('limit:maxWallTimeMs');
+    });
+
+    it('preserves message, block, JSON, and raw caller contracts', async () => {
+        const options = baseOptions({
+            sandbox: {
+                async execute() {
+                    return { answer: 'ok' };
+                }
+            }
+        });
+        const createModel = () => plannerModel('(async () => ({ answer: "ok" }))()', options);
+
+        expect(await createModel().message()).to.equal('{"answer":"ok"}');
+        expect(await createModel().block()).to.equal('{"answer":"ok"}');
+        expect(await createModel().json({ answer: '' })).to.deep.equal({ answer: 'ok' });
+        expect(await createModel().raw()).to.deep.include({ message: '{"answer":"ok"}' });
+    });
+
+    it('can route a named worker through the inherited parent model chain', async () => {
+        const requests = [];
+        const provider = createProvider(async request => {
+            requests.push(request);
+            return {
+                message: requests.length === 1
+                    ? '(async () => "planned")()'
+                    : `parent:${request.options.messages[0].content}`,
+                toolCalls: []
+            };
+        });
+        const model = ModelMix.new()
+            .attach('parent', provider)
+            .use(rlm({
+                maxDepth: 0,
+                workers: {
+                    parent: {
+                        useParent: true,
+                        intelligence: 3,
+                        cost: 2,
+                        speed: 3,
+                        description: 'Current parent chain'
+                    }
+                },
+                limits: baseOptions().limits,
+                sandbox: {
+                    execute({ query }) {
+                        return query({
+                            worker: 'parent',
+                            system: 'Process this fragment.',
+                            message: 'payload'
+                        });
+                    }
+                }
+            }))
+            .addText('Plan this operation.');
+
+        expect(await model.message()).to.equal('parent:payload');
+        expect(requests).to.have.length(2);
+        expect(requests[1].config.system).to.equal('Process this fragment.');
+    });
+
+    it('supports an abstract task without document input', async () => {
+        const options = baseOptions({
+            variables: undefined,
+            sandbox: {
+                execute({ variables }) {
+                    return { externalVariables: Object.keys(variables).length };
+                }
+            }
+        });
+
+        expect(await plannerModel('(async () => ({ externalVariables: 0 }))()', options)
+            .json({ externalVariables: 0 })).to.deep.equal({
+            externalVariables: 0
+        });
+    });
+});

package/plugins/rlm/test/rlm-e2e.test.js

@@ -0,0 +1,338 @@
+const { expect } = require('chai');
+const { MixCustom, ModelMix } = require('../../../index.js');
+const { rlm } = require('..');
+
+function createProvider(handler) {
+    const provider = new MixCustom();
+    provider.create = handler;
+    return provider;
+}
+
+function limits(overrides = {}) {
+    return {
+        maxQueryBytes: 2048,
+        sandboxMemoryBytes: 64 * 1024 * 1024,
+        maxConcurrentQueries: 2,
+        maxCalls: 30,
+        maxOutputBytes: 1024 * 1024,
+        maxGeneratedTokens: 10000,
+        maxWallTimeMs: 30000,
+        ...overrides
+    };
+}
+
+function worker(model, description) {
+    return {
+        model,
+        intelligence: 3,
+        cost: 2,
+        speed: 4,
+        description
+    };
+}
+
+describe('RLM mocked end-to-end execution', () => {
+    it('translates a book by chapter, recursively splits paragraphs, and preserves order', async () => {
+        const book = {
+            chapters: [
+                {
+                    heading: 'Secret chapter one',
+                    content: 'one slow\n\ntwo fast'
+                },
+                {
+                    heading: 'Secret chapter two',
+                    content: 'three fast\n\nfour slow'
+                }
+            ]
+        };
+        const plannerRequests = [];
+        const leafRequests = [];
+        let activeLeafCalls = 0;
+        let peakLeafCalls = 0;
+        const plannerProvider = createProvider(async request => {
+            plannerRequests.push(request);
+            return {
+                message: '(async () => "root plan")()',
+                toolCalls: [],
+                tokens: { input: 10, output: 2, total: 12, cost: 0.001 }
+            };
+        });
+        const workerProvider = createProvider(async request => {
+            if (request.config.system.includes('# Recursive Language Model Planner')) {
+                plannerRequests.push(request);
+                return {
+                    message: '(async () => "chapter plan")()',
+                    toolCalls: [],
+                    tokens: { input: 8, output: 2, total: 10, cost: 0.0005 }
+                };
+            }
+            leafRequests.push(request);
+            activeLeafCalls += 1;
+            peakLeafCalls = Math.max(peakLeafCalls, activeLeafCalls);
+            const paragraph = request.options.messages[0].content;
+            await new Promise(resolve => setTimeout(
+                resolve,
+                paragraph.includes('slow') ? 20 : 2
+            ));
+            activeLeafCalls -= 1;
+            return {
+                message: `ES:${paragraph}`,
+                toolCalls: [],
+                tokens: { input: 3, output: 2, total: 5, cost: 0.0001 }
+            };
+        });
+        const translator = ModelMix.new().attach('translator', workerProvider);
+        const sandbox = {
+            async execute({ variables, query }) {
+                if (variables.chapters) {
+                    const chapters = await Promise.all(variables.chapters.map(chapter => query({
+                        worker: 'translator',
+                        system: 'Translate this chapter to Spanish, preserving paragraph order.',
+                        message: chapter.content
+                    })));
+                    return chapters.join('\n\n# CHAPTER\n\n');
+                }
+                const paragraphs = variables.input.split(/\n\n+/);
+                const translated = await Promise.all(paragraphs.map(paragraph => query({
+                    worker: 'translator',
+                    system: 'Translate this paragraph to Spanish.',
+                    message: paragraph
+                })));
+                return translated.join('\n\n');
+            }
+        };
+        const model = ModelMix.new()
+            .attach('planner', plannerProvider)
+            .use(rlm({
+                maxDepth: 1,
+                variables: book,
+                workers: {
+                    translator: worker(translator, 'Translation and rewriting')
+                },
+                limits: limits(),
+                sandbox
+            }))
+            .addText('Translate this book to neutral Latin American Spanish.');
+
+        const result = await model.raw();
+
+        expect(result.message).to.equal([
+            'ES:one slow',
+            'ES:two fast',
+            '# CHAPTER',
+            'ES:three fast',
+            'ES:four slow'
+        ].join('\n\n'));
+        expect(plannerRequests).to.have.length(3);
+        expect(leafRequests).to.have.length(4);
+        expect(peakLeafCalls).to.equal(2);
+        for (const request of plannerRequests) {
+            expect(request.options.messages).to.have.length(1);
+            expect(request.config.system).to.include('# Recursive Language Model Planner');
+            for (const chapter of book.chapters) {
+                expect(request.config.system).to.not.include(chapter.heading);
+                expect(request.config.system).to.not.include(chapter.content);
+            }
+        }
+        for (const request of leafRequests) {
+            expect(request.options.messages).to.have.length(1);
+        }
+        expect(result.rlm.terminationReason).to.equal('completed');
+        expect(result.rlm.budget).to.include({
+            calls: 7,
+            peakConcurrency: 2
+        });
+        expect(result.rlm.calls.filter(call => call.kind === 'planner')).to.have.length(3);
+        expect(result.rlm.calls.filter(call => call.kind === 'worker' && call.directLeaf))
+            .to.have.length(4);
+        expect(result.tokens).to.deep.include({
+            input: 38,
+            output: 14,
+            total: 52
+        });
+    });
+
+    it('supports a non-translation operation and reports invalid worker choices', async () => {
+        const records = [
+            'Ada works in Engineering.',
+            'Lin works in Design.'
+        ];
+        const classificationPlannerSystems = [];
+        const planner = createProvider(async request => {
+            classificationPlannerSystems.push(request.config.system);
+            return {
+                message: '(async () => "classification plan")()',
+                toolCalls: []
+            };
+        });
+        const classifier = ModelMix.new().attach('classifier', createProvider(async request => ({
+            message: request.options.messages[0].content.includes('Engineering') ? 'engineering' : 'other',
+            toolCalls: []
+        })));
+        const sandbox = {
+            async execute({ variables, query }) {
+                const departments = await Promise.all(variables.records.map(message => query({
+                    worker: 'classifier',
+                    system: 'Classify the department.',
+                    message
+                })));
+                return { departments };
+            }
+        };
+        const model = ModelMix.new()
+            .attach('planner', planner)
+            .use(rlm({
+                maxDepth: 0,
+                variables: { records },
+                workers: {
+                    classifier: worker(classifier, 'Simple classification')
+                },
+                limits: limits(),
+                sandbox
+            }))
+            .addText('Classify every external record.');
+
+        expect(await model.json({ departments: [''] })).to.deep.equal({
+            departments: ['engineering', 'other']
+        });
+        expect(classificationPlannerSystems[0]).to.include('"mode": "json"');
+        expect(classificationPlannerSystems[0]).to.include('"departments"');
+
+        const invalidModel = ModelMix.new()
+            .attach('planner', planner)
+            .use(rlm({
+                maxDepth: 0,
+                variables: { records },
+                workers: {
+                    classifier: worker(classifier, 'Simple classification')
+                },
+                limits: limits(),
+                sandbox: {
+                    execute({ query }) {
+                        return query({
+                            worker: 'missing',
+                            system: 'Classify.',
+                            message: records[0]
+                        });
+                    }
+                }
+            }))
+            .addText('Classify records.');
+        let failure;
+        try {
+            await invalidModel.raw();
+        } catch (error) {
+            failure = error;
+        }
+        expect(failure).to.be.instanceOf(Error);
+        expect(failure.message).to.include('Unknown RLM worker "missing"');
+    });
+
+    it('translates a Markdown book through semantic chapters and paragraphs in isolated-vm', async () => {
+        const markdown = [
+            '# First chapter',
+            '',
+            'one slow',
+            '',
+            'two fast',
+            '',
+            '# Second chapter',
+            '',
+            'three fast',
+            '',
+            'four slow',
+            ''
+        ].join('\n');
+        const plannerRequests = [];
+        const leafRequests = [];
+        let activeLeafCalls = 0;
+        let peakLeafCalls = 0;
+        const rootProgram = `(async () => {
+            const translations = await Promise.all(variables.book.sections.map(chapter => query({
+                worker: 'translator',
+                system: 'Translate this chapter to Spanish, preserving paragraph order.',
+                message: chapter.body.trim()
+            })));
+            return translations.map((translation, index) => (
+                variables.book.sections[index].heading + '\\n\\n' + translation
+            )).join('\\n\\n');
+        })()`;
+        const chapterProgram = `(async () => {
+            const paragraphs = variables.input.split(/\\n\\n+/);
+            const translations = await Promise.all(paragraphs.map(paragraph => query({
+                worker: 'translator',
+                system: 'Translate this paragraph to Spanish.',
+                message: paragraph
+            })));
+            return translations.join('\\n\\n');
+        })()`;
+        const plannerProvider = createProvider(async request => {
+            plannerRequests.push(request);
+            return {
+                message: rootProgram,
+                toolCalls: []
+            };
+        });
+        const translator = ModelMix.new().attach('translator', createProvider(async request => {
+            if (request.config.system.includes('# Recursive Language Model Planner')) {
+                plannerRequests.push(request);
+                return {
+                    message: chapterProgram,
+                    toolCalls: []
+                };
+            }
+            leafRequests.push(request);
+            activeLeafCalls += 1;
+            peakLeafCalls = Math.max(peakLeafCalls, activeLeafCalls);
+            const paragraph = request.options.messages[0].content;
+            await new Promise(resolve => setTimeout(
+                resolve,
+                paragraph.includes('slow') ? 20 : 2
+            ));
+            activeLeafCalls -= 1;
+            return {
+                message: `ES:${paragraph}`,
+                toolCalls: []
+            };
+        }));
+        const model = ModelMix.new()
+            .attach('planner', plannerProvider)
+            .use(rlm({
+                maxDepth: 1,
+                documents: {
+                    book: { format: 'markdown', content: markdown }
+                },
+                workers: {
+                    translator: worker(translator, 'Translation and rewriting')
+                },
+                limits: limits()
+            }))
+            .addText('Translate this book to neutral Latin American Spanish.');
+
+        const result = await model.raw();
+
+        expect(result.message).to.equal([
+            '# First chapter',
+            '',
+            'ES:one slow',
+            '',
+            'ES:two fast',
+            '',
+            '# Second chapter',
+            '',
+            'ES:three fast',
+            '',
+            'ES:four slow'
+        ].join('\n'));
+        expect(plannerRequests).to.have.length(3);
+        expect(leafRequests).to.have.length(4);
+        expect(peakLeafCalls).to.equal(2);
+        for (const request of plannerRequests) {
+            expect(request.config.system).to.not.include('one slow');
+            expect(request.config.system).to.not.include('four slow');
+            expect(request.config.system).to.include('"utf8Bytes"');
+        }
+        expect(plannerRequests[0].config.system).to.include('"path": "book.sections"');
+        expect(plannerRequests[0].config.system).to.include('"items": 2');
+    });
+});