codevault 1.8.4 → 1.8.5

package/dist/tests/providers/token-counter.test.js

@@ -0,0 +1,247 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { getTokenCounter } from '../../providers/token-counter.js';
+// ============= getTokenCounter basic tests =============
+test('getTokenCounter returns function for text-embedding models', async () => {
+    const counter = await getTokenCounter('text-embedding-3-large');
+    if (counter) {
+        assert.equal(typeof counter, 'function');
+    }
+    // If tiktoken not available, counter will be null
+    // This is acceptable behavior
+});
+test('getTokenCounter returns function for text-embedding-3-small', async () => {
+    const counter = await getTokenCounter('text-embedding-3-small');
+    if (counter) {
+        assert.equal(typeof counter, 'function');
+    }
+});
+test('getTokenCounter returns function for ada-002 model', async () => {
+    const counter = await getTokenCounter('text-embedding-ada-002');
+    if (counter) {
+        assert.equal(typeof counter, 'function');
+    }
+});
+test('getTokenCounter returns character-based counter for non-embedding models', async () => {
+    const counter = await getTokenCounter('gpt-4');
+    assert.ok(counter, 'Counter should exist for non-embedding models');
+    assert.equal(typeof counter, 'function');
+});
+test('getTokenCounter returns character-based counter for unknown models', async () => {
+    const counter = await getTokenCounter('unknown-model-xyz');
+    assert.ok(counter, 'Counter should exist');
+    assert.equal(typeof counter, 'function');
+});
+// ============= Character-based estimation tests =============
+test('character-based counter estimates 1 token per 4 characters', async () => {
+    const counter = await getTokenCounter('gpt-4');
+    assert.ok(counter, 'Counter should exist');
+    assert.equal(counter('abcd'), 1); // 4 chars = 1 token
+    assert.equal(counter('abcdefgh'), 2); // 8 chars = 2 tokens
+});
+test('character-based counter rounds up', async () => {
+    const counter = await getTokenCounter('some-chat-model');
+    assert.ok(counter, 'Counter should exist');
+    assert.equal(counter('abc'), 1); // 3 chars, ceil(3/4) = 1
+    assert.equal(counter('abcde'), 2); // 5 chars, ceil(5/4) = 2
+});
+test('character-based counter handles empty string', async () => {
+    const counter = await getTokenCounter('chat-model');
+    assert.ok(counter, 'Counter should exist');
+    assert.equal(counter(''), 0);
+});
+test('character-based counter handles long strings', async () => {
+    const counter = await getTokenCounter('any-model');
+    assert.ok(counter, 'Counter should exist');
+    const longText = 'a'.repeat(10000);
+    assert.equal(counter(longText), 2500);
+});
+// ============= Tiktoken-based counter tests (when available) =============
+test('tiktoken counter returns positive count for text', async () => {
+    const counter = await getTokenCounter('text-embedding-3-large');
+    if (counter) {
+        const count = counter('Hello, world!');
+        assert.equal(typeof count, 'number');
+        assert.ok(count > 0, 'Token count should be positive');
+    }
+});
+test('tiktoken counter handles code snippets', async () => {
+    const counter = await getTokenCounter('text-embedding-ada-002');
+    if (counter) {
+        const code = 'function hello() { return "world"; }';
+        const count = counter(code);
+        assert.equal(typeof count, 'number');
+        assert.ok(count > 0);
+    }
+});
+test('tiktoken counter handles unicode', async () => {
+    const counter = await getTokenCounter('text-embedding-3-small');
+    if (counter) {
+        const unicode = 'Hello World';
+        const count = counter(unicode);
+        assert.equal(typeof count, 'number');
+    }
+});
+test('tiktoken counter handles empty string', async () => {
+    const counter = await getTokenCounter('text-embedding-3-large');
+    if (counter) {
+        const count = counter('');
+        assert.equal(count, 0);
+    }
+});
+test('tiktoken counter handles whitespace', async () => {
+    const counter = await getTokenCounter('text-embedding-ada-002');
+    if (counter) {
+        const whitespace = '   \n\t   ';
+        const count = counter(whitespace);
+        assert.equal(typeof count, 'number');
+        assert.ok(count >= 0);
+    }
+});
+// ============= Encoder caching tests =============
+test('getTokenCounter reuses encoder for same model family', async () => {
+    // Call twice for embedding models - should reuse encoder
+    const counter1 = await getTokenCounter('text-embedding-3-large');
+    const counter2 = await getTokenCounter('text-embedding-3-small');
+    // Both should work (either tiktoken or fallback)
+    if (counter1 && counter2) {
+        const count1 = counter1('test');
+        const count2 = counter2('test');
+        assert.equal(typeof count1, 'number');
+        assert.equal(typeof count2, 'number');
+    }
+});
+test('getTokenCounter called multiple times returns consistent results', async () => {
+    const counter1 = await getTokenCounter('text-embedding-3-large');
+    const counter2 = await getTokenCounter('text-embedding-3-large');
+    if (counter1 && counter2) {
+        const text = 'This is a test sentence for token counting.';
+        assert.equal(counter1(text), counter2(text));
+    }
+});
+// ============= Model name pattern matching tests =============
+test('getTokenCounter matches "text-embedding" pattern', async () => {
+    // These should use tiktoken if available
+    const models = [
+        'text-embedding-3-large',
+        'text-embedding-3-small',
+        'text-embedding-ada-002',
+        'some-text-embedding-model'
+    ];
+    for (const model of models) {
+        const counter = await getTokenCounter(model);
+        // Should return counter (tiktoken or fallback)
+        // For text-embedding models, may return null if tiktoken unavailable
+        // which is valid behavior
+        assert.ok(counter === null || typeof counter === 'function');
+    }
+});
+test('getTokenCounter matches "ada-002" pattern', async () => {
+    const counter = await getTokenCounter('ada-002');
+    // Should use tiktoken if available
+    assert.ok(counter === null || typeof counter === 'function');
+});
+test('getTokenCounter uses fallback for non-matching models', async () => {
+    const models = [
+        'gpt-4',
+        'gpt-3.5-turbo',
+        'claude-2',
+        'custom-llm',
+        ''
+    ];
+    for (const model of models) {
+        const counter = await getTokenCounter(model);
+        assert.ok(counter, `Counter should exist for "${model}"`);
+        // Should use character-based estimation
+        assert.equal(counter('abcd'), 1, `Should estimate 1 token for 4 chars with "${model}"`);
+    }
+});
+// ============= Edge cases =============
+test('getTokenCounter handles special characters', async () => {
+    const counter = await getTokenCounter('text-embedding-3-large');
+    if (counter) {
+        const special = '!@#$%^&*()_+-=[]{}|;:\'",.<>?/`~';
+        const count = counter(special);
+        assert.equal(typeof count, 'number');
+        assert.ok(count >= 0);
+    }
+});
+test('getTokenCounter handles newlines and tabs', async () => {
+    const counter = await getTokenCounter('text-embedding-ada-002');
+    if (counter) {
+        const text = 'line1\nline2\tline3';
+        const count = counter(text);
+        assert.equal(typeof count, 'number');
+        assert.ok(count > 0);
+    }
+});
+test('getTokenCounter handles very long text', async () => {
+    const counter = await getTokenCounter('gpt-4');
+    assert.ok(counter, 'Counter should exist');
+    const longText = 'word '.repeat(10000);
+    const count = counter(longText);
+    assert.equal(typeof count, 'number');
+    assert.ok(count > 0);
+});
+test('getTokenCounter handles JSON content', async () => {
+    const counter = await getTokenCounter('text-embedding-3-small');
+    if (counter) {
+        const json = JSON.stringify({ key: 'value', nested: { array: [1, 2, 3] } });
+        const count = counter(json);
+        assert.equal(typeof count, 'number');
+        assert.ok(count > 0);
+    }
+});
+test('getTokenCounter handles code with comments', async () => {
+    const counter = await getTokenCounter('text-embedding-3-large');
+    if (counter) {
+        const code = `
+// This is a comment
+function test() {
+  /* Multi-line
+     comment */
+  return 42;
+}
+`;
+        const count = counter(code);
+        assert.equal(typeof count, 'number');
+        assert.ok(count > 0);
+    }
+});
+// ============= Consistency tests =============
+test('getTokenCounter produces consistent counts for same input', async () => {
+    const counter = await getTokenCounter('text-embedding-3-large');
+    if (counter) {
+        const text = 'Consistent counting test';
+        const count1 = counter(text);
+        const count2 = counter(text);
+        const count3 = counter(text);
+        assert.equal(count1, count2);
+        assert.equal(count2, count3);
+    }
+});
+test('getTokenCounter character fallback is deterministic', async () => {
+    const counter = await getTokenCounter('some-random-model');
+    assert.ok(counter, 'Counter should exist');
+    const text = 'Deterministic test';
+    const counts = [counter(text), counter(text), counter(text)];
+    assert.equal(counts[0], counts[1]);
+    assert.equal(counts[1], counts[2]);
+});
+// ============= Return type tests =============
+test('getTokenCounter always returns number (not Promise) from returned function', async () => {
+    const counter = await getTokenCounter('gpt-4');
+    assert.ok(counter, 'Counter should exist');
+    const result = counter('test');
+    assert.equal(typeof result, 'number');
+    // Ensure it's not a thenable (Promise-like) - typeof check already confirms it's number
+    assert.ok(typeof result === 'number', 'Result should be a number, not a Promise');
+});
+test('getTokenCounter for embedding model returns sync function', async () => {
+    const counter = await getTokenCounter('text-embedding-3-large');
+    if (counter) {
+        const result = counter('test');
+        assert.equal(typeof result, 'number');
+    }
+});
+//# sourceMappingURL=token-counter.test.js.map

package/dist/tests/providers/token-counter.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"token-counter.test.js","sourceRoot":"","sources":["../../../src/tests/providers/token-counter.test.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,MAAM,MAAM,oBAAoB,CAAC;AACxC,OAAO,EAAE,eAAe,EAAE,MAAM,kCAAkC,CAAC;AAEnE,0DAA0D;AAC1D,IAAI,CAAC,4DAA4D,EAAE,KAAK,IAAI,EAAE;IAC5E,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,wBAAwB,CAAC,CAAC;IAEhE,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,CAAC,KAAK,CAAC,OAAO,OAAO,EAAE,UAAU,CAAC,CAAC;IAC3C,CAAC;IACD,kDAAkD;IAClD,8BAA8B;AAChC,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,6DAA6D,EAAE,KAAK,IAAI,EAAE;IAC7E,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,wBAAwB,CAAC,CAAC;IAEhE,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,CAAC,KAAK,CAAC,OAAO,OAAO,EAAE,UAAU,CAAC,CAAC;IAC3C,CAAC;AACH,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,oDAAoD,EAAE,KAAK,IAAI,EAAE;IACpE,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,wBAAwB,CAAC,CAAC;IAEhE,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,CAAC,KAAK,CAAC,OAAO,OAAO,EAAE,UAAU,CAAC,CAAC;IAC3C,CAAC;AACH,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,0EAA0E,EAAE,KAAK,IAAI,EAAE;IAC1F,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,OAAO,CAAC,CAAC;IAE/C,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,+CAA+C,CAAC,CAAC;IACpE,MAAM,CAAC,KAAK,CAAC,OAAO,OAAO,EAAE,UAAU,CAAC,CAAC;AAC3C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,oEAAoE,EAAE,KAAK,IAAI,EAAE;IACpF,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,mBAAmB,CAAC,CAAC;IAE3D,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,sBAAsB,CAAC,CAAC;IAC3C,MAAM,CAAC,KAAK,CAAC,OAAO,OAAO,EAAE,UAAU,CAAC,CAAC;AAC3C,CAAC,CAAC,CAAC;AAEH,+DAA+D;AAC/D,IAAI,CAAC,4DAA4D,EAAE,KAAK,IAAI,EAAE;IAC5E,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,OAAO,CAAC,CAAC;IAE/C,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,sBAAsB,CAAC,CAAC;IAC3C,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,oBAAoB;IACtD,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,qBAAqB;AAC7D,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,mCAAmC,EAAE,KAAK,IAAI,EAAE;IACnD,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,iBAAiB,CAAC,CAAC;IAEzD,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,sBAAsB,CAAC,CAAC;IAC3C,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,yBAAyB;IAC1D,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,yBAAyB;AAC9D,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;IAC9D,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,YAAY,CAAC,CAAC;IAEpD,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,sBAAsB,CAAC,CAAC;IAC3C,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;AAC/B,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;IAC9D,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,WAAW,CAAC,CAAC;IAEnD,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,sBAAsB,CAAC,CAAC;IAC3C,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACnC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,IAAI,CAAC,CAAC;AACxC,CAAC,CAAC,CAAC;AAEH,4EAA4E;AAC5E,IAAI,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;IAClE,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,wBAAwB,CAAC,CAAC;IAEhE,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,KAAK,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;QACvC,MAAM,CAAC,KAAK,CAAC,OAAO,KAAK,EAAE,QAAQ,CAAC,CAAC;QACrC,MAAM,CAAC,EAAE,CAAC,KAAK,GAAG,CAAC,EAAE,gCAAgC,CAAC,CAAC;IACzD,CAAC;AACH,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,wCAAwC,EAAE,KAAK,IAAI,EAAE;IACxD,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,wBAAwB,CAAC,CAAC;IAEhE,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,IAAI,GAAG,sCAAsC,CAAC;QACpD,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;QAC5B,MAAM,CAAC,KAAK,CAAC,OAAO,KAAK,EAAE,QAAQ,CAAC,CAAC;QACrC,MAAM,CAAC,EAAE,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IACvB,CAAC;AACH,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,kCAAkC,EAAE,KAAK,IAAI,EAAE;IAClD,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,wBAAwB,CAAC,CAAC;IAEhE,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,OAAO,GAAG,aAAa,CAAC;QAC9B,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;QAC/B,MAAM,CAAC,KAAK,CAAC,OAAO,KAAK,EAAE,QAAQ,CAAC,CAAC;IACvC,CAAC;AACH,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;IACvD,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,wBAAwB,CAAC,CAAC;IAEhE,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,KAAK,GAAG,OAAO,CAAC,EAAE,CAAC,CAAC;QAC1B,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IACzB,CAAC;AACH,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,qCAAqC,EAAE,KAAK,IAAI,EAAE;IACrD,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,wBAAwB,CAAC,CAAC;IAEhE,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,UAAU,GAAG,YAAY,CAAC;QAChC,MAAM,KAAK,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;QAClC,MAAM,CAAC,KAAK,CAAC,OAAO,KAAK,EAAE,QAAQ,CAAC,CAAC;QACrC,MAAM,CAAC,EAAE,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;IACxB,CAAC;AACH,CAAC,CAAC,CAAC;AAEH,oDAAoD;AACpD,IAAI,CAAC,sDAAsD,EAAE,KAAK,IAAI,EAAE;IACtE,yDAAyD;IACzD,MAAM,QAAQ,GAAG,MAAM,eAAe,CAAC,wBAAwB,CAAC,CAAC;IACjE,MAAM,QAAQ,GAAG,MAAM,eAAe,CAAC,wBAAwB,CAAC,CAAC;IAEjE,iDAAiD;IACjD,IAAI,QAAQ,IAAI,QAAQ,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;QAChC,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;QAEhC,MAAM,CAAC,KAAK,CAAC,OAAO,MAAM,EAAE,QAAQ,CAAC,CAAC;QACtC,MAAM,CAAC,KAAK,CAAC,OAAO,MAAM,EAAE,QAAQ,CAAC,CAAC;IACxC,CAAC;AACH,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,kEAAkE,EAAE,KAAK,IAAI,EAAE;IAClF,MAAM,QAAQ,GAAG,MAAM,eAAe,CAAC,wBAAwB,CAAC,CAAC;IACjE,MAAM,QAAQ,GAAG,MAAM,eAAe,CAAC,wBAAwB,CAAC,CAAC;IAEjE,IAAI,QAAQ,IAAI,QAAQ,EAAE,CAAC;QACzB,MAAM,IAAI,GAAG,6CAA6C,CAAC;QAC3D,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;IAC/C,CAAC;AACH,CAAC,CAAC,CAAC;AAEH,gEAAgE;AAChE,IAAI,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;IAClE,yCAAyC;IACzC,MAAM,MAAM,GAAG;QACb,wBAAwB;QACxB,wBAAwB;QACxB,wBAAwB;QACxB,2BAA2B;KAC5B,CAAC;IAEF,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,KAAK,CAAC,CAAC;QAC7C,+CAA+C;QAC/C,qEAAqE;QACrE,0BAA0B;QAC1B,MAAM,CAAC,EAAE,CAAC,OAAO,KAAK,IAAI,IAAI,OAAO,OAAO,KAAK,UAAU,CAAC,CAAC;IAC/D,CAAC;AACH,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;IAC3D,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,SAAS,CAAC,CAAC;IACjD,mCAAmC;IACnC,MAAM,CAAC,EAAE,CAAC,OAAO,KAAK,IAAI,IAAI,OAAO,OAAO,KAAK,UAAU,CAAC,CAAC;AAC/D,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,uDAAuD,EAAE,KAAK,IAAI,EAAE;IACvE,MAAM,MAAM,GAAG;QACb,OAAO;QACP,eAAe;QACf,UAAU;QACV,YAAY;QACZ,EAAE;KACH,CAAC;IAEF,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,KAAK,CAAC,CAAC;QAC7C,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,6BAA6B,KAAK,GAAG,CAAC,CAAC;QAC1D,wCAAwC;QACxC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,6CAA6C,KAAK,GAAG,CAAC,CAAC;IAC1F,CAAC;AACH,CAAC,CAAC,CAAC;AAEH,yCAAyC;AACzC,IAAI,CAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;IAC5D,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,wBAAwB,CAAC,CAAC;IAEhE,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,OAAO,GAAG,kCAAkC,CAAC;QACnD,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;QAC/B,MAAM,CAAC,KAAK,CAAC,OAAO,KAAK,EAAE,QAAQ,CAAC,CAAC;QACrC,MAAM,CAAC,EAAE,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;IACxB,CAAC;AACH,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;IAC3D,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,wBAAwB,CAAC,CAAC;IAEhE,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,IAAI,GAAG,qBAAqB,CAAC;QACnC,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;QAC5B,MAAM,CAAC,KAAK,CAAC,OAAO,KAAK,EAAE,QAAQ,CAAC,CAAC;QACrC,MAAM,CAAC,EAAE,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IACvB,CAAC;AACH,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,wCAAwC,EAAE,KAAK,IAAI,EAAE;IACxD,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,OAAO,CAAC,CAAC;IAE/C,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,sBAAsB,CAAC,CAAC;IAC3C,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACvC,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;IAChC,MAAM,CAAC,KAAK,CAAC,OAAO,KAAK,EAAE,QAAQ,CAAC,CAAC;IACrC,MAAM,CAAC,EAAE,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACvB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,sCAAsC,EAAE,KAAK,IAAI,EAAE;IACtD,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,wBAAwB,CAAC,CAAC;IAEhE,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC5E,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;QAC5B,MAAM,CAAC,KAAK,CAAC,OAAO,KAAK,EAAE,QAAQ,CAAC,CAAC;QACrC,MAAM,CAAC,EAAE,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IACvB,CAAC;AACH,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;IAC5D,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,wBAAwB,CAAC,CAAC;IAEhE,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,IAAI,GAAG;;;;;;;CAOhB,CAAC;QACE,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;QAC5B,MAAM,CAAC,KAAK,CAAC,OAAO,KAAK,EAAE,QAAQ,CAAC,CAAC;QACrC,MAAM,CAAC,EAAE,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IACvB,CAAC;AACH,CAAC,CAAC,CAAC;AAEH,gDAAgD;AAChD,IAAI,CAAC,2DAA2D,EAAE,KAAK,IAAI,EAAE;IAC3E,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,wBAAwB,CAAC,CAAC;IAEhE,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,IAAI,GAAG,0BAA0B,CAAC;QACxC,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;QAC7B,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;QAC7B,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;QAE7B,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC7B,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC/B,CAAC;AACH,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,qDAAqD,EAAE,KAAK,IAAI,EAAE;IACrE,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,mBAAmB,CAAC,CAAC;IAE3D,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,sBAAsB,CAAC,CAAC;IAE3C,MAAM,IAAI,GAAG,oBAAoB,CAAC;IAClC,MAAM,MAAM,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,OAAO,CAAC,IAAI,CAAC,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;IAE7D,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IACnC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;AACrC,CAAC,CAAC,CAAC;AAEH,gDAAgD;AAChD,IAAI,CAAC,4EAA4E,EAAE,KAAK,IAAI,EAAE;IAC5F,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,OAAO,CAAC,CAAC;IAE/C,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,sBAAsB,CAAC,CAAC;IAE3C,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAC/B,MAAM,CAAC,KAAK,CAAC,OAAO,MAAM,EAAE,QAAQ,CAAC,CAAC;IACtC,wFAAwF;IACxF,MAAM,CAAC,EAAE,CAAC,OAAO,MAAM,KAAK,QAAQ,EAAE,0CAA0C,CAAC,CAAC;AACpF,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,2DAA2D,EAAE,KAAK,IAAI,EAAE;IAC3E,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,wBAAwB,CAAC,CAAC;IAEhE,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QAC/B,MAAM,CAAC,KAAK,CAAC,OAAO,MAAM,EAAE,QAAQ,CAAC,CAAC;IACxC,CAAC;AACH,CAAC,CAAC,CAAC"}

package/dist/tests/rate-limiter.test.js

@@ -1,6 +1,36 @@
 import test from 'node:test';
 import assert from 'node:assert/strict';
-import { RateLimiter } from '../utils/rate-limiter.js';
+import { setTimeout as delay } from 'timers/promises';
+import { RateLimiter, createRateLimiter } from '../utils/rate-limiter.js';
+// ============================================================================
+// Constructor Tests
+// ============================================================================
+test('RateLimiter constructor with explicit RPM and TPM', () => {
+    const limiter = new RateLimiter(100, 10000);
+    const stats = limiter.getStats();
+    assert.equal(stats.rpm, 100);
+    assert.equal(stats.tpm, 10000);
+    assert.equal(stats.isLimited, true);
+});
+test('RateLimiter constructor with null limits uses defaults from environment', () => {
+    const limiter = new RateLimiter(null, null);
+    const stats = limiter.getStats();
+    // Default behavior when no env vars set - may or may not have limits
+    assert.ok(stats.rpm === null || typeof stats.rpm === 'number');
+});
+test('RateLimiter constructor with custom maxQueueSize', () => {
+    const limiter = new RateLimiter(100, null, 500);
+    const stats = limiter.getStats();
+    assert.equal(stats.maxQueueSize, 500);
+});
+test('RateLimiter default maxQueueSize is 10000', () => {
+    const limiter = new RateLimiter(100, null);
+    const stats = limiter.getStats();
+    assert.equal(stats.maxQueueSize, 10000);
+});
+// ============================================================================
+// Basic Execution Tests
+// ============================================================================
 test('RateLimiter executes tasks under default limits', async () => {
     const limiter = new RateLimiter(1000, null, 10);
     const result = await limiter.execute(() => Promise.resolve('ok'));
@@ -8,4 +38,365 @@ test('RateLimiter executes tasks under default limits', async () => {
     const stats = limiter.getStats();
     assert.ok(stats.requestsInLastMinute >= 0);
 });
+test('RateLimiter executes and returns correct value', async () => {
+    const limiter = new RateLimiter(100, null);
+    const result = await limiter.execute(async () => {
+        await delay(10);
+        return 42;
+    });
+    assert.equal(result, 42);
+});
+test('RateLimiter preserves error from executed function', async () => {
+    const limiter = new RateLimiter(100, null);
+    await assert.rejects(limiter.execute(async () => {
+        throw new Error('Task failed');
+    }), { message: 'Task failed' });
+});
+test('RateLimiter executes multiple tasks sequentially', async () => {
+    const limiter = new RateLimiter(100, null);
+    const results = [];
+    await limiter.execute(async () => { results.push(1); });
+    await limiter.execute(async () => { results.push(2); });
+    await limiter.execute(async () => { results.push(3); });
+    assert.deepEqual(results, [1, 2, 3]);
+});
+// ============================================================================
+// Queue Management Tests
+// ============================================================================
+test('RateLimiter rejects when queue is full', async () => {
+    const limiter = new RateLimiter(1, null, 2); // max 2 items in queue
+    // Fill the queue with slow tasks
+    const slowTask = async () => {
+        await delay(1000);
+        return 'done';
+    };
+    // Start tasks that will fill the queue
+    const task1 = limiter.execute(slowTask);
+    const task2 = limiter.execute(slowTask);
+    const task3 = limiter.execute(slowTask);
+    // Third task should be rejected since queue limit is 2
+    await assert.rejects(task3, {
+        message: 'Rate limiter queue is full (2 items). Too many concurrent requests.'
+    });
+    // Clean up - reset the limiter so tests don't hang
+    limiter.reset();
+});
+test('RateLimiter getStats shows queue utilization', async () => {
+    const limiter = new RateLimiter(1, null, 100);
+    const slowTask = async () => {
+        await delay(50);
+        return 'done';
+    };
+    // Add some tasks to queue
+    const tasks = [
+        limiter.execute(slowTask),
+        limiter.execute(slowTask),
+    ];
+    // Check queue stats
+    const stats = limiter.getStats();
+    assert.ok(stats.queueLength >= 0);
+    assert.ok(stats.queueUtilization.endsWith('%'));
+    // Wait for tasks to complete
+    limiter.reset();
+});
+// ============================================================================
+// RPM (Requests Per Minute) Limiting Tests
+// ============================================================================
+test('RateLimiter records requests in last minute', async () => {
+    const limiter = new RateLimiter(100, null);
+    await limiter.execute(() => Promise.resolve(1));
+    await limiter.execute(() => Promise.resolve(2));
+    await limiter.execute(() => Promise.resolve(3));
+    const stats = limiter.getStats();
+    assert.equal(stats.requestsInLastMinute, 3);
+});
+test('RateLimiter tracks RPM limits correctly', async () => {
+    const limiter = new RateLimiter(5, null);
+    // Execute 5 requests (at limit)
+    for (let i = 0; i < 5; i++) {
+        await limiter.execute(() => Promise.resolve(i));
+    }
+    const stats = limiter.getStats();
+    assert.equal(stats.requestsInLastMinute, 5);
+    assert.equal(stats.isRpmLimited, true);
+});
+// ============================================================================
+// TPM (Tokens Per Minute) Limiting Tests
+// ============================================================================
+test('RateLimiter tracks token usage', async () => {
+    const limiter = new RateLimiter(null, 10000);
+    // Execute with estimated tokens
+    await limiter.execute(() => Promise.resolve('result'), 0, 100);
+    await limiter.execute(() => Promise.resolve('result'), 0, 200);
+    const stats = limiter.getStats();
+    assert.equal(stats.tokensInLastMinute, 300);
+    assert.equal(stats.isTpmLimited, true);
+});
+test('RateLimiter extracts tokens from response usage field', async () => {
+    const limiter = new RateLimiter(null, 10000);
+    // Execute with response containing usage info
+    await limiter.execute(() => Promise.resolve({
+        usage: { total_tokens: 500 }
+    }));
+    const stats = limiter.getStats();
+    assert.equal(stats.tokensInLastMinute, 500);
+});
+test('RateLimiter uses estimated tokens when no usage in response', async () => {
+    const limiter = new RateLimiter(null, 10000);
+    await limiter.execute(() => Promise.resolve('plain result'), 0, 150);
+    const stats = limiter.getStats();
+    assert.equal(stats.tokensInLastMinute, 150);
+});
+// ============================================================================
+// Rate Limit Error Handling Tests
+// ============================================================================
+test('RateLimiter retries on 429 rate limit error', async () => {
+    const limiter = new RateLimiter(1000, null);
+    let attempts = 0;
+    const result = await limiter.execute(async () => {
+        attempts++;
+        if (attempts === 1) {
+            const error = new Error('Rate limit hit');
+            error.status = 429;
+            throw error;
+        }
+        return 'success';
+    });
+    assert.equal(result, 'success');
+    assert.equal(attempts, 2);
+});
+test('RateLimiter retries on rate limit message in error', async () => {
+    const limiter = new RateLimiter(1000, null);
+    let attempts = 0;
+    const result = await limiter.execute(async () => {
+        attempts++;
+        if (attempts === 1) {
+            throw new Error('too many requests');
+        }
+        return 'success';
+    });
+    assert.equal(result, 'success');
+    assert.equal(attempts, 2);
+});
+test('RateLimiter fails after max retries', async () => {
+    const limiter = new RateLimiter(1000, null);
+    let attempts = 0;
+    await assert.rejects(limiter.execute(async () => {
+        attempts++;
+        const error = new Error('Rate limit exceeded');
+        error.status = 429;
+        throw error;
+    }), /Rate limit exceeded after 4 retries/);
+    // Should have attempted 4 times (initial + 3 retries, or 1 + retryDelays.length)
+    assert.ok(attempts > 1);
+});
+test('RateLimiter does not retry non-rate-limit errors', async () => {
+    const limiter = new RateLimiter(1000, null);
+    let attempts = 0;
+    await assert.rejects(limiter.execute(async () => {
+        attempts++;
+        throw new Error('Some other error');
+    }), { message: 'Some other error' });
+    assert.equal(attempts, 1);
+});
+// ============================================================================
+// Reset Tests
+// ============================================================================
+test('RateLimiter reset clears all state', async () => {
+    const limiter = new RateLimiter(100, 10000);
+    // Add some requests
+    await limiter.execute(() => Promise.resolve(1), 0, 100);
+    await limiter.execute(() => Promise.resolve(2), 0, 200);
+    const beforeReset = limiter.getStats();
+    assert.equal(beforeReset.requestsInLastMinute, 2);
+    assert.equal(beforeReset.tokensInLastMinute, 300);
+    // Reset
+    limiter.reset();
+    const afterReset = limiter.getStats();
+    assert.equal(afterReset.requestsInLastMinute, 0);
+    assert.equal(afterReset.tokensInLastMinute, 0);
+    assert.equal(afterReset.queueLength, 0);
+});
+test('RateLimiter reset allows new requests', async () => {
+    const limiter = new RateLimiter(100, null);
+    await limiter.execute(() => Promise.resolve(1));
+    limiter.reset();
+    const result = await limiter.execute(() => Promise.resolve(42));
+    assert.equal(result, 42);
+    const stats = limiter.getStats();
+    assert.equal(stats.requestsInLastMinute, 1);
+});
+// ============================================================================
+// getStats Tests
+// ============================================================================
+test('RateLimiter getStats returns complete statistics', () => {
+    const limiter = new RateLimiter(100, 50000, 500);
+    const stats = limiter.getStats();
+    assert.equal(stats.rpm, 100);
+    assert.equal(stats.tpm, 50000);
+    assert.equal(stats.maxQueueSize, 500);
+    assert.equal(stats.queueLength, 0);
+    assert.equal(stats.requestsInLastMinute, 0);
+    assert.equal(stats.tokensInLastMinute, 0);
+    assert.equal(stats.isRpmLimited, true);
+    assert.equal(stats.isTpmLimited, true);
+    assert.equal(stats.isLimited, true);
+});
+test('RateLimiter getStats isLimited false when no limits', () => {
+    const limiter = new RateLimiter(null, null);
+    const stats = limiter.getStats();
+    // Will only be false if env vars are not set
+    if (stats.rpm === null && stats.tpm === null) {
+        assert.equal(stats.isLimited, false);
+    }
+});
+// ============================================================================
+// createRateLimiter Factory Tests
+// ============================================================================
+test('createRateLimiter uses default limits for OpenAI', () => {
+    const limiter = createRateLimiter('OpenAI');
+    const stats = limiter.getStats();
+    // OpenAI default is rpm: 50, tpm: null
+    assert.equal(stats.rpm, 50);
+    assert.equal(stats.tpm, null);
+});
+test('createRateLimiter uses default limits for Qwen', () => {
+    const limiter = createRateLimiter('Qwen');
+    const stats = limiter.getStats();
+    // Qwen default is rpm: 10000, tpm: 600000
+    assert.equal(stats.rpm, 10000);
+    assert.equal(stats.tpm, 600000);
+});
+test('createRateLimiter uses null limits for unknown provider', () => {
+    const limiter = createRateLimiter('UnknownProvider');
+    const stats = limiter.getStats();
+    // Unknown provider gets null limits (unless env vars set)
+    if (!process.env.CODEVAULT_RATE_LIMIT_RPM && !process.env.CODEVAULT_RATE_LIMIT_TPM && !process.env.CODEVAULT_RATE_LIMIT) {
+        assert.equal(stats.rpm, null);
+        assert.equal(stats.tpm, null);
+    }
+});
+// ============================================================================
+// Concurrent Processing Tests
+// ============================================================================
+test('RateLimiter processes queued tasks in order', async () => {
+    const limiter = new RateLimiter(100, null);
+    const results = [];
+    // Queue multiple tasks concurrently
+    const tasks = [
+        limiter.execute(async () => { results.push(1); return 1; }),
+        limiter.execute(async () => { results.push(2); return 2; }),
+        limiter.execute(async () => { results.push(3); return 3; }),
+    ];
+    await Promise.all(tasks);
+    assert.deepEqual(results, [1, 2, 3]);
+});
+test('RateLimiter handles concurrent execution with delays', async () => {
+    const limiter = new RateLimiter(100, null);
+    const startTime = Date.now();
+    // Execute tasks with small delays
+    await Promise.all([
+        limiter.execute(async () => { await delay(10); return 1; }),
+        limiter.execute(async () => { await delay(10); return 2; }),
+        limiter.execute(async () => { await delay(10); return 3; }),
+    ]);
+    const elapsed = Date.now() - startTime;
+    // Should take at least 30ms since tasks are sequential
+    assert.ok(elapsed >= 25, `Expected >= 25ms but got ${elapsed}ms`);
+});
+// ============================================================================
+// Edge Cases
+// ============================================================================
+test('RateLimiter handles zero estimated tokens', async () => {
+    const limiter = new RateLimiter(100, 10000);
+    await limiter.execute(() => Promise.resolve('result'), 0, 0);
+    const stats = limiter.getStats();
+    assert.equal(stats.tokensInLastMinute, 0);
+});
+test('RateLimiter handles negative retry count gracefully', async () => {
+    const limiter = new RateLimiter(100, null);
+    // This should work even with negative retry count (treated as 0)
+    const result = await limiter.execute(() => Promise.resolve('ok'), -1);
+    assert.equal(result, 'ok');
+});
+test('RateLimiter extracts statusCode as well as status', async () => {
+    const limiter = new RateLimiter(1000, null);
+    let attempts = 0;
+    const result = await limiter.execute(async () => {
+        attempts++;
+        if (attempts === 1) {
+            const error = new Error('Rate limit');
+            error.statusCode = 429;
+            throw error;
+        }
+        return 'success';
+    });
+    assert.equal(result, 'success');
+    assert.equal(attempts, 2);
+});
+test('RateLimiter handles error with 429 in message', async () => {
+    const limiter = new RateLimiter(1000, null);
+    let attempts = 0;
+    const result = await limiter.execute(async () => {
+        attempts++;
+        if (attempts === 1) {
+            throw new Error('HTTP 429: rate limit exceeded');
+        }
+        return 'success';
+    });
+    assert.equal(result, 'success');
+    assert.equal(attempts, 2);
+});
+test('RateLimiter handles null/undefined error gracefully', async () => {
+    const limiter = new RateLimiter(1000, null);
+    // Throwing null should not crash
+    await assert.rejects(limiter.execute(async () => {
+        throw null;
+    }));
+});
+test('RateLimiter getStats filters old request times', async () => {
+    const limiter = new RateLimiter(100, null);
+    await limiter.execute(() => Promise.resolve(1));
+    // Stats should show 1 request
+    let stats = limiter.getStats();
+    assert.equal(stats.requestsInLastMinute, 1);
+    // After a minute, it should be filtered out
+    // We can't wait a minute in tests, so just verify the filter logic exists
+    // by checking the stats structure
+    assert.ok('requestsInLastMinute' in stats);
+});
+// ============================================================================
+// Token Extraction Edge Cases
+// ============================================================================
+test('RateLimiter handles response without usage field', async () => {
+    const limiter = new RateLimiter(null, 10000);
+    await limiter.execute(() => Promise.resolve({ data: 'no usage' }));
+    const stats = limiter.getStats();
+    assert.equal(stats.tokensInLastMinute, 0);
+});
+test('RateLimiter handles response with malformed usage field', async () => {
+    const limiter = new RateLimiter(null, 10000);
+    await limiter.execute(() => Promise.resolve({
+        usage: 'not an object'
+    }));
+    const stats = limiter.getStats();
+    assert.equal(stats.tokensInLastMinute, 0);
+});
+test('RateLimiter handles response with non-numeric total_tokens', async () => {
+    const limiter = new RateLimiter(null, 10000);
+    await limiter.execute(() => Promise.resolve({
+        usage: { total_tokens: 'invalid' }
+    }));
+    const stats = limiter.getStats();
+    assert.equal(stats.tokensInLastMinute, 0);
+});
+test('RateLimiter prefers estimated tokens over response usage when available', async () => {
+    const limiter = new RateLimiter(null, 10000);
+    // When estimatedTokens is provided and non-zero, it should be used
+    await limiter.execute(() => Promise.resolve({
+        usage: { total_tokens: 100 }
+    }), 0, 0); // estimatedTokens = 0, so it falls back to response
+    const stats = limiter.getStats();
+    assert.equal(stats.tokensInLastMinute, 100);
+});
 //# sourceMappingURL=rate-limiter.test.js.map