@mikugg/guidance 0.13.0 → 0.14.0

package/dist/lib/__test__/_trie.test.js

@@ -4,39 +4,39 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 const _trie_1 = __importDefault(require("../_trie")); // replace './Trie' with the actual path of your Trie class
-describe('Trie', () => {
+describe("Trie", () => {
     let trie;
     beforeEach(() => {
         trie = new _trie_1.default();
     });
-    test('addPrefix should add numbers to the trie', () => {
+    test("addPrefix should add numbers to the trie", () => {
         var _a, _b, _c;
         trie.addPrefix([1, 2, 3]);
         expect(trie.root.children.has(1)).toBeTruthy();
         expect((_a = trie.root.children.get(1)) === null || _a === void 0 ? void 0 : _a.children.has(2)).toBeTruthy();
         expect((_c = (_b = trie.root.children.get(1)) === null || _b === void 0 ? void 0 : _b.children.get(2)) === null || _c === void 0 ? void 0 : _c.children.has(3)).toBeTruthy();
     });
-    test('getNextChildren should return correct next children', () => {
+    test("getNextChildren should return correct next children", () => {
         trie.addPrefix([1, 2, 3]);
         trie.addPrefix([1, 2, 4]);
         expect(trie.getNextChildren([1, 2])).toEqual([3, 4]);
     });
-    test('getNextPrefix should return correct next prefix', () => {
+    test("getNextPrefix should return correct next prefix", () => {
         trie.addPrefix([1, 2, 3]);
         trie.addPrefix([1, 2, 3, 4]);
         expect(trie.getNextPrefix([1, 2])).toEqual([1, 2, 3]);
     });
-    test('getNextPrefix should return correct next prefix', () => {
+    test("getNextPrefix should return correct next prefix", () => {
         trie.addPrefix([1, 2, 3, 4, 5]);
         trie.addPrefix([1, 2, 3, 4, 7]);
         expect(trie.getNextPrefix([1, 2])).toEqual([1, 2, 3, 4]);
     });
-    test('getWord should return correct word until the end of the prefix', () => {
+    test("getWord should return correct word until the end of the prefix", () => {
         trie.addPrefix([1, 2, 3]);
         trie.addPrefix([1, 2, 3, 4]);
         expect(trie.getWord([1, 2])).toEqual([1, 2, 3]);
     });
-    test('getWord should return empty array if prefix not found', () => {
+    test("getWord should return empty array if prefix not found", () => {
         trie.addPrefix([1, 2, 3]);
         expect(trie.getWord([4])).toEqual([]);
     });

package/dist/lib/__test__/template.test.js

@@ -25,20 +25,28 @@ const template_1 = require("../template");
 const tokenizer_1 = require("../tokenizer"); // import paths as required
 const token_generator_1 = require("../token-generator");
 class MockTokenGenerator extends token_generator_1.AbstractTokenGenerator {
+    generateTokenLogProgs(
     // eslint-disable-next-line @typescript-eslint/no-unused-vars
-    generateToken(prompt, logit_bias) {
+    prompt, 
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    logit_bias) {
         return __awaiter(this, void 0, void 0, function* () {
-            return '<TOK>';
+            return {
+                "<TOK>": 100,
+            };
         });
     }
+    generateString(
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    prompt, 
     // eslint-disable-next-line @typescript-eslint/no-unused-vars
-    generateString(prompt, options) {
+    options) {
         return __asyncGenerator(this, arguments, function* generateString_1() {
-            return yield __await('generated');
+            return yield __await("generated");
         });
     }
 }
-describe('TemplateProcessor', () => {
+describe("TemplateProcessor", () => {
     let tokenizer;
     let generator;
     let templateProcessor;
@@ -47,67 +55,73 @@ describe('TemplateProcessor', () => {
         generator = new MockTokenGenerator();
         templateProcessor = new template_1.TemplateProcessor(tokenizer, generator);
     });
-    describe('processTemplate', () => {
-        it('should process template with GEN method correctly', () => __awaiter(void 0, void 0, void 0, function* () {
-            const spyGenerateString = jest.spyOn(generator, 'generateString');
-            const result = yield templateProcessor.processTemplate('Hello, {{user}}. The weather is {{GEN weather}}', new Map([['user', 'Hina']]));
-            expect(spyGenerateString).toHaveBeenCalledWith('Hello, Hina. The weather is ', {});
-            expect(result.get('weather')).toEqual('generated');
+    describe("processTemplate", () => {
+        it("should process template with GEN method correctly", () => __awaiter(void 0, void 0, void 0, function* () {
+            const spyGenerateString = jest.spyOn(generator, "generateString");
+            const result = yield templateProcessor.processTemplate("Hello, {{user}}. The weather is {{GEN weather}}", new Map([["user", "Hina"]]));
+            expect(spyGenerateString).toHaveBeenCalledWith("Hello, Hina. The weather is ", {});
+            expect(result.get("weather")).toEqual("generated");
         }));
-        it('should process template with GEN method and stop correctly', () => __awaiter(void 0, void 0, void 0, function* () {
-            const spyGenerateString = jest.spyOn(generator, 'generateString');
-            const result = yield templateProcessor.processTemplate('Hello, {{user}}. The weather is {{GEN weather stop=.}}', new Map([['user', 'Hina']]));
-            expect(spyGenerateString).toHaveBeenCalledWith('Hello, Hina. The weather is ', { stop: '.' });
-            expect(result.get('weather')).toEqual('generated');
+        it("should process template with GEN method and stop correctly", () => __awaiter(void 0, void 0, void 0, function* () {
+            const spyGenerateString = jest.spyOn(generator, "generateString");
+            const result = yield templateProcessor.processTemplate("Hello, {{user}}. The weather is {{GEN weather stop=.}}", new Map([["user", "Hina"]]));
+            expect(spyGenerateString).toHaveBeenCalledWith("Hello, Hina. The weather is ", { stop: "." });
+            expect(result.get("weather")).toEqual("generated");
         }));
-        it('should process template with GEN method and temperature and repetition_penalty and stop correctly', () => __awaiter(void 0, void 0, void 0, function* () {
-            const spyGenerateString = jest.spyOn(generator, 'generateString');
-            const result = yield templateProcessor.processTemplate('Hello, {{user}}. The weather is {{GEN weather temperature=0.5 repetition_penalty=1 stop=.}}', new Map([['user', 'Hina']]));
-            expect(spyGenerateString).toHaveBeenCalledWith('Hello, Hina. The weather is ', { temperature: '0.5', repetition_penalty: '1', stop: '.' });
-            expect(result.get('weather')).toEqual('generated');
+        it("should process template with GEN method and temperature and repetition_penalty and stop correctly", () => __awaiter(void 0, void 0, void 0, function* () {
+            const spyGenerateString = jest.spyOn(generator, "generateString");
+            const result = yield templateProcessor.processTemplate("Hello, {{user}}. The weather is {{GEN weather temperature=0.5 repetition_penalty=1 stop=.}}", new Map([["user", "Hina"]]));
+            expect(spyGenerateString).toHaveBeenCalledWith("Hello, Hina. The weather is ", { temperature: "0.5", repetition_penalty: "1", stop: "." });
+            expect(result.get("weather")).toEqual("generated");
         }));
-        it('should process template with SEL method correctly', () => __awaiter(void 0, void 0, void 0, function* () {
+        it("should process template with SEL method correctly", () => __awaiter(void 0, void 0, void 0, function* () {
             // 1153 = " ra"
-            const spyGenerateToken = jest.spyOn(generator, 'generateToken').mockReturnValue(new Promise((resolve) => resolve(" ra")));
-            const result = yield templateProcessor.processTemplate('Hello, {{user}}. The weather is{{SEL weather options=weatherOptions}}', new Map([
-                ['user', 'Hina'],
-                ['weatherOptions', [' sunny', ' rainy', ' cloudy']]
+            const spyGenerateToken = jest
+                .spyOn(generator, "generateTokenLogProgs")
+                .mockReturnValue(new Promise((resolve) => resolve({ " ra": 0 })));
+            const result = yield templateProcessor.processTemplate("Hello, {{user}}. The weather is{{SEL weather options=weatherOptions}}", new Map([
+                ["user", "Hina"],
+                ["weatherOptions", [" sunny", " rainy", " cloudy"]],
             ]));
-            expect(spyGenerateToken).toHaveBeenCalledWith('Hello, Hina. The weather is', {
-                '6575': 100,
-                '1153': 100,
-                '9570': 100
+            expect(spyGenerateToken).toHaveBeenCalledWith("Hello, Hina. The weather is", {
+                "6575": 100,
+                "1153": 100,
+                "9570": 100,
             });
-            expect(result.get('weather')).toEqual(' rainy');
+            expect(result.get("weather")).toEqual(" rainy");
         }));
-        it('should process template with SEL method in a JSON correctly', () => __awaiter(void 0, void 0, void 0, function* () {
+        it("should process template with SEL method in a JSON correctly", () => __awaiter(void 0, void 0, void 0, function* () {
             const mockGenerateString = function () {
                 return __asyncGenerator(this, arguments, function* () {
-                    yield yield __await('wizard');
+                    yield yield __await("wizard");
                 });
             };
             // 29879 = "s"
-            const spyGenerateToken = jest.spyOn(generator, 'generateToken').mockReturnValue(new Promise((resolve) => resolve("s")));
-            const spyGenerateString = jest.spyOn(generator, 'generateString').mockImplementation(mockGenerateString);
+            const spyGenerateToken = jest
+                .spyOn(generator, "generateTokenLogProgs")
+                .mockReturnValue(new Promise((resolve) => resolve({ s: 0 })));
+            const spyGenerateString = jest
+                .spyOn(generator, "generateString")
+                .mockImplementation(mockGenerateString);
             const result = yield templateProcessor.processTemplate(`RPG Game Character specification
         {
           "name": "{{name}}",
           "job": "{{GEN job stop=",}}",
           "weapon": "{{SEL weapon options=valid_weapons}}",
         }`, new Map([
-                ['name', 'Rudeus'],
-                ['valid_weapons', ["axe", "mace", "sword", "bow", "crossbow"]],
+                ["name", "Rudeus"],
+                ["valid_weapons", ["axe", "mace", "sword", "bow", "crossbow"]],
             ]));
             expect(spyGenerateString).toHaveBeenCalledWith('RPG Game Character specification\n        {\n          "name": "Rudeus",\n          "job": "', { stop: '",' });
             expect(spyGenerateToken).toHaveBeenCalledWith('RPG Game Character specification\n        {\n          "name": "Rudeus",\n          "job": "wizard",\n          "weapon": "', {
-                '29879': 100,
-                '1165': 100,
-                '655': 100,
-                '17729': 100,
-                '19128': 100,
+                "29879": 100,
+                "1165": 100,
+                "655": 100,
+                "17729": 100,
+                "19128": 100,
             });
-            expect(result.get('weapon')).toEqual('sword');
-            expect(result.get('job')).toEqual('wizard');
+            expect(result.get("weapon")).toEqual("sword");
+            expect(result.get("job")).toEqual("wizard");
         }));
     });
 });

package/dist/lib/_mistral-tokenizer.d.ts

@@ -0,0 +1,19 @@
+/**
+ * MIT LICENSE
+ *
+ * Copyright 2023 belladore.ai
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+export interface MistralTokenizer {
+    encode: (text: string, add_bos_token?: boolean, add_preceding_space?: boolean, log_performance?: boolean) => number[];
+    decode: (tokenIds: number[], add_bos_token?: boolean, add_preceding_space?: boolean) => string;
+}
+declare const mistralTokenizer: MistralTokenizer;
+export default mistralTokenizer;
+//# sourceMappingURL=_mistral-tokenizer.d.ts.map

package/dist/lib/_mistral-tokenizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"_mistral-tokenizer.d.ts","sourceRoot":"","sources":["../../src/lib/_mistral-tokenizer.ts"],"names":[],"mappings":"AAEA;;;;;;;;;;;GAWG;AAEH,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,CACN,IAAI,EAAE,MAAM,EACZ,aAAa,CAAC,EAAE,OAAO,EACvB,mBAAmB,CAAC,EAAE,OAAO,EAC7B,eAAe,CAAC,EAAE,OAAO,KACtB,MAAM,EAAE,CAAC;IACd,MAAM,EAAE,CACN,QAAQ,EAAE,MAAM,EAAE,EAClB,aAAa,CAAC,EAAE,OAAO,EACvB,mBAAmB,CAAC,EAAE,OAAO,KAC1B,MAAM,CAAC;CACb;AAED,QAAA,MAAM,gBAAgB,EAAE,gBAAqB,CAAC;AAwgB9C,eAAe,gBAAgB,CAAC"}