@mikugg/guidance 0.17.2 → 0.19.0

package/dist/index.js

@@ -15,13 +15,23 @@ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (
 }) : function(o, v) {
     o["default"] = v;
 });
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Tokenizer = exports.TokenGenerator = exports.Template = exports.Agents = void 0;
 const Agents = __importStar(require("./lib/Agent"));

package/dist/lib/__test__/template.test.js

@@ -12,8 +12,9 @@ var __await = (this && this.__await) || function (v) { return this instanceof __
 var __asyncGenerator = (this && this.__asyncGenerator) || function (thisArg, _arguments, generator) {
     if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
     var g = generator.apply(thisArg, _arguments || []), i, q = [];
-    return i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i;
-    function verb(n) { if (g[n]) i[n] = function (v) { return new Promise(function (a, b) { q.push([n, v, a, b]) > 1 || resume(n, v); }); }; }
+    return i = Object.create((typeof AsyncIterator === "function" ? AsyncIterator : Object).prototype), verb("next"), verb("throw"), verb("return", awaitReturn), i[Symbol.asyncIterator] = function () { return this; }, i;
+    function awaitReturn(f) { return function (v) { return Promise.resolve(v).then(f, reject); }; }
+    function verb(n, f) { if (g[n]) { i[n] = function (v) { return new Promise(function (a, b) { q.push([n, v, a, b]) > 1 || resume(n, v); }); }; if (f) i[n] = f(i[n]); } }
     function resume(n, v) { try { step(g[n](v)); } catch (e) { settle(q[0][3], e); } }
     function step(r) { r.value instanceof __await ? Promise.resolve(r.value.v).then(fulfill, reject) : settle(q[0][2], r); }
     function fulfill(value) { resume("next", value); }

package/dist/lib/template.d.ts

@@ -1,7 +1,7 @@
 import { AbstractTokenizer } from './tokenizer';
 import { AbstractTokenGenerator } from './token-generator';
 export declare enum TEMPLATE_METHODS {
-    SEL = "SEL",
+    SEL = "SEL",// select for a group of options
     GEN = "GEN"
 }
 export declare class TemplateProcessor<TRequestOptions = undefined> {

package/dist/lib/template.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"template.d.ts","sourceRoot":"","sources":["../../src/lib/template.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAChD,OAAO,EAAE,sBAAsB,EAAE,MAAM,mBAAmB,CAAC;AAG3D,oBAAY,gBAAgB;IAC1B,GAAG,QAAQ;IACX,GAAG,QAAQ;CACZ;AAED,qBAAa,iBAAiB,CAAC,eAAe,GAAG,SAAS;IACxD,OAAO,CAAC,SAAS,CAAoB;IACrC,OAAO,CAAC,SAAS,CAA0C;IAC3D,OAAO,CAAC,MAAM,CAAU;gBAEZ,SAAS,EAAE,iBAAiB,EAAE,SAAS,EAAE,sBAAsB,CAAC,eAAe,CAAC;IAMrF,YAAY,CAAC,SAAS,EAAE,iBAAiB;IAIzC,YAAY,CAAC,SAAS,EAAE,sBAAsB,CAAC,eAAe,CAAC;IAIzD,eAAe,CAC1B,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC,EACzC,UAAU,CAAC,EAAE,eAAe,GAC3B,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IASjB,qBAAqB,CACjC,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC,EACzC,UAAU,CAAC,EAAE,eAAe,GAC3B,cAAc,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,IAAI,CAAC;IAuI5C,OAAO,CAAC,cAAc;CAQvB"}
+{"version":3,"file":"template.d.ts","sourceRoot":"","sources":["../../src/lib/template.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAChD,OAAO,EAAE,sBAAsB,EAAE,MAAM,mBAAmB,CAAC;AAG3D,oBAAY,gBAAgB;IAC1B,GAAG,QAAQ,CAAE,gCAAgC;IAC7C,GAAG,QAAQ;CACZ;AAED,qBAAa,iBAAiB,CAAC,eAAe,GAAG,SAAS;IACxD,OAAO,CAAC,SAAS,CAAoB;IACrC,OAAO,CAAC,SAAS,CAA0C;IAC3D,OAAO,CAAC,MAAM,CAAU;gBAEZ,SAAS,EAAE,iBAAiB,EAAE,SAAS,EAAE,sBAAsB,CAAC,eAAe,CAAC;IAMrF,YAAY,CAAC,SAAS,EAAE,iBAAiB;IAIzC,YAAY,CAAC,SAAS,EAAE,sBAAsB,CAAC,eAAe,CAAC;IAIzD,eAAe,CAC1B,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC,EACzC,UAAU,CAAC,EAAE,eAAe,GAC3B,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IASjB,qBAAqB,CACjC,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC,EACzC,UAAU,CAAC,EAAE,eAAe,GAC3B,cAAc,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,IAAI,CAAC;IAuI5C,OAAO,CAAC,cAAc;CAQvB"}

package/dist/lib/template.js

@@ -19,8 +19,9 @@ var __await = (this && this.__await) || function (v) { return this instanceof __
 var __asyncGenerator = (this && this.__asyncGenerator) || function (thisArg, _arguments, generator) {
     if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
     var g = generator.apply(thisArg, _arguments || []), i, q = [];
-    return i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i;
-    function verb(n) { if (g[n]) i[n] = function (v) { return new Promise(function (a, b) { q.push([n, v, a, b]) > 1 || resume(n, v); }); }; }
+    return i = Object.create((typeof AsyncIterator === "function" ? AsyncIterator : Object).prototype), verb("next"), verb("throw"), verb("return", awaitReturn), i[Symbol.asyncIterator] = function () { return this; }, i;
+    function awaitReturn(f) { return function (v) { return Promise.resolve(v).then(f, reject); }; }
+    function verb(n, f) { if (g[n]) { i[n] = function (v) { return new Promise(function (a, b) { q.push([n, v, a, b]) > 1 || resume(n, v); }); }; if (f) i[n] = f(i[n]); } }
     function resume(n, v) { try { step(g[n](v)); } catch (e) { settle(q[0][3], e); } }
     function step(r) { r.value instanceof __await ? Promise.resolve(r.value.v).then(fulfill, reject) : settle(q[0][2], r); }
     function fulfill(value) { resume("next", value); }
@@ -52,8 +53,8 @@ class TemplateProcessor {
         this.generator = generator;
     }
     processTemplate(template, variables, reqOptions) {
-        var _a, e_1, _b, _c;
         return __awaiter(this, void 0, void 0, function* () {
+            var _a, e_1, _b, _c;
             let finalResult = new Map();
             try {
                 for (var _d = true, _e = __asyncValues(this.processTemplateStream(template, variables, reqOptions)), _f; _f = yield _e.next(), _a = _f.done, !_a; _d = true) {

package/dist/lib/templateParser.js

@@ -1,5 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = templateParser;
 function templateParser(template) {
     const patternRegex = /{{(GEN|SEL)\s+(\w+)\s+([^}]+)}}/;
     const match = template.match(patternRegex);
@@ -41,4 +42,3 @@ function templateParser(template) {
         params,
     };
 }
-exports.default = templateParser;

package/dist/lib/token-generator.d.ts

@@ -1,5 +1,6 @@
-import OpenAI, { ClientOptions } from 'openai';
+import { ClientOptions } from 'openai';
 import { CompletionCreateParams } from 'openai/resources/completions.mjs';
+import { RequestOptions } from 'openai/core';
 export declare abstract class AbstractTokenGenerator<TRequestOptions = undefined> {
     abstract generateTokenLogProgs(prompt: string, logit_bias: Record<string, number>, reqOptions?: TRequestOptions): Promise<Record<string, number>>;
     abstract generateString(prompt: string, options: Record<string, number | string | string[]>, reqOptions?: TRequestOptions): AsyncGenerator<string>;
@@ -8,7 +9,7 @@ export declare abstract class AbstractTokenGenerator<TRequestOptions = undefined
  * OpenAI Token Generator
  *
  */
-export declare class OpenAITokenGenerator extends AbstractTokenGenerator<OpenAI.RequestOptions<Record<string, unknown>> | undefined> {
+export declare class OpenAITokenGenerator extends AbstractTokenGenerator<RequestOptions<Record<string, unknown>> | undefined> {
     private openai;
     private model;
     private defaultCompletionParams?;
@@ -17,7 +18,7 @@ export declare class OpenAITokenGenerator extends AbstractTokenGenerator<OpenAI.
         model: string;
         baseURL?: string;
     }, options?: ClientOptions, defaultCompletionParams?: CompletionCreateParams);
-    generateTokenLogProgs(prompt: string, logit_bias: Record<string, number>, reqOptions?: OpenAI.RequestOptions<Record<string, unknown>> | undefined): Promise<Record<string, number>>;
-    generateString(prompt: string, options: Record<string, string | number | string[]>, reqOptions?: OpenAI.RequestOptions<Record<string, unknown>> | undefined): AsyncGenerator<string>;
+    generateTokenLogProgs(prompt: string, logit_bias: Record<string, number>, reqOptions?: RequestOptions<Record<string, unknown>> | undefined): Promise<Record<string, number>>;
+    generateString(prompt: string, options: Record<string, string | number | string[]>, reqOptions?: RequestOptions<Record<string, unknown>> | undefined): AsyncGenerator<string>;
 }
 //# sourceMappingURL=token-generator.d.ts.map

package/dist/lib/token-generator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"token-generator.d.ts","sourceRoot":"","sources":["../../src/lib/token-generator.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,EAAE,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AAC/C,OAAO,EAAE,sBAAsB,EAAE,MAAM,kCAAkC,CAAC;AAE1E,8BAAsB,sBAAsB,CAAC,eAAe,GAAG,SAAS;IACtE,QAAQ,CAAC,qBAAqB,CAC5B,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAClC,UAAU,CAAC,EAAE,eAAe,GAC3B,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAClC,QAAQ,CAAC,cAAc,CACrB,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,EAAE,CAAC,EACnD,UAAU,CAAC,EAAE,eAAe,GAC3B,cAAc,CAAC,MAAM,CAAC;CAC1B;AACD;;;GAGG;AACH,qBAAa,oBAAqB,SAAQ,sBAAsB,CAC9D,MAAM,CAAC,cAAc,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,GAAG,SAAS,CAC3D;IACC,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,uBAAuB,CAAC,CAAyB;gBAGvD,MAAM,EAAE;QACN,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,CAAC,EAAE,MAAM,CAAC;KAClB,EACD,OAAO,CAAC,EAAE,aAAa,EACvB,uBAAuB,CAAC,EAAE,sBAAsB;IAYnC,qBAAqB,CAClC,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAClC,UAAU,CAAC,EAAE,MAAM,CAAC,cAAc,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,GAAG,SAAS,GACtE,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAiBlB,cAAc,CAC5B,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,EAAE,CAAC,EACnD,UAAU,CAAC,EAAE,MAAM,CAAC,cAAc,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,GAAG,SAAS,GACtE,cAAc,CAAC,MAAM,CAAC;CAiB1B"}
+{"version":3,"file":"token-generator.d.ts","sourceRoot":"","sources":["../../src/lib/token-generator.ts"],"names":[],"mappings":"AAAA,OAAe,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AAC/C,OAAO,EAAE,sBAAsB,EAAE,MAAM,kCAAkC,CAAC;AAC1E,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAE7C,8BAAsB,sBAAsB,CAAC,eAAe,GAAG,SAAS;IACtE,QAAQ,CAAC,qBAAqB,CAC5B,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAClC,UAAU,CAAC,EAAE,eAAe,GAC3B,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAClC,QAAQ,CAAC,cAAc,CACrB,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,EAAE,CAAC,EACnD,UAAU,CAAC,EAAE,eAAe,GAC3B,cAAc,CAAC,MAAM,CAAC;CAC1B;AACD;;;GAGG;AACH,qBAAa,oBAAqB,SAAQ,sBAAsB,CAAC,cAAc,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,GAAG,SAAS,CAAC;IACnH,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,uBAAuB,CAAC,CAAyB;gBAGvD,MAAM,EAAE;QACN,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,CAAC,EAAE,MAAM,CAAC;KAClB,EACD,OAAO,CAAC,EAAE,aAAa,EACvB,uBAAuB,CAAC,EAAE,sBAAsB;IAYnC,qBAAqB,CAClC,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAClC,UAAU,CAAC,EAAE,cAAc,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,GAAG,SAAS,GAC/D,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAiBlB,cAAc,CAC5B,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,EAAE,CAAC,EACnD,UAAU,CAAC,EAAE,cAAc,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,GAAG,SAAS,GAC/D,cAAc,CAAC,MAAM,CAAC;CAiB1B"}

package/dist/lib/token-generator.js

@@ -19,8 +19,9 @@ var __asyncValues = (this && this.__asyncValues) || function (o) {
 var __asyncGenerator = (this && this.__asyncGenerator) || function (thisArg, _arguments, generator) {
     if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
     var g = generator.apply(thisArg, _arguments || []), i, q = [];
-    return i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i;
-    function verb(n) { if (g[n]) i[n] = function (v) { return new Promise(function (a, b) { q.push([n, v, a, b]) > 1 || resume(n, v); }); }; }
+    return i = Object.create((typeof AsyncIterator === "function" ? AsyncIterator : Object).prototype), verb("next"), verb("throw"), verb("return", awaitReturn), i[Symbol.asyncIterator] = function () { return this; }, i;
+    function awaitReturn(f) { return function (v) { return Promise.resolve(v).then(f, reject); }; }
+    function verb(n, f) { if (g[n]) { i[n] = function (v) { return new Promise(function (a, b) { q.push([n, v, a, b]) > 1 || resume(n, v); }); }; if (f) i[n] = f(i[n]); } }
     function resume(n, v) { try { step(g[n](v)); } catch (e) { settle(q[0][3], e); } }
     function step(r) { r.value instanceof __await ? Promise.resolve(r.value.v).then(fulfill, reject) : settle(q[0][2], r); }
     function fulfill(value) { resume("next", value); }
@@ -48,8 +49,8 @@ class OpenAITokenGenerator extends AbstractTokenGenerator {
         this.defaultCompletionParams = defaultCompletionParams;
     }
     generateTokenLogProgs(prompt, logit_bias, reqOptions) {
-        var _a;
         return __awaiter(this, void 0, void 0, function* () {
+            var _a;
             const result = yield this.openai.completions.create({
                 stream: false,
                 model: this.model,
@@ -64,24 +65,24 @@ class OpenAITokenGenerator extends AbstractTokenGenerator {
         });
     }
     generateString(prompt, options, reqOptions) {
-        var _a;
         return __asyncGenerator(this, arguments, function* generateString_1() {
-            var _b, e_1, _c, _d;
+            var _a, e_1, _b, _c;
+            var _d;
             const stream = yield __await(this.openai.completions.create(Object.assign(Object.assign(Object.assign({}, this.defaultCompletionParams), options), { model: this.model, prompt, stream: true }), reqOptions));
             let result = '';
             try {
-                for (var _e = true, stream_1 = __asyncValues(stream), stream_1_1; stream_1_1 = yield __await(stream_1.next()), _b = stream_1_1.done, !_b; _e = true) {
-                    _d = stream_1_1.value;
+                for (var _e = true, stream_1 = __asyncValues(stream), stream_1_1; stream_1_1 = yield __await(stream_1.next()), _a = stream_1_1.done, !_a; _e = true) {
+                    _c = stream_1_1.value;
                     _e = false;
-                    const chunk = _d;
-                    result += (_a = chunk.choices[0]) === null || _a === void 0 ? void 0 : _a.text;
+                    const chunk = _c;
+                    result += (_d = chunk.choices[0]) === null || _d === void 0 ? void 0 : _d.text;
                     yield yield __await(result);
                 }
             }
             catch (e_1_1) { e_1 = { error: e_1_1 }; }
             finally {
                 try {
-                    if (!_e && !_b && (_c = stream_1.return)) yield __await(_c.call(stream_1));
+                    if (!_e && !_a && (_b = stream_1.return)) yield __await(_b.call(stream_1));
                 }
                 finally { if (e_1) throw e_1.error; }
             }

package/dist/lib/tokenizer.d.ts

@@ -19,4 +19,13 @@ export declare class MistralTokenizer extends AbstractTokenizer {
     decodeString(arr: number[], add_bos_token?: boolean, add_preceding_space?: boolean): string;
     getEOS(): string;
 }
+export declare class LenMLTokenizer extends AbstractTokenizer {
+    private tokenizer;
+    private tokenizerPromise;
+    constructor(baseUrl: string);
+    ready(): Promise<void>;
+    encodeString(str: string, add_special_tokens?: boolean): number[];
+    decodeString(arr: number[], skip_special_tokens?: boolean, clean_up_tokenization_spaces?: boolean): string;
+    getEOS(): string;
+}
 //# sourceMappingURL=tokenizer.d.ts.map

package/dist/lib/tokenizer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"tokenizer.d.ts","sourceRoot":"","sources":["../../src/lib/tokenizer.ts"],"names":[],"mappings":"AAIA,8BAAsB,iBAAiB;IAC9B,IAAI,EAAE,MAAM,CAAc;IACjC,QAAQ,CAAC,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE;IAC5C,QAAQ,CAAC,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM;IAC5C,QAAQ,CAAC,MAAM,IAAI,MAAM;CAC1B;AAED,qBAAa,cAAe,SAAQ,iBAAiB;IAC1C,YAAY,CACnB,GAAG,EAAE,MAAM,EACX,aAAa,CAAC,EAAE,OAAO,EACvB,mBAAmB,CAAC,EAAE,OAAO,EAC7B,eAAe,CAAC,EAAE,OAAO,GACxB,MAAM,EAAE;IAWF,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,aAAa,CAAC,EAAE,OAAO,EAAE,mBAAmB,CAAC,EAAE,OAAO,GAAG,MAAM;IAQ3F,MAAM,IAAI,MAAM;CAG1B;AAED,qBAAa,YAAa,SAAQ,iBAAiB;IACxC,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE;IAMnC,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM;IAInC,MAAM,IAAI,MAAM;CAG1B;AAED,qBAAa,gBAAiB,SAAQ,iBAAiB;IAC5C,YAAY,CACnB,GAAG,EAAE,MAAM,EACX,aAAa,CAAC,EAAE,OAAO,EACvB,mBAAmB,CAAC,EAAE,OAAO,EAC7B,eAAe,CAAC,EAAE,OAAO,GACxB,MAAM,EAAE;IAWF,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,aAAa,CAAC,EAAE,OAAO,EAAE,mBAAmB,CAAC,EAAE,OAAO,GAAG,MAAM;IAQ3F,MAAM,IAAI,MAAM;CAG1B"}
+{"version":3,"file":"tokenizer.d.ts","sourceRoot":"","sources":["../../src/lib/tokenizer.ts"],"names":[],"mappings":"AAIA,8BAAsB,iBAAiB;IAC9B,IAAI,EAAE,MAAM,CAAc;IACjC,QAAQ,CAAC,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE;IAC5C,QAAQ,CAAC,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM;IAC5C,QAAQ,CAAC,MAAM,IAAI,MAAM;CAC1B;AAED,qBAAa,cAAe,SAAQ,iBAAiB;IAC1C,YAAY,CACnB,GAAG,EAAE,MAAM,EACX,aAAa,CAAC,EAAE,OAAO,EACvB,mBAAmB,CAAC,EAAE,OAAO,EAC7B,eAAe,CAAC,EAAE,OAAO,GACxB,MAAM,EAAE;IAWF,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,aAAa,CAAC,EAAE,OAAO,EAAE,mBAAmB,CAAC,EAAE,OAAO,GAAG,MAAM;IAQ3F,MAAM,IAAI,MAAM;CAG1B;AAED,qBAAa,YAAa,SAAQ,iBAAiB;IACxC,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE;IAMnC,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM;IAInC,MAAM,IAAI,MAAM;CAG1B;AAED,qBAAa,gBAAiB,SAAQ,iBAAiB;IAC5C,YAAY,CACnB,GAAG,EAAE,MAAM,EACX,aAAa,CAAC,EAAE,OAAO,EACvB,mBAAmB,CAAC,EAAE,OAAO,EAC7B,eAAe,CAAC,EAAE,OAAO,GACxB,MAAM,EAAE;IAWF,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,aAAa,CAAC,EAAE,OAAO,EAAE,mBAAmB,CAAC,EAAE,OAAO,GAAG,MAAM;IAQ3F,MAAM,IAAI,MAAM;CAG1B;AAED,qBAAa,cAAe,SAAQ,iBAAiB;IACnD,OAAO,CAAC,SAAS,CAWD;IAEhB,OAAO,CAAC,gBAAgB,CAAgB;gBAE5B,OAAO,EAAE,MAAM;IAWrB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAInB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,kBAAkB,GAAE,OAAc,GAAG,MAAM,EAAE;IAQvE,YAAY,CACnB,GAAG,EAAE,MAAM,EAAE,EACb,mBAAmB,GAAE,OAAc,EACnC,4BAA4B,GAAE,OAAc,GAC3C,MAAM;IASA,MAAM,IAAI,MAAM;CAG1B"}

package/dist/lib/tokenizer.js

@@ -1,12 +1,22 @@
 "use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.MistralTokenizer = exports.GTPTokenizer = exports.LLaMATokenizer = exports.AbstractTokenizer = void 0;
+exports.LenMLTokenizer = exports.MistralTokenizer = exports.GTPTokenizer = exports.LLaMATokenizer = exports.AbstractTokenizer = void 0;
 const _llama_tokenizer_1 = __importDefault(require("./_llama-tokenizer"));
 const gpt_tokenizer_1 = require("gpt-tokenizer");
 const _mistral_tokenizer_1 = __importDefault(require("./_mistral-tokenizer"));
+const tokenizers_1 = require("@lenml/tokenizers");
 class AbstractTokenizer {
     constructor() {
         this.name = 'abstract';
@@ -73,3 +83,39 @@ class MistralTokenizer extends AbstractTokenizer {
     }
 }
 exports.MistralTokenizer = MistralTokenizer;
+class LenMLTokenizer extends AbstractTokenizer {
+    constructor(baseUrl) {
+        super();
+        this.tokenizer = null;
+        const sourceUrls = {
+            tokenizerJSON: `${baseUrl}/tokenizer.json?download=true`,
+            tokenizerConfig: `${baseUrl}/tokenizer_config.json?download=true`,
+        };
+        this.tokenizerPromise = tokenizers_1.TokenizerLoader.fromPreTrainedUrls(sourceUrls).then((tokenizer) => {
+            this.tokenizer = tokenizer;
+        });
+    }
+    ready() {
+        return __awaiter(this, void 0, void 0, function* () {
+            return this.tokenizerPromise;
+        });
+    }
+    encodeString(str, add_special_tokens = true) {
+        var _a;
+        return (((_a = this.tokenizer) === null || _a === void 0 ? void 0 : _a.encode(str, {
+            add_special_tokens,
+        })) || []);
+    }
+    decodeString(arr, skip_special_tokens = true, clean_up_tokenization_spaces = true) {
+        var _a;
+        return (((_a = this.tokenizer) === null || _a === void 0 ? void 0 : _a.decode(arr, {
+            skip_special_tokens,
+            clean_up_tokenization_spaces,
+        })) || '');
+    }
+    getEOS() {
+        var _a;
+        return ((_a = this.tokenizer) === null || _a === void 0 ? void 0 : _a.eos_token) || '';
+    }
+}
+exports.LenMLTokenizer = LenMLTokenizer;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mikugg/guidance",
-  "version": "0.17.2",
+  "version": "0.19.0",
   "description": "",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -27,8 +27,9 @@
     "registry": "https://registry.npmjs.org/"
   },
   "dependencies": {
+    "@lenml/tokenizers": "^3.4.0",
     "gpt-tokenizer": "^2.1.2",
-    "openai": "^4.19.0"
+    "openai": "4.19.0"
   },
   "devDependencies": {
     "@types/jest": "^29.5.8",
@@ -38,7 +39,7 @@
     "jest": "^29.7.0",
     "ts-jest": "^29.1.1",
     "ts-node": "^10.9.1",
-    "typescript": "^5.2.2"
+    "typescript": "^5.7.3"
   },
-  "gitHead": "a25fb27825b1989604a4f0d9d4a51d2b11baac07"
+  "gitHead": "09c9f8f83a5fabf161d64ee359d47745911d26a4"
 }