gpt-tokenizer 1.0.2 → 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cjs/encoder.d.ts +1 -0
- package/cjs/encoder.js +2 -1
- package/cjs/encoder.js.map +1 -1
- package/esm/encoder.d.ts +1 -0
- package/esm/encoder.js +1 -1
- package/esm/encoder.js.map +1 -1
- package/package.json +1 -1
- package/src/encoder.ts +1 -1
package/cjs/encoder.d.ts
CHANGED
package/cjs/encoder.js
CHANGED
|
@@ -5,7 +5,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
5
5
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
6
6
|
};
|
|
7
7
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
8
|
-
exports.decode = exports.decodeAsyncGenerator = exports.decodeGenerator = exports.endsWithIncompleteUtfPairSurrogate = exports.decodeToken = exports.encode = exports.isWithinTokenLimit = void 0;
|
|
8
|
+
exports.decode = exports.decodeAsyncGenerator = exports.decodeGenerator = exports.endsWithIncompleteUtfPairSurrogate = exports.decodeToken = exports.encode = exports.isWithinTokenLimit = exports.encodeGenerator = void 0;
|
|
9
9
|
// eslint-disable-next-line @typescript-eslint/ban-ts-comment, @typescript-eslint/prefer-ts-expect-error
|
|
10
10
|
// @ts-ignore
|
|
11
11
|
const bpeRanks_json_1 = __importDefault(require("../data/bpeRanks.json"));
|
|
@@ -106,6 +106,7 @@ function* encodeGenerator(text, cache = new Map()) {
|
|
|
106
106
|
yield newTokens;
|
|
107
107
|
}
|
|
108
108
|
}
|
|
109
|
+
exports.encodeGenerator = encodeGenerator;
|
|
109
110
|
/**
|
|
110
111
|
* @returns {false | number} false if token limit is exceeded, otherwise the number of tokens
|
|
111
112
|
*/
|
package/cjs/encoder.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"encoder.js","sourceRoot":"","sources":["../src/encoder.ts"],"names":[],"mappings":";AAAA,0CAA0C;AAC1C,8CAA8C;;;;;;AAE9C,wGAAwG;AACxG,aAAa;AACb,0EAAgD;AAChD,wGAAwG;AACxG,aAAa;AACb,wEAA8C;AAC9C,mCAOgB;AAEhB,MAAM,OAAO,GAA2B,sBAAW,CAAA;AACnD,MAAM,QAAQ,GAA2B,uBAAY,CAAA;AAErD,MAAM,OAAO,GAA2B,IAAA,eAAO,EAC7C,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,EACtB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CACrB,CAAA;AAED,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAA;AACrC,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAA;AAErC,4CAA4C;AAC5C,MAAM,gBAAgB,GAAG,CAAC,IAAI,CAAC,CAAA;AAE/B,SAAS,sBAAsB;IAC7B,MAAM,UAAU,GAAG,IAAA,aAAK,EAAC,IAAA,mBAAW,EAAC,GAAG,CAAC,EAAE,IAAA,mBAAW,EAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAA;IAChE,MAAM,YAAY,GAAG,IAAA,aAAK,EAAC,IAAA,mBAAW,EAAC,GAAG,CAAC,EAAE,IAAA,mBAAW,EAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAA;IAClE,MAAM,YAAY,GAAG,IAAA,aAAK,EAAC,IAAA,mBAAW,EAAC,GAAG,CAAC,EAAE,IAAA,mBAAW,EAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAA;IAElE,MAAM,iBAAiB,GAAG,CAAC,GAAG,UAAU,EAAE,GAAG,YAAY,EAAE,GAAG,YAAY,CAAC,CAAA;IAC3E,MAAM,gBAAgB,GAAG,CAAC,GAAG,iBAAiB,CAAC,CAAA;IAE/C,IAAI,kBAAkB,GAAG,CAAC,CAAA;IAC1B,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,gBAAgB,EAAE,SAAS,EAAE,EAAE;QACjE,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YAC1C,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;YACjC,gBAAgB,CAAC,IAAI,CAAC,gBAAgB,GAAG,kBAAkB,CAAC,CAAA;YAC5D,kBAAkB,IAAI,CAAC,CAAA;SACxB;KACF;IAED,MAAM,YAAY,GAAG,gBAAgB,CAAC,GAAG,CAAC,uBAAe,CAAC,CAAA;IAC1D,OAAO,IAAI,GAAG,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,CAAA;AACxE,CAAC;AAED,MAAM,GAAG,GACP,8EAA8E,CAAA;AAEhF,MAAM,WAAW,GAAG,sBAAsB,EAAE,CAAA;AAC5C,MAAM,WAAW,GAAwB,IAAI,GAAG,CAC9C,CAAC,GAAG,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CACzC,CAAA;AAED,SAAS,GAAG,CAAC,KAAa,EAAE,QAA6B,IAAI,GAAG,EAAE;IAChE,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE;QACpB,OAAO,KAAK,CAAC,GAAG,CAAC,KAAK,CAAE,CAAA;KACzB;IAED,IAAI,IAAI,GAAG,IAAA,kBAAU,EAAC,KAAK,CAAC,CAAA;IAC5B,IAAI,KAAK,GAAG,IAAA,gBAAQ,EAAC,IAAI,CAAC,CAAA;IAE1B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;QACtB,OAAO,KAAK,CAAA;KACb;IAED,iDAAiD;IACjD,OAAO,IAAI,EAAE;QACX,MAAM,QAAQ,GAAkC,IAAI,GAAG,EAAE,CAAA;QACzD,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;YACrB,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;YACrC,QAAQ,CAAC,GAAG;YACV,4CAA4C;YAC5C,OAAO,IAAI,KAAK,WAAW,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAChE,IAAI,CACL,CAAA;QACH,CAAC,CAAC,CAAA;QACF,MAAM,IAAI,GAAG,CAAC,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAA;QAEjC,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,CAAA;QAE9C,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,QAAQ,CAAC,EAAE;YAC9C,MAAK;SACN;QAED,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,GAAG,MAAM,CAAA;QAC9B,IAAI,OAAO,GAAa,EAAE,CAAA;QAC1B,IAAI,CAAC,GAAG,CAAC,CAAA;QAET,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE;YACtB,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA;YAChC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE;gBACZ,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;gBACvC,MAAK;aACN;YACD,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;YAC1C,CAAC,GAAG,CAAC,CAAA;YAEL,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,MAAM,EAAE;gBACtE,OAAO,CAAC,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,CAAA;gBAC5B,CAAC,IAAI,CAAC,CAAA;aACP;iBAAM;gBACL,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC,CAAA;gBACtB,CAAC,IAAI,CAAC,CAAA;aACP;SACF;QAED,IAAI,GAAG,OAAO,CAAA;QACd,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE;YACrB,MAAK;SACN;aAAM;YACL,KAAK,GAAG,IAAA,gBAAQ,EAAC,IAAI,CAAC,CAAA;SACvB;KACF;IAED,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IAC7B,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;IAExB,OAAO,MAAM,CAAA;AACf,CAAC;AAED,
|
|
1
|
+
{"version":3,"file":"encoder.js","sourceRoot":"","sources":["../src/encoder.ts"],"names":[],"mappings":";AAAA,0CAA0C;AAC1C,8CAA8C;;;;;;AAE9C,wGAAwG;AACxG,aAAa;AACb,0EAAgD;AAChD,wGAAwG;AACxG,aAAa;AACb,wEAA8C;AAC9C,mCAOgB;AAEhB,MAAM,OAAO,GAA2B,sBAAW,CAAA;AACnD,MAAM,QAAQ,GAA2B,uBAAY,CAAA;AAErD,MAAM,OAAO,GAA2B,IAAA,eAAO,EAC7C,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,EACtB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CACrB,CAAA;AAED,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAA;AACrC,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAA;AAErC,4CAA4C;AAC5C,MAAM,gBAAgB,GAAG,CAAC,IAAI,CAAC,CAAA;AAE/B,SAAS,sBAAsB;IAC7B,MAAM,UAAU,GAAG,IAAA,aAAK,EAAC,IAAA,mBAAW,EAAC,GAAG,CAAC,EAAE,IAAA,mBAAW,EAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAA;IAChE,MAAM,YAAY,GAAG,IAAA,aAAK,EAAC,IAAA,mBAAW,EAAC,GAAG,CAAC,EAAE,IAAA,mBAAW,EAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAA;IAClE,MAAM,YAAY,GAAG,IAAA,aAAK,EAAC,IAAA,mBAAW,EAAC,GAAG,CAAC,EAAE,IAAA,mBAAW,EAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAA;IAElE,MAAM,iBAAiB,GAAG,CAAC,GAAG,UAAU,EAAE,GAAG,YAAY,EAAE,GAAG,YAAY,CAAC,CAAA;IAC3E,MAAM,gBAAgB,GAAG,CAAC,GAAG,iBAAiB,CAAC,CAAA;IAE/C,IAAI,kBAAkB,GAAG,CAAC,CAAA;IAC1B,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,gBAAgB,EAAE,SAAS,EAAE,EAAE;QACjE,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YAC1C,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;YACjC,gBAAgB,CAAC,IAAI,CAAC,gBAAgB,GAAG,kBAAkB,CAAC,CAAA;YAC5D,kBAAkB,IAAI,CAAC,CAAA;SACxB;KACF;IAED,MAAM,YAAY,GAAG,gBAAgB,CAAC,GAAG,CAAC,uBAAe,CAAC,CAAA;IAC1D,OAAO,IAAI,GAAG,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,CAAA;AACxE,CAAC;AAED,MAAM,GAAG,GACP,8EAA8E,CAAA;AAEhF,MAAM,WAAW,GAAG,sBAAsB,EAAE,CAAA;AAC5C,MAAM,WAAW,GAAwB,IAAI,GAAG,CAC9C,CAAC,GAAG,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CACzC,CAAA;AAED,SAAS,GAAG,CAAC,KAAa,EAAE,QAA6B,IAAI,GAAG,EAAE;IAChE,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE;QACpB,OAAO,KAAK,CAAC,GAAG,CAAC,KAAK,CAAE,CAAA;KACzB;IAED,IAAI,IAAI,GAAG,IAAA,kBAAU,EAAC,KAAK,CAAC,CAAA;IAC5B,IAAI,KAAK,GAAG,IAAA,gBAAQ,EAAC,IAAI,CAAC,CAAA;IAE1B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;QACtB,OAAO,KAAK,CAAA;KACb;IAED,iDAAiD;IACjD,OAAO,IAAI,EAAE;QACX,MAAM,QAAQ,GAAkC,IAAI,GAAG,EAAE,CAAA;QACzD,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;YACrB,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;YACrC,QAAQ,CAAC,GAAG;YACV,4CAA4C;YAC5C,OAAO,IAAI,KAAK,WAAW,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAChE,IAAI,CACL,CAAA;QACH,CAAC,CAAC,CAAA;QACF,MAAM,IAAI,GAAG,CAAC,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAA;QAEjC,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,CAAA;QAE9C,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,QAAQ,CAAC,EAAE;YAC9C,MAAK;SACN;QAED,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,GAAG,MAAM,CAAA;QAC9B,IAAI,OAAO,GAAa,EAAE,CAAA;QAC1B,IAAI,CAAC,GAAG,CAAC,CAAA;QAET,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE;YACtB,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA;YAChC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE;gBACZ,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;gBACvC,MAAK;aACN;YACD,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;YAC1C,CAAC,GAAG,CAAC,CAAA;YAEL,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,MAAM,EAAE;gBACtE,OAAO,CAAC,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,CAAA;gBAC5B,CAAC,IAAI,CAAC,CAAA;aACP;iBAAM;gBACL,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC,CAAA;gBACtB,CAAC,IAAI,CAAC,CAAA;aACP;SACF;QAED,IAAI,GAAG,OAAO,CAAA;QACd,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE;YACrB,MAAK;SACN;aAAM;YACL,KAAK,GAAG,IAAA,gBAAQ,EAAC,IAAI,CAAC,CAAA;SACvB;KACF;IAED,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IAC7B,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;IAExB,OAAO,MAAM,CAAA;AACf,CAAC;AAED,QAAe,CAAC,CAAC,eAAe,CAC9B,IAAY,EACZ,QAA6B,IAAI,GAAG,EAAE;IAEtC,KAAK,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;QACtC,KAAK,GAAG,CAAC,GAAG,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;aACnC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;aACpC,IAAI,CAAC,EAAE,CAAC,CAAA;QAEX,MAAM,SAAS,GAAG,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC;aAChC,KAAK,CAAC,GAAG,CAAC;aACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC,CAAA;QAE1B,MAAM,SAAS,CAAA;KAChB;AACH,CAAC;AAfD,0CAeC;AAED;;GAEG;AACH,SAAgB,kBAAkB,CAChC,IAAY,EACZ,UAAkB,EAClB,QAA6B,IAAI,GAAG,EAAE;IAEtC,MAAM,cAAc,GAAG,eAAe,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;IACnD,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE;QACnC,KAAK,IAAI,MAAM,CAAC,MAAM,CAAA;QACtB,IAAI,KAAK,GAAG,UAAU,EAAE;YACtB,OAAO,KAAK,CAAA;SACb;KACF;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAdD,gDAcC;AAED,SAAgB,MAAM,CACpB,IAAY,EACZ,QAA6B,IAAI,GAAG,EAAE;IAEtC,OAAO,CAAC,GAAG,eAAe,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;AAClD,CAAC;AALD,wBAKC;AAED,SAAgB,WAAW,CAAC,KAAa;IACvC,MAAM,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,CAAA;IACnC,IAAI,OAAO,YAAY,KAAK,WAAW,EAAE;QACvC,OAAO,EAAE,CAAA;KACV;IACD,MAAM,YAAY,GAAG,IAAA,kBAAU,EAAC,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,CAAA;IAC7E,OAAO,WAAW,CAAC,MAAM,CAAC,IAAI,UAAU,CAAC,YAAY,CAAC,EAAE;QACtD,MAAM,EAAE,IAAI;KACb,CAAC,CAAA;AACJ,CAAC;AATD,kCASC;AAED,MAAM,oBAAoB,GAAG,KAAM,CAAA;AACnC,MAAM,kBAAkB,GAAG,KAAM,CAAA;AACjC,SAAgB,kCAAkC,CAAC,MAAc;IAC/D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAA;IACrC,kDAAkD;IAClD,MAAM,YAAY,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IACzD,OAAO,CACL,YAAY,IAAI,oBAAoB,IAAI,YAAY,IAAI,kBAAkB,CAC3E,CAAA;AACH,CAAC;AAPD,gFAOC;AAED,QAAe,CAAC,CAAC,eAAe,CAC9B,MAAwB;IAExB,IAAI,MAAM,GAAG,EAAE,CAAA;IAEf,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;QAC1B,MAAM,IAAI,WAAW,CAAC,KAAK,CAAC,CAAA;QAE5B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,kCAAkC,CAAC,MAAM,CAAC,EAAE;YACrE,yEAAyE;YACzE,uCAAuC;YACvC,SAAQ;SACT;aAAM;YACL,MAAM,MAAM,CAAA;YACZ,eAAe;YACf,MAAM,GAAG,EAAE,CAAA;SACZ;KACF;IAED,+CAA+C;IAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;QACrB,MAAM,MAAM,CAAA;KACb;AACH,CAAC;AAvBD,0CAuBC;AAED;;;;;GAKG;AACI,KAAK,SAAS,CAAC,CAAC,oBAAoB,CACzC,MAA6B;IAE7B,IAAI,MAAM,GAAG,EAAE,CAAA;IAEf,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE;QAChC,MAAM,IAAI,WAAW,CAAC,KAAK,CAAC,CAAA;QAE5B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,kCAAkC,CAAC,MAAM,CAAC,EAAE;YACrE,yEAAyE;YACzE,uCAAuC;YACvC,SAAQ;SACT;aAAM;YACL,MAAM,MAAM,CAAA;YACZ,eAAe;YACf,MAAM,GAAG,EAAE,CAAA;SACZ;KACF;IAED,+CAA+C;IAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;QACrB,MAAM,MAAM,CAAA;KACb;AACH,CAAC;AAvBD,oDAuBC;AAED,SAAgB,MAAM,CAAC,MAAgB;IACrC,OAAO,CAAC,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;AAC9C,CAAC;AAFD,wBAEC"}
|
package/esm/encoder.d.ts
CHANGED
package/esm/encoder.js
CHANGED
|
@@ -89,7 +89,7 @@ function bpe(token, cache = new Map()) {
|
|
|
89
89
|
cache.set(token, result);
|
|
90
90
|
return result;
|
|
91
91
|
}
|
|
92
|
-
function* encodeGenerator(text, cache = new Map()) {
|
|
92
|
+
export function* encodeGenerator(text, cache = new Map()) {
|
|
93
93
|
for (let [token] of text.matchAll(pat)) {
|
|
94
94
|
token = [...textEncoder.encode(token)]
|
|
95
95
|
.map((x) => byteEncoder.get(x) ?? '')
|
package/esm/encoder.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"encoder.js","sourceRoot":"","sources":["../src/encoder.ts"],"names":[],"mappings":"AAAA,0CAA0C;AAC1C,8CAA8C;AAE9C,wGAAwG;AACxG,aAAa;AACb,OAAO,YAAY,MAAM,uBAAuB,CAAA;AAChD,wGAAwG;AACxG,aAAa;AACb,OAAO,WAAW,MAAM,sBAAsB,CAAA;AAC9C,OAAO,EACL,OAAO,EACP,WAAW,EACX,eAAe,EACf,QAAQ,EACR,KAAK,EACL,UAAU,GACX,MAAM,SAAS,CAAA;AAEhB,MAAM,OAAO,GAA2B,WAAW,CAAA;AACnD,MAAM,QAAQ,GAA2B,YAAY,CAAA;AAErD,MAAM,OAAO,GAA2B,OAAO,CAC7C,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,EACtB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CACrB,CAAA;AAED,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAA;AACrC,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAA;AAErC,4CAA4C;AAC5C,MAAM,gBAAgB,GAAG,CAAC,IAAI,CAAC,CAAA;AAE/B,SAAS,sBAAsB;IAC7B,MAAM,UAAU,GAAG,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAA;IAChE,MAAM,YAAY,GAAG,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAA;IAClE,MAAM,YAAY,GAAG,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAA;IAElE,MAAM,iBAAiB,GAAG,CAAC,GAAG,UAAU,EAAE,GAAG,YAAY,EAAE,GAAG,YAAY,CAAC,CAAA;IAC3E,MAAM,gBAAgB,GAAG,CAAC,GAAG,iBAAiB,CAAC,CAAA;IAE/C,IAAI,kBAAkB,GAAG,CAAC,CAAA;IAC1B,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,gBAAgB,EAAE,SAAS,EAAE,EAAE;QACjE,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YAC1C,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;YACjC,gBAAgB,CAAC,IAAI,CAAC,gBAAgB,GAAG,kBAAkB,CAAC,CAAA;YAC5D,kBAAkB,IAAI,CAAC,CAAA;SACxB;KACF;IAED,MAAM,YAAY,GAAG,gBAAgB,CAAC,GAAG,CAAC,eAAe,CAAC,CAAA;IAC1D,OAAO,IAAI,GAAG,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,CAAA;AACxE,CAAC;AAED,MAAM,GAAG,GACP,8EAA8E,CAAA;AAEhF,MAAM,WAAW,GAAG,sBAAsB,EAAE,CAAA;AAC5C,MAAM,WAAW,GAAwB,IAAI,GAAG,CAC9C,CAAC,GAAG,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CACzC,CAAA;AAED,SAAS,GAAG,CAAC,KAAa,EAAE,QAA6B,IAAI,GAAG,EAAE;IAChE,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE;QACpB,OAAO,KAAK,CAAC,GAAG,CAAC,KAAK,CAAE,CAAA;KACzB;IAED,IAAI,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,CAAA;IAC5B,IAAI,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAA;IAE1B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;QACtB,OAAO,KAAK,CAAA;KACb;IAED,iDAAiD;IACjD,OAAO,IAAI,EAAE;QACX,MAAM,QAAQ,GAAkC,IAAI,GAAG,EAAE,CAAA;QACzD,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;YACrB,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;YACrC,QAAQ,CAAC,GAAG;YACV,4CAA4C;YAC5C,OAAO,IAAI,KAAK,WAAW,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAChE,IAAI,CACL,CAAA;QACH,CAAC,CAAC,CAAA;QACF,MAAM,IAAI,GAAG,CAAC,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAA;QAEjC,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,CAAA;QAE9C,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,QAAQ,CAAC,EAAE;YAC9C,MAAK;SACN;QAED,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,GAAG,MAAM,CAAA;QAC9B,IAAI,OAAO,GAAa,EAAE,CAAA;QAC1B,IAAI,CAAC,GAAG,CAAC,CAAA;QAET,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE;YACtB,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA;YAChC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE;gBACZ,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;gBACvC,MAAK;aACN;YACD,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;YAC1C,CAAC,GAAG,CAAC,CAAA;YAEL,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,MAAM,EAAE;gBACtE,OAAO,CAAC,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,CAAA;gBAC5B,CAAC,IAAI,CAAC,CAAA;aACP;iBAAM;gBACL,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC,CAAA;gBACtB,CAAC,IAAI,CAAC,CAAA;aACP;SACF;QAED,IAAI,GAAG,OAAO,CAAA;QACd,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE;YACrB,MAAK;SACN;aAAM;YACL,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAA;SACvB;KACF;IAED,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IAC7B,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;IAExB,OAAO,MAAM,CAAA;AACf,CAAC;AAED,
|
|
1
|
+
{"version":3,"file":"encoder.js","sourceRoot":"","sources":["../src/encoder.ts"],"names":[],"mappings":"AAAA,0CAA0C;AAC1C,8CAA8C;AAE9C,wGAAwG;AACxG,aAAa;AACb,OAAO,YAAY,MAAM,uBAAuB,CAAA;AAChD,wGAAwG;AACxG,aAAa;AACb,OAAO,WAAW,MAAM,sBAAsB,CAAA;AAC9C,OAAO,EACL,OAAO,EACP,WAAW,EACX,eAAe,EACf,QAAQ,EACR,KAAK,EACL,UAAU,GACX,MAAM,SAAS,CAAA;AAEhB,MAAM,OAAO,GAA2B,WAAW,CAAA;AACnD,MAAM,QAAQ,GAA2B,YAAY,CAAA;AAErD,MAAM,OAAO,GAA2B,OAAO,CAC7C,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,EACtB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CACrB,CAAA;AAED,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAA;AACrC,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAA;AAErC,4CAA4C;AAC5C,MAAM,gBAAgB,GAAG,CAAC,IAAI,CAAC,CAAA;AAE/B,SAAS,sBAAsB;IAC7B,MAAM,UAAU,GAAG,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAA;IAChE,MAAM,YAAY,GAAG,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAA;IAClE,MAAM,YAAY,GAAG,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAA;IAElE,MAAM,iBAAiB,GAAG,CAAC,GAAG,UAAU,EAAE,GAAG,YAAY,EAAE,GAAG,YAAY,CAAC,CAAA;IAC3E,MAAM,gBAAgB,GAAG,CAAC,GAAG,iBAAiB,CAAC,CAAA;IAE/C,IAAI,kBAAkB,GAAG,CAAC,CAAA;IAC1B,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,gBAAgB,EAAE,SAAS,EAAE,EAAE;QACjE,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YAC1C,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;YACjC,gBAAgB,CAAC,IAAI,CAAC,gBAAgB,GAAG,kBAAkB,CAAC,CAAA;YAC5D,kBAAkB,IAAI,CAAC,CAAA;SACxB;KACF;IAED,MAAM,YAAY,GAAG,gBAAgB,CAAC,GAAG,CAAC,eAAe,CAAC,CAAA;IAC1D,OAAO,IAAI,GAAG,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,CAAA;AACxE,CAAC;AAED,MAAM,GAAG,GACP,8EAA8E,CAAA;AAEhF,MAAM,WAAW,GAAG,sBAAsB,EAAE,CAAA;AAC5C,MAAM,WAAW,GAAwB,IAAI,GAAG,CAC9C,CAAC,GAAG,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CACzC,CAAA;AAED,SAAS,GAAG,CAAC,KAAa,EAAE,QAA6B,IAAI,GAAG,EAAE;IAChE,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE;QACpB,OAAO,KAAK,CAAC,GAAG,CAAC,KAAK,CAAE,CAAA;KACzB;IAED,IAAI,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,CAAA;IAC5B,IAAI,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAA;IAE1B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;QACtB,OAAO,KAAK,CAAA;KACb;IAED,iDAAiD;IACjD,OAAO,IAAI,EAAE;QACX,MAAM,QAAQ,GAAkC,IAAI,GAAG,EAAE,CAAA;QACzD,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;YACrB,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;YACrC,QAAQ,CAAC,GAAG;YACV,4CAA4C;YAC5C,OAAO,IAAI,KAAK,WAAW,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAChE,IAAI,CACL,CAAA;QACH,CAAC,CAAC,CAAA;QACF,MAAM,IAAI,GAAG,CAAC,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAA;QAEjC,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,CAAA;QAE9C,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,QAAQ,CAAC,EAAE;YAC9C,MAAK;SACN;QAED,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,GAAG,MAAM,CAAA;QAC9B,IAAI,OAAO,GAAa,EAAE,CAAA;QAC1B,IAAI,CAAC,GAAG,CAAC,CAAA;QAET,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE;YACtB,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA;YAChC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE;gBACZ,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;gBACvC,MAAK;aACN;YACD,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;YAC1C,CAAC,GAAG,CAAC,CAAA;YAEL,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,MAAM,EAAE;gBACtE,OAAO,CAAC,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,CAAA;gBAC5B,CAAC,IAAI,CAAC,CAAA;aACP;iBAAM;gBACL,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC,CAAA;gBACtB,CAAC,IAAI,CAAC,CAAA;aACP;SACF;QAED,IAAI,GAAG,OAAO,CAAA;QACd,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE;YACrB,MAAK;SACN;aAAM;YACL,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAA;SACvB;KACF;IAED,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IAC7B,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;IAExB,OAAO,MAAM,CAAA;AACf,CAAC;AAED,MAAM,SAAS,CAAC,CAAC,eAAe,CAC9B,IAAY,EACZ,QAA6B,IAAI,GAAG,EAAE;IAEtC,KAAK,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;QACtC,KAAK,GAAG,CAAC,GAAG,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;aACnC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;aACpC,IAAI,CAAC,EAAE,CAAC,CAAA;QAEX,MAAM,SAAS,GAAG,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC;aAChC,KAAK,CAAC,GAAG,CAAC;aACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC,CAAA;QAE1B,MAAM,SAAS,CAAA;KAChB;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAChC,IAAY,EACZ,UAAkB,EAClB,QAA6B,IAAI,GAAG,EAAE;IAEtC,MAAM,cAAc,GAAG,eAAe,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;IACnD,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE;QACnC,KAAK,IAAI,MAAM,CAAC,MAAM,CAAA;QACtB,IAAI,KAAK,GAAG,UAAU,EAAE;YACtB,OAAO,KAAK,CAAA;SACb;KACF;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,MAAM,UAAU,MAAM,CACpB,IAAY,EACZ,QAA6B,IAAI,GAAG,EAAE;IAEtC,OAAO,CAAC,GAAG,eAAe,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;AAClD,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,KAAa;IACvC,MAAM,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,CAAA;IACnC,IAAI,OAAO,YAAY,KAAK,WAAW,EAAE;QACvC,OAAO,EAAE,CAAA;KACV;IACD,MAAM,YAAY,GAAG,UAAU,CAAC,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,CAAA;IAC7E,OAAO,WAAW,CAAC,MAAM,CAAC,IAAI,UAAU,CAAC,YAAY,CAAC,EAAE;QACtD,MAAM,EAAE,IAAI;KACb,CAAC,CAAA;AACJ,CAAC;AAED,MAAM,oBAAoB,GAAG,KAAM,CAAA;AACnC,MAAM,kBAAkB,GAAG,KAAM,CAAA;AACjC,MAAM,UAAU,kCAAkC,CAAC,MAAc;IAC/D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAA;IACrC,kDAAkD;IAClD,MAAM,YAAY,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IACzD,OAAO,CACL,YAAY,IAAI,oBAAoB,IAAI,YAAY,IAAI,kBAAkB,CAC3E,CAAA;AACH,CAAC;AAED,MAAM,SAAS,CAAC,CAAC,eAAe,CAC9B,MAAwB;IAExB,IAAI,MAAM,GAAG,EAAE,CAAA;IAEf,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;QAC1B,MAAM,IAAI,WAAW,CAAC,KAAK,CAAC,CAAA;QAE5B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,kCAAkC,CAAC,MAAM,CAAC,EAAE;YACrE,yEAAyE;YACzE,uCAAuC;YACvC,SAAQ;SACT;aAAM;YACL,MAAM,MAAM,CAAA;YACZ,eAAe;YACf,MAAM,GAAG,EAAE,CAAA;SACZ;KACF;IAED,+CAA+C;IAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;QACrB,MAAM,MAAM,CAAA;KACb;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,oBAAoB,CACzC,MAA6B;IAE7B,IAAI,MAAM,GAAG,EAAE,CAAA;IAEf,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE;QAChC,MAAM,IAAI,WAAW,CAAC,KAAK,CAAC,CAAA;QAE5B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,kCAAkC,CAAC,MAAM,CAAC,EAAE;YACrE,yEAAyE;YACzE,uCAAuC;YACvC,SAAQ;SACT;aAAM;YACL,MAAM,MAAM,CAAA;YACZ,eAAe;YACf,MAAM,GAAG,EAAE,CAAA;SACZ;KACF;IAED,+CAA+C;IAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;QACrB,MAAM,MAAM,CAAA;KACb;AACH,CAAC;AAED,MAAM,UAAU,MAAM,CAAC,MAAgB;IACrC,OAAO,CAAC,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;AAC9C,CAAC"}
|
package/package.json
CHANGED
package/src/encoder.ts
CHANGED
|
@@ -126,7 +126,7 @@ function bpe(token: string, cache: Map<string, string> = new Map()): string {
|
|
|
126
126
|
return result
|
|
127
127
|
}
|
|
128
128
|
|
|
129
|
-
function* encodeGenerator(
|
|
129
|
+
export function* encodeGenerator(
|
|
130
130
|
text: string,
|
|
131
131
|
cache: Map<string, string> = new Map(),
|
|
132
132
|
): Generator<number[], void, undefined> {
|