gpt-tokenizer 2.0.0-beta.1 → 2.0.0-beta.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/README.md +125 -43
  2. package/cjs/GptEncoding.d.ts +5 -0
  3. package/cjs/GptEncoding.js +6 -0
  4. package/cjs/GptEncoding.js.map +1 -1
  5. package/cjs/GptEncoding.test.js +38 -0
  6. package/cjs/GptEncoding.test.js.map +1 -1
  7. package/cjs/encoding/cl100k_base.d.ts +4 -0
  8. package/cjs/encoding/cl100k_base.js +17 -0
  9. package/cjs/encoding/cl100k_base.js.map +1 -1
  10. package/cjs/encoding/p50k_base.d.ts +4 -0
  11. package/cjs/encoding/p50k_base.js +17 -0
  12. package/cjs/encoding/p50k_base.js.map +1 -1
  13. package/cjs/encoding/p50k_edit.d.ts +4 -0
  14. package/cjs/encoding/p50k_edit.js +17 -0
  15. package/cjs/encoding/p50k_edit.js.map +1 -1
  16. package/cjs/encoding/r50k_base.d.ts +4 -0
  17. package/cjs/encoding/r50k_base.js +17 -0
  18. package/cjs/encoding/r50k_base.js.map +1 -1
  19. package/cjs/modelParams.js +12 -16
  20. package/cjs/modelParams.js.map +1 -1
  21. package/cjs/specialTokens.d.ts +5 -0
  22. package/cjs/specialTokens.js +9 -0
  23. package/cjs/specialTokens.js.map +1 -0
  24. package/data/TestPlans.txt +1024 -0
  25. package/dist/cl100k_base.js +1 -1
  26. package/dist/cl100k_base.js.map +1 -1
  27. package/dist/p50k_base.js +1 -1
  28. package/dist/p50k_base.js.map +1 -1
  29. package/dist/p50k_edit.js +1 -1
  30. package/dist/p50k_edit.js.map +1 -1
  31. package/dist/r50k_base.js +1 -1
  32. package/dist/r50k_base.js.map +1 -1
  33. package/esm/GptEncoding.d.ts +5 -0
  34. package/esm/GptEncoding.js +8 -1
  35. package/esm/GptEncoding.js.map +1 -1
  36. package/esm/GptEncoding.test.js +35 -0
  37. package/esm/GptEncoding.test.js.map +1 -1
  38. package/esm/encoding/cl100k_base.d.ts +4 -0
  39. package/esm/encoding/cl100k_base.js +3 -0
  40. package/esm/encoding/cl100k_base.js.map +1 -1
  41. package/esm/encoding/p50k_base.d.ts +4 -0
  42. package/esm/encoding/p50k_base.js +3 -0
  43. package/esm/encoding/p50k_base.js.map +1 -1
  44. package/esm/encoding/p50k_edit.d.ts +4 -0
  45. package/esm/encoding/p50k_edit.js +3 -0
  46. package/esm/encoding/p50k_edit.js.map +1 -1
  47. package/esm/encoding/r50k_base.d.ts +4 -0
  48. package/esm/encoding/r50k_base.js +3 -0
  49. package/esm/encoding/r50k_base.js.map +1 -1
  50. package/esm/modelParams.js +1 -5
  51. package/esm/modelParams.js.map +1 -1
  52. package/esm/specialTokens.d.ts +5 -0
  53. package/esm/specialTokens.js +6 -0
  54. package/esm/specialTokens.js.map +1 -0
  55. package/package.json +5 -5
  56. package/src/GptEncoding.test.ts +40 -0
  57. package/src/GptEncoding.ts +13 -0
  58. package/src/encoding/cl100k_base.ts +4 -0
  59. package/src/encoding/p50k_base.ts +4 -0
  60. package/src/encoding/p50k_edit.ts +4 -0
  61. package/src/encoding/r50k_base.ts +4 -0
  62. package/src/modelParams.ts +7 -6
  63. package/src/specialTokens.ts +5 -0
package/esm/GptEncoding.d.ts CHANGED
@@ -2,6 +2,11 @@ import { type EncodingName, type ModelName } from './mapping.js';
2
2
  import { type GetMergeableRanksAsyncFn, type GetMergeableRanksFn } from './modelParams.js';
3
3
  export declare const DISALLOW_ALL_SPECIAL_TOKENS = "all";
4
4
  export declare class GptEncoding {
5
+ static EndOfPrompt: string;
6
+ static EndOfText: string;
7
+ static FimMiddle: string;
8
+ static FimPrefix: string;
9
+ static FimSuffix: string;
5
10
  decoder: TextDecoder;
6
11
  private bytePairEncodingCoreProcessor;
7
12
  private specialTokenMapping;
package/esm/GptEncoding.js CHANGED
@@ -3,9 +3,15 @@ import { BytePairEncodingCore } from './BytePairEncodingCore.js';
3
3
  import { escapeRegExp } from './escapeRegExp.js';
4
4
  import { modelToEncodingMap, } from './mapping.js';
5
5
  import { getModelParams, getModelParamsAsync, } from './modelParams.js';
6
+ import { EndOfPrompt, EndOfText, FimMiddle, FimPrefix, FimSuffix, } from './specialTokens.js';
6
7
  import { endsWithIncompleteUtfPairSurrogate } from './utfUtil.js';
7
8
  export const DISALLOW_ALL_SPECIAL_TOKENS = 'all';
8
- export class GptEncoding {
9
+ class GptEncoding {
10
+ static EndOfPrompt = EndOfPrompt;
11
+ static EndOfText = EndOfText;
12
+ static FimMiddle = FimMiddle;
13
+ static FimPrefix = FimPrefix;
14
+ static FimSuffix = FimSuffix;
9
15
  decoder = new TextDecoder('utf8');
10
16
  bytePairEncodingCoreProcessor;
11
17
  specialTokenMapping;
@@ -148,4 +154,5 @@ export class GptEncoding {
148
154
  return [...this.decodeGenerator(inputTokensToDecode)].join('');
149
155
  }
150
156
  }
157
+ export { GptEncoding };
151
158
  //# sourceMappingURL=GptEncoding.js.map
package/esm/GptEncoding.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"GptEncoding.js","sourceRoot":"","sources":["../src/GptEncoding.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAA;AAChE,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAA;AAChD,OAAO,EAGL,kBAAkB,GACnB,MAAM,cAAc,CAAA;AACrB,OAAO,EAIL,cAAc,EACd,mBAAmB,GACpB,MAAM,kBAAkB,CAAA;AACzB,OAAO,EAAE,kCAAkC,EAAE,MAAM,cAAc,CAAA;AAEjE,MAAM,CAAC,MAAM,2BAA2B,GAAG,KAAK,CAAA;AAEhD,MAAM,OAAO,WAAW;IACtB,OAAO,GAAG,IAAI,WAAW,CAAC,MAAM,CAAC,CAAA;IACzB,6BAA6B,CAAsB;IACnD,mBAAmB,CAAqB;IAEhD,YAAoB,EAClB,eAAe,EACf,sBAAsB,EACtB,mBAAmB,EACnB,sBAAsB,GACV;QACZ,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAC5B,WAAW,CAAC,kBAAkB,CAAC,sBAAsB,CAAC,EACtD,WAAW,CAAC,kBAAkB,CAAC,mBAAmB,CAAC,CACpD,CAAA;QACD,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAA;QAE9C,IAAI,sBAAsB,KAAK,SAAS,EAAE;YACxC,IACE,sBAAsB,CAAC,IAAI,GAAG,mBAAmB,CAAC,IAAI;gBACtD,sBAAsB,EACtB;gBACA,MAAM,IAAI,KAAK,CACb,sFAAsF,CACvF,CAAA;aACF;YAED,IAAI,aAAa,KAAK,sBAAsB,GAAG,CAAC,EAAE;gBAChD,MAAM,IAAI,KAAK,CACb,gEAAgE,CACjE,CAAA;aACF;SACF;QAED,IAAI,CAAC,6BAA6B,GAAG,IAAI,oBAAoB,CAAC;YAC5D,eAAe,EAAE,sBAAsB;YACvC,mBAAmB,EAAE,mBAAmB;YACxC,eAAe;SAChB,CAAC,CAAA;QAEF,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACpC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACpC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACtD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACtD,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAChE,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAC9D,CAAC;IAED,MAAM,CAAC,cAAc,CACnB,YAA0B,EAC1B,iBAAsC;QAEtC,MAAM,WAAW,GAAG,cAAc,CAAC,YAAY,EAAE,iBAAiB,CAAC,CAAA;QACnE,OAAO,IAAI,WAAW,CAAC,WAAW,CAAC,CAAA;IACrC,CAAC;IAED,MAAM,CAAC,sBAAsB,CAC3B,SAAoB,EACpB,iBAAsC;QAEtC,MAAM,YAAY,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAA;QAClD,OAAO,WAAW,CAAC,cAAc,CAAC,YAAY,EAAE,iBAAiB,CAAC,CAAA;IACpE,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAC9B,YAA0B,EAC1B,iBAA2C;QAE3C,MAAM,WAAW,GAAG,MAAM,mBAAmB,CAC3C,YAAY,EACZ,iBAAiB,CAClB,CAAA;QACD,OAAO,IAAI,WAAW,CAAC,WAAW,CAAC,CAAA;IACrC,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,2BAA2B,CACtC,SAAoB,EACpB,iBAA2C;QAE3C,MAAM,YAAY,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAA;QAClD,OAAO,WAAW,CAAC,mBAAmB,CAAC,YAAY,EAAE,iBAAiB,CAAC,CAAA;IACzE,CAAC;IAEO,MAAM,CAAC,kBAAkB,CAAC,GAAyB;QACzD,IAAI,GAAG,GAAG,CAAC,CAAA;QACX,GAAG,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;YAClB,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAA;QAC1B,CAAC,CAAC,CAAA;QACF,OAAO,GAAG,CAAA;IACZ,CAAC;IAEO,MAAM,CAAC,iBAAiB,CAAC,MAAmB;QAClD,MAAM,aAAa,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,CAAA;QACnD,MAAM,KAAK,GAAG,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACrC,OAAO,IAAI,MAAM,CAAC,IAAI,KAAK,GAAG,CAAC,CAAA;IACjC,CAAC;IAED,eAAe,CACb,YAAoB,EACpB,iBAAiB,IAAI,GAAG,EAAU,EAClC,oBAAiC,IAAI,GAAG,CAAS;QAC/C,2BAA2B;KAC5B,CAAC;QAEF,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAS,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,CAAC,CAAA;QAEzE,IAAI,iBAAiB,CAAC,GAAG,CAAC,2BAA2B,CAAC,EAAE;YACtD,iBAAiB,GAAG,IAAI,GAAG,CAAS,gBAAgB,CAAC,CAAA;YACrD,iBAAiB,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;SAC/D;QAED,IAAI,cAAc,CAAC,GAAG,CAAC,2BAA2B,CAAC,EAAE;YACnD,cAAc,GAAG,gBAAgB,CAAA;SAClC;QAED,IAAI,iBAAiB,CAAC,IAAI,GAAG,CAAC,EAAE;YAC9B,MAAM,YAAY,GAAG,WAAW,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,CAAA;YACrE,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,YAAY,CAAC,CAAA;YAC9C,IAAI,KAAK,KAAK,IAAI,EAAE;gBAClB,MAAM,IAAI,KAAK,CAAC,mCAAmC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;aAC/D;SACF;QAED,OAAO,IAAI,CAAC,6BAA6B,CAAC,YAAY,CACpD,YAAY,EACZ,cAAc,CACf,CAAA;IACH,CAAC;IAED,MAAM,CACJ,YAAoB,EACpB,iBAAiB,IAAI,GAAG,EAAU,EAClC,oBAAiC,IAAI,GAAG,CAAS;QAC/C,2BAA2B;KAC5B,CAAC;QAEF,OAAO;YACL,GAAG,IAAI,CAAC,eAAe,CAAC,YAAY,EAAE,cAAc,EAAE,iBAAiB,CAAC;SACzE,CAAC,IAAI,EAAE,CAAA;IACV,CAAC;IAED;;OAEG;IACH,kBAAkB,CAAC,IAAY,EAAE,UAAkB;QACjD,MAAM,cAAc,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,CAAA;QACjD,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE;YACnC,KAAK,IAAI,MAAM,CAAC,MAAM,CAAA;YACtB,IAAI,KAAK,GAAG,UAAU,EAAE;gBACtB,OAAO,KAAK,CAAA;aACb;SACF;QACD,OAAO,KAAK,CAAA;IACd,CAAC;IAED,CAAC,eAAe,CACd,mBAAqC;QAErC,MAAM,oBAAoB,GACxB,IAAI,CAAC,6BAA6B,CAAC,YAAY,CAAC,mBAAmB,CAAC,CAAA;QAEtE,IAAI,MAAM,GAAG,EAAE,CAAA;QAEf,KAAK,MAAM,WAAW,IAAI,oBAAoB,EAAE;YAC9C,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;YAE5D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,kCAAkC,CAAC,MAAM,CAAC,EAAE;gBACrE,yEAAyE;gBACzE,uCAAuC;gBACvC,SAAQ;aACT;iBAAM;gBACL,MAAM,MAAM,CAAA;gBACZ,eAAe;gBACf,MAAM,GAAG,EAAE,CAAA;aACZ;SACF;QAED,+CAA+C;QAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;YACrB,MAAM,MAAM,CAAA;SACb;IACH,CAAC;IAED,KAAK,CAAC,CAAC,oBAAoB,CACzB,mBAA0C;QAE1C,MAAM,oBAAoB,GACxB,IAAI,CAAC,6BAA6B,CAAC,iBAAiB,CAAC,mBAAmB,CAAC,CAAA;QAE3E,IAAI,MAAM,GAAG,EAAE,CAAA;QAEf,IAAI,KAAK,EAAE,MAAM,WAAW,IAAI,oBAAoB,EAAE;YACpD,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;YAE5D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,kCAAkC,CAAC,MAAM,CAAC,EAAE;gBACrE,yEAAyE;gBACzE,uCAAuC;gBACvC,SAAQ;aACT;iBAAM;gBACL,MAAM,MAAM,CAAA;gBACZ,eAAe;gBACf,MAAM,GAAG,EAAE,CAAA;aACZ;SACF;QAED,+CAA+C;QAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;YACrB,MAAM,MAAM,CAAA;SACb;IACH,CAAC;IAED,MAAM,CAAC,mBAAqC;QAC1C,OAAO,CAAC,GAAG,IAAI,CAAC,eAAe,CAAC,mBAAmB,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAChE,CAAC;CACF"}
1
+ {"version":3,"file":"GptEncoding.js","sourceRoot":"","sources":["../src/GptEncoding.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAA;AAChE,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAA;AAChD,OAAO,EAGL,kBAAkB,GACnB,MAAM,cAAc,CAAA;AACrB,OAAO,EAIL,cAAc,EACd,mBAAmB,GACpB,MAAM,kBAAkB,CAAA;AACzB,OAAO,EACL,WAAW,EACX,SAAS,EACT,SAAS,EACT,SAAS,EACT,SAAS,GACV,MAAM,oBAAoB,CAAA;AAC3B,OAAO,EAAE,kCAAkC,EAAE,MAAM,cAAc,CAAA;AAEjE,MAAM,CAAC,MAAM,2BAA2B,GAAG,KAAK,CAAA;AAEhD,MAAa,WAAW;IACtB,MAAM,CAAC,WAAW,GAAG,WAAW,CAAA;IAChC,MAAM,CAAC,SAAS,GAAG,SAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,SAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,SAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,SAAS,CAAA;IAE5B,OAAO,GAAG,IAAI,WAAW,CAAC,MAAM,CAAC,CAAA;IACzB,6BAA6B,CAAsB;IACnD,mBAAmB,CAAqB;IAEhD,YAAoB,EAClB,eAAe,EACf,sBAAsB,EACtB,mBAAmB,EACnB,sBAAsB,GACV;QACZ,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAC5B,WAAW,CAAC,kBAAkB,CAAC,sBAAsB,CAAC,EACtD,WAAW,CAAC,kBAAkB,CAAC,mBAAmB,CAAC,CACpD,CAAA;QACD,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAA;QAE9C,IAAI,sBAAsB,KAAK,SAAS,EAAE;YACxC,IACE,sBAAsB,CAAC,IAAI,GAAG,mBAAmB,CAAC,IAAI;gBACtD,sBAAsB,EACtB;gBACA,MAAM,IAAI,KAAK,CACb,sFAAsF,CACvF,CAAA;aACF;YAED,IAAI,aAAa,KAAK,sBAAsB,GAAG,CAAC,EAAE;gBAChD,MAAM,IAAI,KAAK,CACb,gEAAgE,CACjE,CAAA;aACF;SACF;QAED,IAAI,CAAC,6BAA6B,GAAG,IAAI,oBAAoB,CAAC;YAC5D,eAAe,EAAE,sBAAsB;YACvC,mBAAmB,EAAE,mBAAmB;YACxC,eAAe;SAChB,CAAC,CAAA;QAEF,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACpC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACpC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACtD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACtD,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAChE,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAC9D,CAAC;IAED,MAAM,CAAC,cAAc,CACnB,YAA0B,EAC1B,iBAAsC;QAEtC,MAAM,WAAW,GAAG,cAAc,CAAC,YAAY,EAAE,iBAAiB,CAAC,CAAA;QACnE,OAAO,IAAI,WAAW,CAAC,WAAW,CAAC,CAAA;IACrC,CAAC;IAED,MAAM,CAAC,sBAAsB,CAC3B,SAAoB,EACpB,iBAAsC;QAEtC,MAAM,YAAY,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAA;QAClD,OAAO,WAAW,CAAC,cAAc,CAAC,YAAY,EAAE,iBAAiB,CAAC,CAAA;IACpE,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAC9B,YAA0B,EAC1B,iBAA2C;QAE3C,MAAM,WAAW,GAAG,MAAM,mBAAmB,CAC3C,YAAY,EACZ,iBAAiB,CAClB,CAAA;QACD,OAAO,IAAI,WAAW,CAAC,WAAW,CAAC,CAAA;IACrC,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,2BAA2B,CACtC,SAAoB,EACpB,iBAA2C;QAE3C,MAAM,YAAY,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAA;QAClD,OAAO,WAAW,CAAC,mBAAmB,CAAC,YAAY,EAAE,iBAAiB,CAAC,CAAA;IACzE,CAAC;IAEO,MAAM,CAAC,kBAAkB,CAAC,GAAyB;QACzD,IAAI,GAAG,GAAG,CAAC,CAAA;QACX,GAAG,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;YAClB,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAA;QAC1B,CAAC,CAAC,CAAA;QACF,OAAO,GAAG,CAAA;IACZ,CAAC;IAEO,MAAM,CAAC,iBAAiB,CAAC,MAAmB;QAClD,MAAM,aAAa,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,CAAA;QACnD,MAAM,KAAK,GAAG,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACrC,OAAO,IAAI,MAAM,CAAC,IAAI,KAAK,GAAG,CAAC,CAAA;IACjC,CAAC;IAED,eAAe,CACb,YAAoB,EACpB,iBAAiB,IAAI,GAAG,EAAU,EAClC,oBAAiC,IAAI,GAAG,CAAS;QAC/C,2BAA2B;KAC5B,CAAC;QAEF,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAS,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,CAAC,CAAA;QAEzE,IAAI,iBAAiB,CAAC,GAAG,CAAC,2BAA2B,CAAC,EAAE;YACtD,iBAAiB,GAAG,IAAI,GAAG,CAAS,gBAAgB,CAAC,CAAA;YACrD,iBAAiB,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;SAC/D;QAED,IAAI,cAAc,CAAC,GAAG,CAAC,2BAA2B,CAAC,EAAE;YACnD,cAAc,GAAG,gBAAgB,CAAA;SAClC;QAED,IAAI,iBAAiB,CAAC,IAAI,GAAG,CAAC,EAAE;YAC9B,MAAM,YAAY,GAAG,WAAW,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,CAAA;YACrE,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,YAAY,CAAC,CAAA;YAC9C,IAAI,KAAK,KAAK,IAAI,EAAE;gBAClB,MAAM,IAAI,KAAK,CAAC,mCAAmC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;aAC/D;SACF;QAED,OAAO,IAAI,CAAC,6BAA6B,CAAC,YAAY,CACpD,YAAY,EACZ,cAAc,CACf,CAAA;IACH,CAAC;IAED,MAAM,CACJ,YAAoB,EACpB,iBAAiB,IAAI,GAAG,EAAU,EAClC,oBAAiC,IAAI,GAAG,CAAS;QAC/C,2BAA2B;KAC5B,CAAC;QAEF,OAAO;YACL,GAAG,IAAI,CAAC,eAAe,CAAC,YAAY,EAAE,cAAc,EAAE,iBAAiB,CAAC;SACzE,CAAC,IAAI,EAAE,CAAA;IACV,CAAC;IAED;;OAEG;IACH,kBAAkB,CAAC,IAAY,EAAE,UAAkB;QACjD,MAAM,cAAc,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,CAAA;QACjD,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE;YACnC,KAAK,IAAI,MAAM,CAAC,MAAM,CAAA;YACtB,IAAI,KAAK,GAAG,UAAU,EAAE;gBACtB,OAAO,KAAK,CAAA;aACb;SACF;QACD,OAAO,KAAK,CAAA;IACd,CAAC;IAED,CAAC,eAAe,CACd,mBAAqC;QAErC,MAAM,oBAAoB,GACxB,IAAI,CAAC,6BAA6B,CAAC,YAAY,CAAC,mBAAmB,CAAC,CAAA;QAEtE,IAAI,MAAM,GAAG,EAAE,CAAA;QAEf,KAAK,MAAM,WAAW,IAAI,oBAAoB,EAAE;YAC9C,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;YAE5D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,kCAAkC,CAAC,MAAM,CAAC,EAAE;gBACrE,yEAAyE;gBACzE,uCAAuC;gBACvC,SAAQ;aACT;iBAAM;gBACL,MAAM,MAAM,CAAA;gBACZ,eAAe;gBACf,MAAM,GAAG,EAAE,CAAA;aACZ;SACF;QAED,+CAA+C;QAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;YACrB,MAAM,MAAM,CAAA;SACb;IACH,CAAC;IAED,KAAK,CAAC,CAAC,oBAAoB,CACzB,mBAA0C;QAE1C,MAAM,oBAAoB,GACxB,IAAI,CAAC,6BAA6B,CAAC,iBAAiB,CAAC,mBAAmB,CAAC,CAAA;QAE3E,IAAI,MAAM,GAAG,EAAE,CAAA;QAEf,IAAI,KAAK,EAAE,MAAM,WAAW,IAAI,oBAAoB,EAAE;YACpD,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;YAE5D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,kCAAkC,CAAC,MAAM,CAAC,EAAE;gBACrE,yEAAyE;gBACzE,uCAAuC;gBACvC,SAAQ;aACT;iBAAM;gBACL,MAAM,MAAM,CAAA;gBACZ,eAAe;gBACf,MAAM,GAAG,EAAE,CAAA;aACZ;SACF;QAED,+CAA+C;QAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;YACrB,MAAM,MAAM,CAAA;SACb;IACH,CAAC;IAED,MAAM,CAAC,mBAAqC;QAC1C,OAAO,CAAC,GAAG,IAAI,CAAC,eAAe,CAAC,mBAAmB,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAChE,CAAC;;SA5NU,WAAW"}
package/esm/GptEncoding.test.js CHANGED
@@ -1,3 +1,5 @@
1
+ import fs from 'fs';
2
+ import path from 'path';
1
3
  import { GptEncoding } from './GptEncoding.js';
2
4
  import { encodingNames } from './mapping.js';
3
5
  import { resolveEncoding } from './resolveEncoding.js';
@@ -36,6 +38,8 @@ const results = {
36
38
  p50k_edit: sharedResults,
37
39
  r50k_base: sharedResults,
38
40
  };
41
+ // eslint-disable-next-line @typescript-eslint/no-use-before-define
42
+ const testPlans = loadTestPlans();
39
43
  describe.each(encodingNames)('%s', (encodingName) => {
40
44
  const encoding = GptEncoding.getEncodingApi(encodingName, resolveEncoding);
41
45
  const { decode, decodeGenerator, decodeAsyncGenerator, encode, isWithinTokenLimit, } = encoding;
@@ -114,5 +118,36 @@ describe.each(encodingNames)('%s', (encodingName) => {
114
118
  expect(isWithinTokenLimit(str, 300)).toBe(result[str].length);
115
119
  });
116
120
  });
121
+ describe('test plan', () => {
122
+ testPlans[encodingName].forEach(({ sample, encoded }) => {
123
+ it(`encodes ${sample}`, () => {
124
+ expect(encode(sample)).toEqual(encoded);
125
+ });
126
+ it(`decodes ${sample}`, () => {
127
+ expect(decode(encoded)).toEqual(sample);
128
+ });
129
+ });
130
+ });
117
131
  });
132
+ function loadTestPlans() {
133
+ const testPlanPath = path.join(__dirname, '../data/TestPlans.txt');
134
+ const testPlanData = fs.readFileSync(testPlanPath, 'utf8');
135
+ const tests = {
136
+ cl100k_base: [],
137
+ p50k_base: [],
138
+ p50k_edit: [],
139
+ r50k_base: [],
140
+ };
141
+ testPlanData.split('\n\n').forEach((testPlan) => {
142
+ const [encodingNameLine, sampleLine, encodedLine] = testPlan.split('\n');
143
+ if (!encodingNameLine || !sampleLine || !encodedLine)
144
+ return;
145
+ const encodingName = encodingNameLine.split(': ')[1];
146
+ tests[encodingName].push({
147
+ sample: sampleLine.split(': ').slice(1).join(': ') ?? '',
148
+ encoded: JSON.parse(encodedLine.split(': ')[1] ?? '[]'),
149
+ });
150
+ });
151
+ return tests;
152
+ }
118
153
  //# sourceMappingURL=GptEncoding.test.js.map
package/esm/GptEncoding.test.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"GptEncoding.test.js","sourceRoot":"","sources":["../src/GptEncoding.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAA;AAC9C,OAAO,EAAqB,aAAa,EAAE,MAAM,cAAc,CAAA;AAC/D,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAA;AAEtD,MAAM,aAAa,GAAG;IACpB,KAAK,EAAE,CAAC,GAAG,CAAC;IACZ,GAAG,EAAE,CAAC,GAAG,CAAC;IACV,mBAAmB,EAAE,CAAC,IAAK,EAAE,GAAG,EAAE,GAAG,EAAE,IAAK,CAAC;IAC7C,WAAW,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,KAAM,CAAC;IAC/B,mBAAmB,EAAE,CAAC,KAAM,EAAE,KAAM,EAAE,GAAG,EAAE,GAAG,EAAE,KAAM,EAAE,GAAG,EAAE,GAAG,CAAC;IACjE,uBAAuB,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,CAAC;IAClE,6CAA6C,EAAE;QAC7C,IAAK,EAAE,KAAM,EAAE,KAAM,EAAE,GAAG,EAAE,KAAM,EAAE,KAAM,EAAE,IAAK,EAAE,IAAK;KACzD;IACD,8CAA8C,EAAE;QAC9C,KAAM,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,IAAK,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE;QAC1E,EAAE,EAAE,EAAE,EAAE,EAAE;KACX;CACF,CAAA;AAED,MAAM,OAAO,GAAG;IACd,WAAW,EAAE;QACX,KAAK,EAAE,CAAC,GAAG,CAAC;QACZ,GAAG,EAAE,CAAC,GAAG,CAAC;QACV,mBAAmB,EAAE,CAAC,IAAK,EAAE,GAAG,EAAE,IAAK,EAAE,IAAK,CAAC;QAC/C,WAAW,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,KAAM,CAAC;QAC/B,mBAAmB,EAAE,CAAC,KAAM,EAAE,KAAM,EAAE,GAAG,EAAE,IAAK,EAAE,KAAM,EAAE,GAAG,EAAE,GAAG,CAAC;QACnE,uBAAuB,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,CAAC;QAClE,6CAA6C,EAAE;YAC7C,IAAK,EAAE,IAAK,EAAE,GAAG,EAAE,KAAM,EAAE,GAAG,EAAE,IAAK;SACtC;QACD,8CAA8C,EAAE;YAC9C,KAAM,EAAE,EAAE,EAAE,GAAG,EAAE,IAAK,EAAE,GAAG,EAAE,IAAK,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,KAAM,EAAE,EAAE,EAAE,GAAG;YACtE,IAAK,EAAE,KAAM;SACd;KACF;IACD,SAAS,EAAE,aAAa;IACxB,SAAS,EAAE,aAAa;IACxB,SAAS,EAAE,aAAa;CACe,CAAA;AAEzC,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,IAAI,EAAE,CAAC,YAA0B,EAAE,EAAE;IAChE,MAAM,QAAQ,GAAG,WAAW,CAAC,cAAc,CAAC,YAAY,EAAE,eAAe,CAAC,CAAA;IAE1E,MAAM,EACJ,MAAM,EACN,eAAe,EACf,oBAAoB,EACpB,MAAM,EACN,kBAAkB,GACnB,GAAG,QAAQ,CAAA;IAEZ,MAAM,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAAA;IAEpC,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;QACnC,EAAE,CAAC,cAAc,EAAE,GAAG,EAAE;YACtB,MAAM,GAAG,GAAG,EAAE,CAAA;YACd,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;YAC/B,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YAC1C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC5C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YACf,MAAM,GAAG,GAAG,GAAG,CAAA;YACf,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YACzC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YAC1C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAChD,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;YACb,MAAM,GAAG,GAAG,IAAI,CAAA;YAChB,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;YACvC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QAC1C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,aAAa,EAAE,GAAG,EAAE;YACrB,MAAM,GAAG,GAAG,mBAAmB,CAAA;YAC/B,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YACxC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAC9C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;QAC7D,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,kBAAkB,EAAE,GAAG,EAAE;YAC1B,MAAM,GAAG,GAAG,aAAa,CAAA;YACzB,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;YAC/C,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAA;QACpE,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE;YAChB,MAAM,GAAG,GAAG,mBAAmB,CAAA;YAC/B,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YACxC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAC9C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;QAC/D,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;YAC7C,MAAM,GAAG,GAAG,mBAAmB,CAAA;YAC/B,MAAM,SAAS,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YAC9C,MAAM,CAAC,uBAAuB,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;gBAC/C,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAC5C,CAAC,CAAC,CAAA;QACJ,CAAC,CAAC,CAAA;QAEF,KAAK,SAAS,CAAC,CAAC,wBAAwB;YACtC,MAAM,GAAG,GAAG,mBAAmB,CAAA;YAC/B,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,GAAG,CAAC,EAAE;gBAC/B,4CAA4C;gBAC5C,MAAM,MAAM,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;aACnC;QACH,CAAC;QAED,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;YACzD,MAAM,SAAS,GAAG,oBAAoB,CAAC,wBAAwB,EAAE,CAAC,CAAA;YAClE,MAAM,OAAO,GAAG,CAAC,GAAG,MAAM,CAAC,uBAAuB,CAAC,CAAA;YACnD,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,SAAS,EAAE;gBACnC,MAAM,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;aACvC;QACH,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,sBAAsB,EAAE,GAAG,EAAE;YAC9B,MAAM,GAAG,GAAG,6CAA6C,CAAA;YAEzD,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YACxC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QAC1C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,kBAAkB,EAAE,GAAG,EAAE;YAC1B,MAAM,GAAG,GAAG,8CAA8C,CAAA;YAC1D,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YAC9C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YACnE,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;QAC/D,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA"}
1
+ {"version":3,"file":"GptEncoding.test.js","sourceRoot":"","sources":["../src/GptEncoding.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,IAAI,CAAA;AACnB,OAAO,IAAI,MAAM,MAAM,CAAA;AACvB,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAA;AAC9C,OAAO,EAAqB,aAAa,EAAE,MAAM,cAAc,CAAA;AAC/D,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAA;AAEtD,MAAM,aAAa,GAAG;IACpB,KAAK,EAAE,CAAC,GAAG,CAAC;IACZ,GAAG,EAAE,CAAC,GAAG,CAAC;IACV,mBAAmB,EAAE,CAAC,IAAK,EAAE,GAAG,EAAE,GAAG,EAAE,IAAK,CAAC;IAC7C,WAAW,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,KAAM,CAAC;IAC/B,mBAAmB,EAAE,CAAC,KAAM,EAAE,KAAM,EAAE,GAAG,EAAE,GAAG,EAAE,KAAM,EAAE,GAAG,EAAE,GAAG,CAAC;IACjE,uBAAuB,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,CAAC;IAClE,6CAA6C,EAAE;QAC7C,IAAK,EAAE,KAAM,EAAE,KAAM,EAAE,GAAG,EAAE,KAAM,EAAE,KAAM,EAAE,IAAK,EAAE,IAAK;KACzD;IACD,8CAA8C,EAAE;QAC9C,KAAM,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,IAAK,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE;QAC1E,EAAE,EAAE,EAAE,EAAE,EAAE;KACX;CACF,CAAA;AAED,MAAM,OAAO,GAAG;IACd,WAAW,EAAE;QACX,KAAK,EAAE,CAAC,GAAG,CAAC;QACZ,GAAG,EAAE,CAAC,GAAG,CAAC;QACV,mBAAmB,EAAE,CAAC,IAAK,EAAE,GAAG,EAAE,IAAK,EAAE,IAAK,CAAC;QAC/C,WAAW,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,KAAM,CAAC;QAC/B,mBAAmB,EAAE,CAAC,KAAM,EAAE,KAAM,EAAE,GAAG,EAAE,IAAK,EAAE,KAAM,EAAE,GAAG,EAAE,GAAG,CAAC;QACnE,uBAAuB,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,CAAC;QAClE,6CAA6C,EAAE;YAC7C,IAAK,EAAE,IAAK,EAAE,GAAG,EAAE,KAAM,EAAE,GAAG,EAAE,IAAK;SACtC;QACD,8CAA8C,EAAE;YAC9C,KAAM,EAAE,EAAE,EAAE,GAAG,EAAE,IAAK,EAAE,GAAG,EAAE,IAAK,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,KAAM,EAAE,EAAE,EAAE,GAAG;YACtE,IAAK,EAAE,KAAM;SACd;KACF;IACD,SAAS,EAAE,aAAa;IACxB,SAAS,EAAE,aAAa;IACxB,SAAS,EAAE,aAAa;CACe,CAAA;AAEzC,mEAAmE;AACnE,MAAM,SAAS,GAAG,aAAa,EAAE,CAAA;AAEjC,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,IAAI,EAAE,CAAC,YAA0B,EAAE,EAAE;IAChE,MAAM,QAAQ,GAAG,WAAW,CAAC,cAAc,CAAC,YAAY,EAAE,eAAe,CAAC,CAAA;IAE1E,MAAM,EACJ,MAAM,EACN,eAAe,EACf,oBAAoB,EACpB,MAAM,EACN,kBAAkB,GACnB,GAAG,QAAQ,CAAA;IAEZ,MAAM,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAAA;IAEpC,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;QACnC,EAAE,CAAC,cAAc,EAAE,GAAG,EAAE;YACtB,MAAM,GAAG,GAAG,EAAE,CAAA;YACd,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;YAC/B,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YAC1C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC5C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YACf,MAAM,GAAG,GAAG,GAAG,CAAA;YACf,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YACzC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YAC1C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAChD,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;YACb,MAAM,GAAG,GAAG,IAAI,CAAA;YAChB,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;YACvC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QAC1C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,aAAa,EAAE,GAAG,EAAE;YACrB,MAAM,GAAG,GAAG,mBAAmB,CAAA;YAC/B,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YACxC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAC9C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;QAC7D,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,kBAAkB,EAAE,GAAG,EAAE;YAC1B,MAAM,GAAG,GAAG,aAAa,CAAA;YACzB,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;YAC/C,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAA;QACpE,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE;YAChB,MAAM,GAAG,GAAG,mBAAmB,CAAA;YAC/B,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YACxC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAC9C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;QAC/D,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;YAC7C,MAAM,GAAG,GAAG,mBAAmB,CAAA;YAC/B,MAAM,SAAS,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YAC9C,MAAM,CAAC,uBAAuB,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;gBAC/C,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAC5C,CAAC,CAAC,CAAA;QACJ,CAAC,CAAC,CAAA;QAEF,KAAK,SAAS,CAAC,CAAC,wBAAwB;YACtC,MAAM,GAAG,GAAG,mBAAmB,CAAA;YAC/B,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,GAAG,CAAC,EAAE;gBAC/B,4CAA4C;gBAC5C,MAAM,MAAM,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;aACnC;QACH,CAAC;QAED,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;YACzD,MAAM,SAAS,GAAG,oBAAoB,CAAC,wBAAwB,EAAE,CAAC,CAAA;YAClE,MAAM,OAAO,GAAG,CAAC,GAAG,MAAM,CAAC,uBAAuB,CAAC,CAAA;YACnD,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,SAAS,EAAE;gBACnC,MAAM,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;aACvC;QACH,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,sBAAsB,EAAE,GAAG,EAAE;YAC9B,MAAM,GAAG,GAAG,6CAA6C,CAAA;YAEzD,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YACxC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QAC1C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,kBAAkB,EAAE,GAAG,EAAE;YAC1B,MAAM,GAAG,GAAG,8CAA8C,CAAA;YAC1D,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YAC9C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YACnE,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;QAC/D,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,WAAW,EAAE,GAAG,EAAE;QACzB,SAAS,CAAC,YAAY,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,EAAE;YACtD,EAAE,CAAC,WAAW,MAAM,EAAE,EAAE,GAAG,EAAE;gBAC3B,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;YACzC,CAAC,CAAC,CAAA;YACF,EAAE,CAAC,WAAW,MAAM,EAAE,EAAE,GAAG,EAAE;gBAC3B,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;YACzC,CAAC,CAAC,CAAA;QACJ,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA;AAEF,SAAS,aAAa;IACpB,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,uBAAuB,CAAC,CAAA;IAClE,MAAM,YAAY,GAAG,EAAE,CAAC,YAAY,CAAC,YAAY,EAAE,MAAM,CAAC,CAAA;IAC1D,MAAM,KAAK,GAGP;QACF,WAAW,EAAE,EAAE;QACf,SAAS,EAAE,EAAE;QACb,SAAS,EAAE,EAAE;QACb,SAAS,EAAE,EAAE;KACd,CAAA;IACD,YAAY,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;QAC9C,MAAM,CAAC,gBAAgB,EAAE,UAAU,EAAE,WAAW,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QACxE,IAAI,CAAC,gBAAgB,IAAI,CAAC,UAAU,IAAI,CAAC,WAAW;YAAE,OAAM;QAC5D,MAAM,YAAY,GAAG,gBAAgB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAiB,CAAA;QACpE,KAAK,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC;YACvB,MAAM,EAAE,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;YACxD,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;SACxD,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IACF,OAAO,KAAK,CAAA;AACd,CAAC"}
package/esm/encoding/cl100k_base.d.ts CHANGED
@@ -1,2 +1,6 @@
1
+ import { GptEncoding } from '../GptEncoding.js';
2
+ export * from '../specialTokens.js';
3
+ declare const api: GptEncoding;
1
4
  declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void, unknown>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, unknown>, encode: (lineToEncode: string, allowedSpecial?: Set<string>, disallowedSpecial?: Set<string>) => number[], encodeGenerator: (lineToEncode: string, allowedSpecial?: Set<string>, disallowedSpecial?: Set<string>) => Generator<number[], number, unknown>, isWithinTokenLimit: (text: string, tokenLimit: number) => number | false;
2
5
  export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, };
6
+ export default api;
package/esm/encoding/cl100k_base.js CHANGED
@@ -2,7 +2,10 @@
2
2
  import { convertTokenBytePairEncodingFromTuples } from '../convertTokenBytePairEncodingFromTuples.js';
3
3
  import encoder from '../encodings/cl100k_base.js';
4
4
  import { GptEncoding } from '../GptEncoding.js';
5
+ export * from '../specialTokens.js';
5
6
  const api = GptEncoding.getEncodingApi('cl100k_base', () => convertTokenBytePairEncodingFromTuples(encoder));
6
7
  const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, } = api;
7
8
  export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, };
9
+ // eslint-disable-next-line import/no-default-export
10
+ export default api;
8
11
  //# sourceMappingURL=cl100k_base.js.map
package/esm/encoding/cl100k_base.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"cl100k_base.js","sourceRoot":"","sources":["../../src/encoding/cl100k_base.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,EAAE,sCAAsC,EAAE,MAAM,8CAA8C,CAAA;AACrG,OAAO,OAAO,MAAM,6BAA6B,CAAA;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,MAAM,GAAG,GAAG,WAAW,CAAC,cAAc,CAAC,aAAa,EAAE,GAAG,EAAE,CACzD,sCAAsC,CAAC,OAAO,CAAC,CAChD,CAAA;AACD,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,GACnB,GAAG,GAAG,CAAA;AACP,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,GACnB,CAAA"}
1
+ {"version":3,"file":"cl100k_base.js","sourceRoot":"","sources":["../../src/encoding/cl100k_base.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,EAAE,sCAAsC,EAAE,MAAM,8CAA8C,CAAA;AACrG,OAAO,OAAO,MAAM,6BAA6B,CAAA;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,qBAAqB,CAAA;AAEnC,MAAM,GAAG,GAAG,WAAW,CAAC,cAAc,CAAC,aAAa,EAAE,GAAG,EAAE,CACzD,sCAAsC,CAAC,OAAO,CAAC,CAChD,CAAA;AACD,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,GACnB,GAAG,GAAG,CAAA;AACP,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,GACnB,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}
package/esm/encoding/p50k_base.d.ts CHANGED
@@ -1,2 +1,6 @@
1
+ import { GptEncoding } from '../GptEncoding.js';
2
+ export * from '../specialTokens.js';
3
+ declare const api: GptEncoding;
1
4
  declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void, unknown>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, unknown>, encode: (lineToEncode: string, allowedSpecial?: Set<string>, disallowedSpecial?: Set<string>) => number[], encodeGenerator: (lineToEncode: string, allowedSpecial?: Set<string>, disallowedSpecial?: Set<string>) => Generator<number[], number, unknown>, isWithinTokenLimit: (text: string, tokenLimit: number) => number | false;
2
5
  export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, };
6
+ export default api;
package/esm/encoding/p50k_base.js CHANGED
@@ -2,7 +2,10 @@
2
2
  import { convertTokenBytePairEncodingFromTuples } from '../convertTokenBytePairEncodingFromTuples.js';
3
3
  import encoder from '../encodings/p50k_base.js';
4
4
  import { GptEncoding } from '../GptEncoding.js';
5
+ export * from '../specialTokens.js';
5
6
  const api = GptEncoding.getEncodingApi('p50k_base', () => convertTokenBytePairEncodingFromTuples(encoder));
6
7
  const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, } = api;
7
8
  export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, };
9
+ // eslint-disable-next-line import/no-default-export
10
+ export default api;
8
11
  //# sourceMappingURL=p50k_base.js.map
package/esm/encoding/p50k_base.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"p50k_base.js","sourceRoot":"","sources":["../../src/encoding/p50k_base.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,EAAE,sCAAsC,EAAE,MAAM,8CAA8C,CAAA;AACrG,OAAO,OAAO,MAAM,2BAA2B,CAAA;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,MAAM,GAAG,GAAG,WAAW,CAAC,cAAc,CAAC,WAAW,EAAE,GAAG,EAAE,CACvD,sCAAsC,CAAC,OAAO,CAAC,CAChD,CAAA;AACD,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,GACnB,GAAG,GAAG,CAAA;AACP,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,GACnB,CAAA"}
1
+ {"version":3,"file":"p50k_base.js","sourceRoot":"","sources":["../../src/encoding/p50k_base.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,EAAE,sCAAsC,EAAE,MAAM,8CAA8C,CAAA;AACrG,OAAO,OAAO,MAAM,2BAA2B,CAAA;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,qBAAqB,CAAA;AAEnC,MAAM,GAAG,GAAG,WAAW,CAAC,cAAc,CAAC,WAAW,EAAE,GAAG,EAAE,CACvD,sCAAsC,CAAC,OAAO,CAAC,CAChD,CAAA;AACD,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,GACnB,GAAG,GAAG,CAAA;AACP,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,GACnB,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}
package/esm/encoding/p50k_edit.d.ts CHANGED
@@ -1,2 +1,6 @@
1
+ import { GptEncoding } from '../GptEncoding.js';
2
+ export * from '../specialTokens.js';
3
+ declare const api: GptEncoding;
1
4
  declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void, unknown>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, unknown>, encode: (lineToEncode: string, allowedSpecial?: Set<string>, disallowedSpecial?: Set<string>) => number[], encodeGenerator: (lineToEncode: string, allowedSpecial?: Set<string>, disallowedSpecial?: Set<string>) => Generator<number[], number, unknown>, isWithinTokenLimit: (text: string, tokenLimit: number) => number | false;
2
5
  export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, };
6
+ export default api;
package/esm/encoding/p50k_edit.js CHANGED
@@ -2,7 +2,10 @@
2
2
  import { convertTokenBytePairEncodingFromTuples } from '../convertTokenBytePairEncodingFromTuples.js';
3
3
  import encoder from '../encodings/p50k_base.js';
4
4
  import { GptEncoding } from '../GptEncoding.js';
5
+ export * from '../specialTokens.js';
5
6
  const api = GptEncoding.getEncodingApi('p50k_edit', () => convertTokenBytePairEncodingFromTuples(encoder));
6
7
  const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, } = api;
7
8
  export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, };
9
+ // eslint-disable-next-line import/no-default-export
10
+ export default api;
8
11
  //# sourceMappingURL=p50k_edit.js.map
package/esm/encoding/p50k_edit.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"p50k_edit.js","sourceRoot":"","sources":["../../src/encoding/p50k_edit.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,EAAE,sCAAsC,EAAE,MAAM,8CAA8C,CAAA;AACrG,OAAO,OAAO,MAAM,2BAA2B,CAAA;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,MAAM,GAAG,GAAG,WAAW,CAAC,cAAc,CAAC,WAAW,EAAE,GAAG,EAAE,CACvD,sCAAsC,CAAC,OAAO,CAAC,CAChD,CAAA;AACD,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,GACnB,GAAG,GAAG,CAAA;AACP,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,GACnB,CAAA"}
1
+ {"version":3,"file":"p50k_edit.js","sourceRoot":"","sources":["../../src/encoding/p50k_edit.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,EAAE,sCAAsC,EAAE,MAAM,8CAA8C,CAAA;AACrG,OAAO,OAAO,MAAM,2BAA2B,CAAA;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,qBAAqB,CAAA;AAEnC,MAAM,GAAG,GAAG,WAAW,CAAC,cAAc,CAAC,WAAW,EAAE,GAAG,EAAE,CACvD,sCAAsC,CAAC,OAAO,CAAC,CAChD,CAAA;AACD,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,GACnB,GAAG,GAAG,CAAA;AACP,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,GACnB,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}
package/esm/encoding/r50k_base.d.ts CHANGED
@@ -1,2 +1,6 @@
1
+ import { GptEncoding } from '../GptEncoding.js';
2
+ export * from '../specialTokens.js';
3
+ declare const api: GptEncoding;
1
4
  declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void, unknown>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, unknown>, encode: (lineToEncode: string, allowedSpecial?: Set<string>, disallowedSpecial?: Set<string>) => number[], encodeGenerator: (lineToEncode: string, allowedSpecial?: Set<string>, disallowedSpecial?: Set<string>) => Generator<number[], number, unknown>, isWithinTokenLimit: (text: string, tokenLimit: number) => number | false;
2
5
  export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, };
6
+ export default api;
package/esm/encoding/r50k_base.js CHANGED
@@ -2,7 +2,10 @@
2
2
  import { convertTokenBytePairEncodingFromTuples } from '../convertTokenBytePairEncodingFromTuples.js';
3
3
  import encoder from '../encodings/r50k_base.js';
4
4
  import { GptEncoding } from '../GptEncoding.js';
5
+ export * from '../specialTokens.js';
5
6
  const api = GptEncoding.getEncodingApi('r50k_base', () => convertTokenBytePairEncodingFromTuples(encoder));
6
7
  const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, } = api;
7
8
  export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, };
9
+ // eslint-disable-next-line import/no-default-export
10
+ export default api;
8
11
  //# sourceMappingURL=r50k_base.js.map
package/esm/encoding/r50k_base.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"r50k_base.js","sourceRoot":"","sources":["../../src/encoding/r50k_base.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,EAAE,sCAAsC,EAAE,MAAM,8CAA8C,CAAA;AACrG,OAAO,OAAO,MAAM,2BAA2B,CAAA;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,MAAM,GAAG,GAAG,WAAW,CAAC,cAAc,CAAC,WAAW,EAAE,GAAG,EAAE,CACvD,sCAAsC,CAAC,OAAO,CAAC,CAChD,CAAA;AACD,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,GACnB,GAAG,GAAG,CAAA;AACP,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,GACnB,CAAA"}
1
+ {"version":3,"file":"r50k_base.js","sourceRoot":"","sources":["../../src/encoding/r50k_base.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,EAAE,sCAAsC,EAAE,MAAM,8CAA8C,CAAA;AACrG,OAAO,OAAO,MAAM,2BAA2B,CAAA;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,qBAAqB,CAAA;AAEnC,MAAM,GAAG,GAAG,WAAW,CAAC,cAAc,CAAC,WAAW,EAAE,GAAG,EAAE,CACvD,sCAAsC,CAAC,OAAO,CAAC,CAChD,CAAA;AACD,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,GACnB,GAAG,GAAG,CAAA;AACP,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,GACnB,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}
package/esm/modelParams.js CHANGED
@@ -1,10 +1,6 @@
1
1
  /* eslint-disable no-magic-numbers */
2
2
  import { EncoderMap } from './EncoderMap.js';
3
- const EndOfText = '<|endoftext|>';
4
- const FimPrefix = '<|fim_prefix|>';
5
- const FimMiddle = '<|fim_middle|>';
6
- const FimSuffix = '<|fim_suffix|>';
7
- const EndOfPrompt = '<|endofprompt|>';
3
+ import { EndOfPrompt, EndOfText, FimMiddle, FimPrefix, FimSuffix, } from './specialTokens.js';
8
4
  const tokenSplitRegex = /'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+/gu;
9
5
  function R50KBase(mergeableBytePairRanks) {
10
6
  return {
package/esm/modelParams.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"modelParams.js","sourceRoot":"","sources":["../src/modelParams.ts"],"names":[],"mappings":"AAAA,qCAAqC;AACrC,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAoB5C,MAAM,SAAS,GAAG,eAAe,CAAA;AACjC,MAAM,SAAS,GAAG,gBAAgB,CAAA;AAClC,MAAM,SAAS,GAAG,gBAAgB,CAAA;AAClC,MAAM,SAAS,GAAG,gBAAgB,CAAA;AAClC,MAAM,WAAW,GAAG,iBAAiB,CAAA;AAErC,MAAM,eAAe,GACnB,8EAA8E,CAAA;AAEhF,SAAS,QAAQ,CAAC,sBAAkC;IAClD,OAAO;QACL,sBAAsB,EAAE,KAAM;QAC9B,eAAe;QACf,sBAAsB;QACtB,mBAAmB,EAAE,IAAI,GAAG,CAAiB,CAAC,CAAC,SAAS,EAAE,KAAM,CAAC,CAAC,CAAC;KACpE,CAAA;AACH,CAAC;AAED,SAAS,QAAQ,CAAC,sBAAkC;IAClD,OAAO;QACL,sBAAsB,EAAE,KAAM;QAC9B,eAAe;QACf,sBAAsB;QACtB,mBAAmB,EAAE,IAAI,GAAG,CAAiB,CAAC,CAAC,SAAS,EAAE,KAAM,CAAC,CAAC,CAAC;KACpE,CAAA;AACH,CAAC;AAED,SAAS,QAAQ,CAAC,sBAAkC;IAClD,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAiB;QAClD,CAAC,SAAS,EAAE,KAAM,CAAC;QACnB,CAAC,SAAS,EAAE,KAAM,CAAC;QACnB,CAAC,SAAS,EAAE,KAAM,CAAC;QACnB,CAAC,SAAS,EAAE,KAAM,CAAC;KACpB,CAAC,CAAA;IAEF,OAAO;QACL,eAAe;QACf,sBAAsB;QACtB,mBAAmB;KACpB,CAAA;AACH,CAAC;AAED,SAAS,UAAU,CAAC,sBAAkC;IACpD,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAiB;QAClD,CAAC,SAAS,EAAE,MAAO,CAAC;QACpB,CAAC,SAAS,EAAE,MAAO,CAAC;QACpB,CAAC,SAAS,EAAE,MAAO,CAAC;QACpB,CAAC,SAAS,EAAE,MAAO,CAAC;QACpB,CAAC,WAAW,EAAE,MAAO,CAAC;KACvB,CAAC,CAAA;IAEF,OAAO;QACL,eAAe,EACb,uHAAuH;QACzH,sBAAsB;QACtB,mBAAmB;KACpB,CAAA;AACH,CAAC;AAOD,MAAM,UAAU,cAAc,CAC5B,YAA0B,EAC1B,iBAAsC;IAEtC,MAAM,sBAAsB,GAAG,iBAAiB,CAAC,YAAY,CAAC,CAAA;IAC9D,QAAQ,YAAY,CAAC,WAAW,EAAE,EAAE;QAClC,KAAK,WAAW;YACd,OAAO,QAAQ,CAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,WAAW;YACd,OAAO,QAAQ,CAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,WAAW;YACd,OAAO,QAAQ,CAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,aAAa;YAChB,OAAO,UAAU,CAAC,sBAAsB,CAAC,CAAA;QAE3C;YACE,MAAM,IAAI,KAAK,CAAC,0BAA0B,YAAY,EAAE,CAAC,CAAA;KAC5D;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,YAA0B,EAC1B,iBAA2C;IAE3C,MAAM,sBAAsB,GAAG,MAAM,iBAAiB,CAAC,YAAY,CAAC,CAAA;IACpE,OAAO,cAAc,CAAC,YAAY,EAAE,GAAG,EAAE,CAAC,sBAAsB,CAAC,CAAA;AACnE,CAAC"}
1
+ {"version":3,"file":"modelParams.js","sourceRoot":"","sources":["../src/modelParams.ts"],"names":[],"mappings":"AAAA,qCAAqC;AACrC,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAE5C,OAAO,EACL,WAAW,EACX,SAAS,EACT,SAAS,EACT,SAAS,EACT,SAAS,GACV,MAAM,oBAAoB,CAAA;AAmB3B,MAAM,eAAe,GACnB,8EAA8E,CAAA;AAEhF,SAAS,QAAQ,CAAC,sBAAkC;IAClD,OAAO;QACL,sBAAsB,EAAE,KAAM;QAC9B,eAAe;QACf,sBAAsB;QACtB,mBAAmB,EAAE,IAAI,GAAG,CAAiB,CAAC,CAAC,SAAS,EAAE,KAAM,CAAC,CAAC,CAAC;KACpE,CAAA;AACH,CAAC;AAED,SAAS,QAAQ,CAAC,sBAAkC;IAClD,OAAO;QACL,sBAAsB,EAAE,KAAM;QAC9B,eAAe;QACf,sBAAsB;QACtB,mBAAmB,EAAE,IAAI,GAAG,CAAiB,CAAC,CAAC,SAAS,EAAE,KAAM,CAAC,CAAC,CAAC;KACpE,CAAA;AACH,CAAC;AAED,SAAS,QAAQ,CAAC,sBAAkC;IAClD,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAiB;QAClD,CAAC,SAAS,EAAE,KAAM,CAAC;QACnB,CAAC,SAAS,EAAE,KAAM,CAAC;QACnB,CAAC,SAAS,EAAE,KAAM,CAAC;QACnB,CAAC,SAAS,EAAE,KAAM,CAAC;KACpB,CAAC,CAAA;IAEF,OAAO;QACL,eAAe;QACf,sBAAsB;QACtB,mBAAmB;KACpB,CAAA;AACH,CAAC;AAED,SAAS,UAAU,CAAC,sBAAkC;IACpD,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAiB;QAClD,CAAC,SAAS,EAAE,MAAO,CAAC;QACpB,CAAC,SAAS,EAAE,MAAO,CAAC;QACpB,CAAC,SAAS,EAAE,MAAO,CAAC;QACpB,CAAC,SAAS,EAAE,MAAO,CAAC;QACpB,CAAC,WAAW,EAAE,MAAO,CAAC;KACvB,CAAC,CAAA;IAEF,OAAO;QACL,eAAe,EACb,uHAAuH;QACzH,sBAAsB;QACtB,mBAAmB;KACpB,CAAA;AACH,CAAC;AAOD,MAAM,UAAU,cAAc,CAC5B,YAA0B,EAC1B,iBAAsC;IAEtC,MAAM,sBAAsB,GAAG,iBAAiB,CAAC,YAAY,CAAC,CAAA;IAC9D,QAAQ,YAAY,CAAC,WAAW,EAAE,EAAE;QAClC,KAAK,WAAW;YACd,OAAO,QAAQ,CAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,WAAW;YACd,OAAO,QAAQ,CAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,WAAW;YACd,OAAO,QAAQ,CAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,aAAa;YAChB,OAAO,UAAU,CAAC,sBAAsB,CAAC,CAAA;QAE3C;YACE,MAAM,IAAI,KAAK,CAAC,0BAA0B,YAAY,EAAE,CAAC,CAAA;KAC5D;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,YAA0B,EAC1B,iBAA2C;IAE3C,MAAM,sBAAsB,GAAG,MAAM,iBAAiB,CAAC,YAAY,CAAC,CAAA;IACpE,OAAO,cAAc,CAAC,YAAY,EAAE,GAAG,EAAE,CAAC,sBAAsB,CAAC,CAAA;AACnE,CAAC"}
package/esm/specialTokens.d.ts ADDED
@@ -0,0 +1,5 @@
1
+ export declare const EndOfText = "<|endoftext|>";
2
+ export declare const FimPrefix = "<|fim_prefix|>";
3
+ export declare const FimMiddle = "<|fim_middle|>";
4
+ export declare const FimSuffix = "<|fim_suffix|>";
5
+ export declare const EndOfPrompt = "<|endofprompt|>";
package/esm/specialTokens.js ADDED
@@ -0,0 +1,6 @@
1
+ export const EndOfText = '<|endoftext|>';
2
+ export const FimPrefix = '<|fim_prefix|>';
3
+ export const FimMiddle = '<|fim_middle|>';
4
+ export const FimSuffix = '<|fim_suffix|>';
5
+ export const EndOfPrompt = '<|endofprompt|>';
6
+ //# sourceMappingURL=specialTokens.js.map
package/esm/specialTokens.js.map ADDED
@@ -0,0 +1 @@
1
+ {"version":3,"file":"specialTokens.js","sourceRoot":"","sources":["../src/specialTokens.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,SAAS,GAAG,eAAe,CAAA;AACxC,MAAM,CAAC,MAAM,SAAS,GAAG,gBAAgB,CAAA;AACzC,MAAM,CAAC,MAAM,SAAS,GAAG,gBAAgB,CAAA;AACzC,MAAM,CAAC,MAAM,SAAS,GAAG,gBAAgB,CAAA;AACzC,MAAM,CAAC,MAAM,WAAW,GAAG,iBAAiB,CAAA"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "gpt-tokenizer",
3
- "version": "2.0.0-beta.1",
3
+ "version": "2.0.0-beta.2",
4
4
  "description": "BPE Encoder Decoder for GPT-2 / GPT-3",
5
5
  "keywords": [
6
6
  "BPE",
@@ -76,10 +76,10 @@
76
76
  "build:cjs": "yarn rrun tsc --outDir cjs --module commonjs --target es2022 --project tsconfig-cjs.json",
77
77
  "build:esm": "yarn rrun tsc --outDir esm --module esnext --target es2022 && echo '{\"name\": \"gpt-tokenizer\", \"type\": \"module\"}' > ./esm/package.json",
78
78
  "build:umd": "yarn build:umd:cl100k_base && yarn build:umd:p50k_base && yarn build:umd:p50k_edit && yarn build:umd:r50k_base",
79
- "build:umd:cl100k_base": "beemo webpack --entry='./src/main.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPT3Encoder' --env 'export=default' --env 'filename=cl100k_base.js'",
80
- "build:umd:p50k_base": "beemo webpack --entry='./src/encoding/p50k_base.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPT3Encoder' --env 'export=default' --env 'filename=p50k_base.js'",
81
- "build:umd:p50k_edit": "beemo webpack --entry='./src/encoding/p50k_edit.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPT3Encoder' --env 'export=default' --env 'filename=p50k_edit.js'",
82
- "build:umd:r50k_base": "beemo webpack --entry='./src/encoding/r50k_base.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPT3Encoder' --env 'export=default' --env 'filename=r50k_base.js'",
79
+ "build:umd:cl100k_base": "beemo webpack --entry='./src/main.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPTTokenizer' --env 'export=default' --env 'filename=cl100k_base.js'",
80
+ "build:umd:p50k_base": "beemo webpack --entry='./src/encoding/p50k_base.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPTTokenizer' --env 'export=default' --env 'filename=p50k_base.js'",
81
+ "build:umd:p50k_edit": "beemo webpack --entry='./src/encoding/p50k_edit.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPTTokenizer' --env 'export=default' --env 'filename=p50k_edit.js'",
82
+ "build:umd:r50k_base": "beemo webpack --entry='./src/encoding/r50k_base.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPTTokenizer' --env 'export=default' --env 'filename=r50k_base.js'",
83
83
  "clean": "git clean -dfX --exclude=node_modules src && beemo typescript:sync-project-refs",
84
84
  "format": "yarn rrun prettier --write \"./{src,tests,.config}/**/!(*.d).{.js,jsx,ts,tsx,json,md}\"",
85
85
  "postinstallDev": "yarn prepare",
package/src/GptEncoding.test.ts CHANGED
@@ -1,3 +1,5 @@
1
+ import fs from 'fs'
2
+ import path from 'path'
1
3
  import { GptEncoding } from './GptEncoding.js'
2
4
  import { type EncodingName, encodingNames } from './mapping.js'
3
5
  import { resolveEncoding } from './resolveEncoding.js'
@@ -39,6 +41,9 @@ const results = {
39
41
  r50k_base: sharedResults,
40
42
  } satisfies Record<EncodingName, unknown>
41
43
 
44
+ // eslint-disable-next-line @typescript-eslint/no-use-before-define
45
+ const testPlans = loadTestPlans()
46
+
42
47
  describe.each(encodingNames)('%s', (encodingName: EncodingName) => {
43
48
  const encoding = GptEncoding.getEncodingApi(encodingName, resolveEncoding)
44
49
 
@@ -137,4 +142,39 @@ describe.each(encodingNames)('%s', (encodingName: EncodingName) => {
137
142
  expect(isWithinTokenLimit(str, 300)).toBe(result[str].length)
138
143
  })
139
144
  })
145
+
146
+ describe('test plan', () => {
147
+ testPlans[encodingName].forEach(({ sample, encoded }) => {
148
+ it(`encodes ${sample}`, () => {
149
+ expect(encode(sample)).toEqual(encoded)
150
+ })
151
+ it(`decodes ${sample}`, () => {
152
+ expect(decode(encoded)).toEqual(sample)
153
+ })
154
+ })
155
+ })
140
156
  })
157
+
158
+ function loadTestPlans() {
159
+ const testPlanPath = path.join(__dirname, '../data/TestPlans.txt')
160
+ const testPlanData = fs.readFileSync(testPlanPath, 'utf8')
161
+ const tests: Record<
162
+ EncodingName,
163
+ { sample: string; encoded: readonly number[] }[]
164
+ > = {
165
+ cl100k_base: [],
166
+ p50k_base: [],
167
+ p50k_edit: [],
168
+ r50k_base: [],
169
+ }
170
+ testPlanData.split('\n\n').forEach((testPlan) => {
171
+ const [encodingNameLine, sampleLine, encodedLine] = testPlan.split('\n')
172
+ if (!encodingNameLine || !sampleLine || !encodedLine) return
173
+ const encodingName = encodingNameLine.split(': ')[1] as EncodingName
174
+ tests[encodingName].push({
175
+ sample: sampleLine.split(': ').slice(1).join(': ') ?? '',
176
+ encoded: JSON.parse(encodedLine.split(': ')[1] ?? '[]'),
177
+ })
178
+ })
179
+ return tests
180
+ }
package/src/GptEncoding.ts CHANGED
@@ -13,11 +13,24 @@ import {
13
13
  getModelParams,
14
14
  getModelParamsAsync,
15
15
  } from './modelParams.js'
16
+ import {
17
+ EndOfPrompt,
18
+ EndOfText,
19
+ FimMiddle,
20
+ FimPrefix,
21
+ FimSuffix,
22
+ } from './specialTokens.js'
16
23
  import { endsWithIncompleteUtfPairSurrogate } from './utfUtil.js'
17
24
 
18
25
  export const DISALLOW_ALL_SPECIAL_TOKENS = 'all'
19
26
 
20
27
  export class GptEncoding {
28
+ static EndOfPrompt = EndOfPrompt
29
+ static EndOfText = EndOfText
30
+ static FimMiddle = FimMiddle
31
+ static FimPrefix = FimPrefix
32
+ static FimSuffix = FimSuffix
33
+
21
34
  decoder = new TextDecoder('utf8')
22
35
  private bytePairEncodingCoreProcessor: BytePairEncodingCore
23
36
  private specialTokenMapping: Map<string, number>
package/src/encoding/cl100k_base.ts CHANGED
@@ -3,6 +3,8 @@ import { convertTokenBytePairEncodingFromTuples } from '../convertTokenBytePairE
3
3
  import encoder from '../encodings/cl100k_base.js'
4
4
  import { GptEncoding } from '../GptEncoding.js'
5
5
 
6
+ export * from '../specialTokens.js'
7
+
6
8
  const api = GptEncoding.getEncodingApi('cl100k_base', () =>
7
9
  convertTokenBytePairEncodingFromTuples(encoder),
8
10
  )
@@ -22,3 +24,5 @@ export {
22
24
  encodeGenerator,
23
25
  isWithinTokenLimit,
24
26
  }
27
+ // eslint-disable-next-line import/no-default-export
28
+ export default api
package/src/encoding/p50k_base.ts CHANGED
@@ -3,6 +3,8 @@ import { convertTokenBytePairEncodingFromTuples } from '../convertTokenBytePairE
3
3
  import encoder from '../encodings/p50k_base.js'
4
4
  import { GptEncoding } from '../GptEncoding.js'
5
5
 
6
+ export * from '../specialTokens.js'
7
+
6
8
  const api = GptEncoding.getEncodingApi('p50k_base', () =>
7
9
  convertTokenBytePairEncodingFromTuples(encoder),
8
10
  )
@@ -22,3 +24,5 @@ export {
22
24
  encodeGenerator,
23
25
  isWithinTokenLimit,
24
26
  }
27
+ // eslint-disable-next-line import/no-default-export
28
+ export default api
package/src/encoding/p50k_edit.ts CHANGED
@@ -3,6 +3,8 @@ import { convertTokenBytePairEncodingFromTuples } from '../convertTokenBytePairE
3
3
  import encoder from '../encodings/p50k_base.js'
4
4
  import { GptEncoding } from '../GptEncoding.js'
5
5
 
6
+ export * from '../specialTokens.js'
7
+
6
8
  const api = GptEncoding.getEncodingApi('p50k_edit', () =>
7
9
  convertTokenBytePairEncodingFromTuples(encoder),
8
10
  )
@@ -22,3 +24,5 @@ export {
22
24
  encodeGenerator,
23
25
  isWithinTokenLimit,
24
26
  }
27
+ // eslint-disable-next-line import/no-default-export
28
+ export default api
package/src/encoding/r50k_base.ts CHANGED
@@ -3,6 +3,8 @@ import { convertTokenBytePairEncodingFromTuples } from '../convertTokenBytePairE
3
3
  import encoder from '../encodings/r50k_base.js'
4
4
  import { GptEncoding } from '../GptEncoding.js'
5
5
 
6
+ export * from '../specialTokens.js'
7
+
6
8
  const api = GptEncoding.getEncodingApi('r50k_base', () =>
7
9
  convertTokenBytePairEncodingFromTuples(encoder),
8
10
  )
@@ -22,3 +24,5 @@ export {
22
24
  encodeGenerator,
23
25
  isWithinTokenLimit,
24
26
  }
27
+ // eslint-disable-next-line import/no-default-export
28
+ export default api
package/src/modelParams.ts CHANGED
@@ -1,6 +1,13 @@
1
1
  /* eslint-disable no-magic-numbers */
2
2
  import { EncoderMap } from './EncoderMap.js'
3
3
  import type { EncodingName } from './mapping.js'
4
+ import {
5
+ EndOfPrompt,
6
+ EndOfText,
7
+ FimMiddle,
8
+ FimPrefix,
9
+ FimSuffix,
10
+ } from './specialTokens.js'
4
11
 
5
12
  export interface ModelParams {
6
13
  /**
@@ -19,12 +26,6 @@ export interface ModelParams {
19
26
  specialTokenMapping: Map<string, number>
20
27
  }
21
28
 
22
- const EndOfText = '<|endoftext|>'
23
- const FimPrefix = '<|fim_prefix|>'
24
- const FimMiddle = '<|fim_middle|>'
25
- const FimSuffix = '<|fim_suffix|>'
26
- const EndOfPrompt = '<|endofprompt|>'
27
-
28
29
  const tokenSplitRegex =
29
30
  /'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+/gu
30
31
 
package/src/specialTokens.ts ADDED
@@ -0,0 +1,5 @@
1
+ export const EndOfText = '<|endoftext|>'
2
+ export const FimPrefix = '<|fim_prefix|>'
3
+ export const FimMiddle = '<|fim_middle|>'
4
+ export const FimSuffix = '<|fim_suffix|>'
5
+ export const EndOfPrompt = '<|endofprompt|>'