mojentic 1.2.0 → 1.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/context/shared-working-memory.d.ts.map +1 -1
- package/dist/context/shared-working-memory.js +0 -1
- package/dist/context/shared-working-memory.js.map +1 -1
- package/dist/llm/chat-session.d.ts +2 -2
- package/dist/llm/chat-session.d.ts.map +1 -1
- package/dist/llm/chat-session.js.map +1 -1
- package/dist/llm/gateways/index.d.ts +1 -0
- package/dist/llm/gateways/index.d.ts.map +1 -1
- package/dist/llm/gateways/index.js +1 -0
- package/dist/llm/gateways/index.js.map +1 -1
- package/dist/llm/gateways/tokenizer.d.ts +29 -0
- package/dist/llm/gateways/tokenizer.d.ts.map +1 -0
- package/dist/llm/gateways/tokenizer.js +3 -0
- package/dist/llm/gateways/tokenizer.js.map +1 -0
- package/dist/llm/gateways/tokenizerGateway.d.ts +21 -13
- package/dist/llm/gateways/tokenizerGateway.d.ts.map +1 -1
- package/dist/llm/gateways/tokenizerGateway.js +45 -18
- package/dist/llm/gateways/tokenizerGateway.js.map +1 -1
- package/package.json +15 -11
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"shared-working-memory.d.ts","sourceRoot":"","sources":["../../src/context/shared-working-memory.ts"],"names":[],"mappings":"AAAA;;GAEG;
|
|
1
|
+
{"version":3,"file":"shared-working-memory.d.ts","sourceRoot":"","sources":["../../src/context/shared-working-memory.ts"],"names":[],"mappings":"AAAA;;GAEG;AA8CH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,qBAAa,mBAAmB;IAC9B,OAAO,CAAC,cAAc,CAA0B;IAEhD;;;;OAIG;gBACS,oBAAoB,GAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAM;IAK9D;;;;OAIG;IACH,gBAAgB,IAAI,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;IAI3C;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,oBAAoB,CAAC,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI;CAGnE"}
|
|
@@ -13,7 +13,6 @@ exports.SharedWorkingMemory = void 0;
|
|
|
13
13
|
* @returns The merged object
|
|
14
14
|
*/
|
|
15
15
|
function deepMerge(target, source) {
|
|
16
|
-
// eslint-disable-next-line security/detect-object-injection
|
|
17
16
|
const result = { ...target };
|
|
18
17
|
for (const key of Object.keys(source)) {
|
|
19
18
|
// eslint-disable-next-line security/detect-object-injection
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"shared-working-memory.js","sourceRoot":"","sources":["../../src/context/shared-working-memory.ts"],"names":[],"mappings":";AAAA;;GAEG;;;AAEH;;;;;;;GAOG;AACH,SAAS,SAAS,CAChB,MAA+B,EAC/B,MAA+B;IAE/B,
|
|
1
|
+
{"version":3,"file":"shared-working-memory.js","sourceRoot":"","sources":["../../src/context/shared-working-memory.ts"],"names":[],"mappings":";AAAA;;GAEG;;;AAEH;;;;;;;GAOG;AACH,SAAS,SAAS,CAChB,MAA+B,EAC/B,MAA+B;IAE/B,MAAM,MAAM,GAA4B,EAAE,GAAG,MAAM,EAAE,CAAC;IAEtD,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;QACtC,4DAA4D;QAC5D,MAAM,WAAW,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;QAChC,4DAA4D;QAC5D,MAAM,WAAW,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;QAEhC,IACE,WAAW;YACX,OAAO,WAAW,KAAK,QAAQ;YAC/B,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC;YAC3B,WAAW;YACX,OAAO,WAAW,KAAK,QAAQ;YAC/B,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,EAC3B,CAAC;YACD,mCAAmC;YACnC,4DAA4D;YAC5D,MAAM,CAAC,GAAG,CAAC,GAAG,SAAS,CACrB,WAAsC,EACtC,WAAsC,CACvC,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,wCAAwC;YACxC,4DAA4D;YAC5D,MAAM,CAAC,GAAG,CAAC,GAAG,WAAW,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,MAAa,mBAAmB;IACtB,cAAc,CAA0B;IAEhD;;;;OAIG;IACH,YAAY,uBAAgD,EAAE;QAC5D,6DAA6D;QAC7D,IAAI,CAAC,cAAc,GAAG,EAAE,GAAG,oBAAoB,EAAE,CAAC;IACpD,CAAC;IAED;;;;OAIG;IACH,gBAAgB;QACd,OAAO,EAAE,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;IACpC,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,oBAAoB,CAAC,aAAsC;QACzD,IAAI,CAAC,cAAc,GAAG,SAAS,CAAC,IAAI,CAAC,cAAc,EAAE,aAAa,CAAC,CAAC;IACtE,CAAC;CACF;AAjDD,kDAiDC"}
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
import { LlmBroker } from './broker';
|
|
5
5
|
import { LlmMessage } from './models';
|
|
6
6
|
import { LlmTool } from './tools';
|
|
7
|
-
import {
|
|
7
|
+
import { Tokenizer } from './gateways/tokenizer';
|
|
8
8
|
/**
|
|
9
9
|
* Configuration options for ChatSession
|
|
10
10
|
*/
|
|
@@ -12,7 +12,7 @@ export interface ChatSessionConfig {
|
|
|
12
12
|
systemPrompt?: string;
|
|
13
13
|
tools?: LlmTool[];
|
|
14
14
|
maxContext?: number;
|
|
15
|
-
tokenizerGateway?:
|
|
15
|
+
tokenizerGateway?: Tokenizer;
|
|
16
16
|
temperature?: number;
|
|
17
17
|
}
|
|
18
18
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chat-session.d.ts","sourceRoot":"","sources":["../../src/llm/chat-session.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,EAAE,UAAU,EAAe,MAAM,UAAU,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"chat-session.d.ts","sourceRoot":"","sources":["../../src/llm/chat-session.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,EAAE,UAAU,EAAe,MAAM,UAAU,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AAWjD;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,OAAO,EAAE,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,gBAAgB,CAAC,EAAE,SAAS,CAAC;IAC7B,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,qBAAa,WAAW;IACtB,OAAO,CAAC,QAAQ,CAAyB;IACzC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAY;IAChC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAY;IACnC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAY;IAE7C;;;;;OAKG;gBACS,GAAG,EAAE,SAAS,EAAE,MAAM,GAAE,iBAAsB;IAe1D;;;;;;;;;;;;OAYG;IACG,IAAI,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IA4B1C;;;;;;;;;;;;;;OAcG;IACI,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC;IA2BxD;;;;OAIG;IACH,WAAW,IAAI,UAAU,EAAE;IAK3B;;OAEG;IACH,KAAK,IAAI,IAAI;IAKb;;;;;OAKG;IACH,OAAO,CAAC,aAAa;IAcrB;;;;;OAKG;IACH,OAAO,CAAC,iBAAiB;IASzB;;;OAGG;IACH,OAAO,CAAC,yBAAyB;IAWjC;;OAEG;IACH,OAAO,IAAI,IAAI;CAGhB"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chat-session.js","sourceRoot":"","sources":["../../src/llm/chat-session.ts"],"names":[],"mappings":";AAAA;;GAEG;;;AAGH,qCAAmD;
|
|
1
|
+
{"version":3,"file":"chat-session.js","sourceRoot":"","sources":["../../src/llm/chat-session.ts"],"names":[],"mappings":";AAAA;;GAEG;;;AAGH,qCAAmD;AAGnD,kEAA+D;AAC/D,oCAAgC;AAoBhC;;;;;;;;;;;;;;;;GAgBG;AACH,MAAa,WAAW;IACd,QAAQ,GAAsB,EAAE,CAAC;IACxB,GAAG,CAAY;IACf,YAAY,CAAS;IACrB,KAAK,CAAa;IAClB,UAAU,CAAS;IACnB,WAAW,CAAS;IACpB,gBAAgB,CAAY;IAE7C;;;;;OAKG;IACH,YAAY,GAAc,EAAE,SAA4B,EAAE;QACxD,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;QACf,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,YAAY,IAAI,8BAA8B,CAAC;QAC1E,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;QAC1B,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,IAAI,KAAK,CAAC;QAC7C,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,GAAG,CAAC;QAC7C,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC,gBAAgB,IAAI,IAAI,mCAAgB,EAAE,CAAC;QAE1E,gCAAgC;QAChC,IAAI,CAAC,aAAa,CAAC;YACjB,IAAI,EAAE,oBAAW,CAAC,MAAM;YACxB,OAAO,EAAE,IAAI,CAAC,YAAY;SAC3B,CAAC,CAAC;IACL,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,IAAI,CAAC,KAAa;QACtB,mBAAmB;QACnB,IAAI,CAAC,aAAa,CAAC;YACjB,IAAI,EAAE,oBAAW,CAAC,IAAI;YACtB,OAAO,EAAE,KAAK;SACf,CAAC,CAAC;QAEH,oBAAoB;QACpB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAwB,EAAE,IAAI,CAAC,KAAK,EAAE;YAChF,WAAW,EAAE,IAAI,CAAC,WAAW;SAC9B,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;YACf,MAAM,MAAM,CAAC,KAAK,CAAC;QACrB,CAAC;QAED,wCAAwC;QACxC,IAAI,CAAC,yBAAyB,EAAE,CAAC;QAEjC,yBAAyB;QACzB,IAAI,CAAC,aAAa,CAAC;YACjB,IAAI,EAAE,oBAAW,CAAC,SAAS;YAC3B,OAAO,EAAE,MAAM,CAAC,KAAK;SACtB,CAAC,CAAC;QAEH,OAAO,MAAM,CAAC,KAAK,CAAC;IACtB,CAAC;IAED;;;;;;;;;;;;;;OAcG;IACH,KAAK,CAAC,CAAC,UAAU,CAAC,KAAa;QAC7B,IAAI,CAAC,aAAa,CAAC;YACjB,IAAI,EAAE,oBAAW,CAAC,IAAI;YACtB,OAAO,EAAE,KAAK;SACf,CAAC,CAAC;QAEH,MAAM,WAAW,GAAa,EAAE,CAAC;QACjC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,IAAI,CAAC,GAAG,CAAC,cAAc,CAChD,IAAI,CAAC,QAAwB,EAC7B,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,EACjC,IAAI,CAAC,KAAK,CACX,EAAE,CAAC;YACF,IAAI,CAAC,IAAA,YAAI,EAAC,MAAM,CAAC,EAAE,CAAC;gBAClB,MAAM,MAAM,CAAC,KAAK,CAAC;YACrB,CAAC;YACD,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC/B,MAAM,MAAM,CAAC,KAAK,CAAC;QACrB,CAAC;QAED,IAAI,CAAC,yBAAyB,EAAE,CAAC;QAEjC,IAAI,CAAC,aAAa,CAAC;YACjB,IAAI,EAAE,oBAAW,CAAC,SAAS;YAC3B,OAAO,EAAE,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;SAC9B,CAAC,CAAC;IACL,CAAC;IAED;;;;OAIG;IACH,WAAW;QACT,6DAA6D;QAC7D,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,EAAE,GAAG,GAAG,EAAE,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC;IAC7D,CAAC;IAED;;OAEG;IACH,KAAK;QACH,MAAM,aAAa,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;QACvC,IAAI,CAAC,QAAQ,GAAG,CAAC,aAAa,CAAC,CAAC;IAClC,CAAC;IAED;;;;;OAKG;IACK,aAAa,CAAC,OAAmB;QACvC,MAAM,YAAY,GAAG,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;QACrD,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAEjC,iDAAiD;QACjD,IAAI,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QAE/E,OAAO,WAAW,GAAG,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACjE,4DAA4D;YAC5D,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC9C,WAAW,IAAI,OAAO,CAAC,WAAW,CAAC;QACrC,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACK,iBAAiB,CAAC,OAAmB;QAC3C,IAAI,CAAC,OAAO,CAAC,OAAO,IAAI,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;YAC5D,OAAO,EAAE,GAAG,OAAO,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC;QACxC,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC7D,OAAO,EAAE,GAAG,OAAO,EAAE,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;IACpD,CAAC;IAED;;;OAGG;IACK,yBAAyB;QAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC9C,iGAAiG;YACjG,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;YAC7B,IAAI,CAAC,CAAC,aAAa,IAAI,GAAG,CAAC,IAAI,GAAG,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;gBAC7D,iGAAiG;gBACjG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAiB,CAAC,CAAC;YAC/D,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACH,OAAO;QACL,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC;IAC/B,CAAC;CACF;AA3LD,kCA2LC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/llm/gateways/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,cAAc,UAAU,CAAC;AACzB,cAAc,UAAU,CAAC;AACzB,cAAc,yBAAyB,CAAC;AACxC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,oBAAoB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/llm/gateways/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,cAAc,UAAU,CAAC;AACzB,cAAc,UAAU,CAAC;AACzB,cAAc,yBAAyB,CAAC;AACxC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,aAAa,CAAC;AAC5B,cAAc,oBAAoB,CAAC"}
|
|
@@ -21,5 +21,6 @@ __exportStar(require("./ollama"), exports);
|
|
|
21
21
|
__exportStar(require("./openai"), exports);
|
|
22
22
|
__exportStar(require("./openai-model-registry"), exports);
|
|
23
23
|
__exportStar(require("./openai-messages-adapter"), exports);
|
|
24
|
+
__exportStar(require("./tokenizer"), exports);
|
|
24
25
|
__exportStar(require("./tokenizerGateway"), exports);
|
|
25
26
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/llm/gateways/index.ts"],"names":[],"mappings":";AAAA;;GAEG;;;;;;;;;;;;;;;;AAEH,2CAAyB;AACzB,2CAAyB;AACzB,0DAAwC;AACxC,4DAA0C;AAC1C,qDAAmC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/llm/gateways/index.ts"],"names":[],"mappings":";AAAA;;GAEG;;;;;;;;;;;;;;;;AAEH,2CAAyB;AACzB,2CAAyB;AACzB,0DAAwC;AACxC,4DAA0C;AAC1C,8CAA4B;AAC5B,qDAAmC"}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tokenizer interface for encoding and decoding text to/from token IDs.
|
|
3
|
+
*
|
|
4
|
+
* Implementations wrap encoding libraries (e.g. gpt-tokenizer) so that
|
|
5
|
+
* ChatSession depends on this interface rather than a concrete class,
|
|
6
|
+
* enabling custom tokenizer injection and simpler testing.
|
|
7
|
+
*/
|
|
8
|
+
export interface Tokenizer {
|
|
9
|
+
/**
|
|
10
|
+
* Encodes text into an array of token IDs.
|
|
11
|
+
*
|
|
12
|
+
* @param text - The text to encode
|
|
13
|
+
* @returns An array of token IDs
|
|
14
|
+
*/
|
|
15
|
+
encode(text: string): number[];
|
|
16
|
+
/**
|
|
17
|
+
* Decodes an array of token IDs back into text.
|
|
18
|
+
*
|
|
19
|
+
* @param tokens - The array of token IDs to decode
|
|
20
|
+
* @returns The decoded text
|
|
21
|
+
*/
|
|
22
|
+
decode(tokens: number[]): string;
|
|
23
|
+
/**
|
|
24
|
+
* Releases any resources held by the tokenizer.
|
|
25
|
+
* Implementations backed by pure JS may treat this as a no-op.
|
|
26
|
+
*/
|
|
27
|
+
free(): void;
|
|
28
|
+
}
|
|
29
|
+
//# sourceMappingURL=tokenizer.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tokenizer.d.ts","sourceRoot":"","sources":["../../../src/llm/gateways/tokenizer.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,MAAM,WAAW,SAAS;IACxB;;;;;OAKG;IACH,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAE/B;;;;;OAKG;IACH,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC;IAEjC;;;OAGG;IACH,IAAI,IAAI,IAAI,CAAC;CACd"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tokenizer.js","sourceRoot":"","sources":["../../../src/llm/gateways/tokenizer.ts"],"names":[],"mappings":""}
|
|
@@ -1,28 +1,35 @@
|
|
|
1
|
+
import { Tokenizer } from './tokenizer';
|
|
1
2
|
/**
|
|
2
|
-
* Gateway for tokenizing and detokenizing text using
|
|
3
|
+
* Gateway for tokenizing and detokenizing text using gpt-tokenizer.
|
|
3
4
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
5
|
+
* A pure-TypeScript BPE tokenizer with no WebAssembly dependency — safe to
|
|
6
|
+
* use in any JavaScript runtime, including Cloudflare Workers, Next.js Edge,
|
|
7
|
+
* and Deno.
|
|
8
|
+
*
|
|
9
|
+
* Implements the {@link Tokenizer} interface so it can be injected wherever
|
|
10
|
+
* a tokenizer is needed (e.g. {@link ChatSession}).
|
|
9
11
|
*
|
|
10
12
|
* @example
|
|
11
13
|
* ```typescript
|
|
12
14
|
* const tokenizer = new TokenizerGateway();
|
|
13
15
|
* const tokens = tokenizer.encode("Hello, world!");
|
|
14
16
|
* const text = tokenizer.decode(tokens);
|
|
17
|
+
* tokenizer.free(); // no-op for pure-JS tokenizers, safe to call
|
|
15
18
|
* ```
|
|
16
19
|
*/
|
|
17
|
-
export declare class TokenizerGateway {
|
|
18
|
-
private
|
|
20
|
+
export declare class TokenizerGateway implements Tokenizer {
|
|
21
|
+
private readonly encodeText;
|
|
22
|
+
private readonly decodeTokens;
|
|
19
23
|
/**
|
|
20
24
|
* Creates a new TokenizerGateway instance.
|
|
21
25
|
*
|
|
22
|
-
* @param
|
|
23
|
-
*
|
|
26
|
+
* @param encoding - The BPE encoding to use. Defaults to `"cl100k_base"`
|
|
27
|
+
* (used by GPT-4 and GPT-3.5-turbo). Other supported values:
|
|
28
|
+
* `"p50k_base"` (GPT-3), `"r50k_base"` (older models),
|
|
29
|
+
* `"o200k_base"` (GPT-4o).
|
|
24
30
|
*/
|
|
25
|
-
constructor(
|
|
31
|
+
constructor(encoding?: string);
|
|
32
|
+
private static selectEncoding;
|
|
26
33
|
/**
|
|
27
34
|
* Encodes text into tokens.
|
|
28
35
|
*
|
|
@@ -53,8 +60,9 @@ export declare class TokenizerGateway {
|
|
|
53
60
|
*/
|
|
54
61
|
decode(tokens: number[]): string;
|
|
55
62
|
/**
|
|
56
|
-
*
|
|
57
|
-
*
|
|
63
|
+
* No-op for pure-JavaScript tokenizers — there is no WebAssembly memory
|
|
64
|
+
* to release. Exists so that code written against the {@link Tokenizer}
|
|
65
|
+
* interface continues to work without modification.
|
|
58
66
|
*/
|
|
59
67
|
free(): void;
|
|
60
68
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tokenizerGateway.d.ts","sourceRoot":"","sources":["../../../src/llm/gateways/tokenizerGateway.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"tokenizerGateway.d.ts","sourceRoot":"","sources":["../../../src/llm/gateways/tokenizerGateway.ts"],"names":[],"mappings":"AAgBA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAIxC;;;;;;;;;;;;;;;;;GAiBG;AACH,qBAAa,gBAAiB,YAAW,SAAS;IAChD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAA6B;IACxD,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAuC;IAEpE;;;;;;;OAOG;gBACS,QAAQ,GAAE,MAAsB;IAQ5C,OAAO,CAAC,MAAM,CAAC,cAAc;IAgB7B;;;;;;;;;;;;OAYG;IACH,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE;IAI9B;;;;;;;;;;;;;OAaG;IACH,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM;IAIhC;;;;OAIG;IACH,IAAI,IAAI,IAAI;CAGb"}
|
|
@@ -1,33 +1,60 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.TokenizerGateway = void 0;
|
|
4
|
-
|
|
4
|
+
// Use cjs/ sub-paths directly — the project uses moduleResolution:"node" which
|
|
5
|
+
// does not process package.json "exports" fields, so the advertised subpath
|
|
6
|
+
// aliases (e.g. 'gpt-tokenizer/encoding/p50k_base') are not visible to tsc.
|
|
7
|
+
// The cjs/ directory is physically present under node_modules and is resolved
|
|
8
|
+
// by the classic Node resolver without any exports-field support.
|
|
9
|
+
const cl100k_base_1 = require("gpt-tokenizer/cjs/encoding/cl100k_base");
|
|
10
|
+
const p50k_base_1 = require("gpt-tokenizer/cjs/encoding/p50k_base");
|
|
11
|
+
const r50k_base_1 = require("gpt-tokenizer/cjs/encoding/r50k_base");
|
|
12
|
+
const o200k_base_1 = require("gpt-tokenizer/cjs/encoding/o200k_base");
|
|
5
13
|
/**
|
|
6
|
-
* Gateway for tokenizing and detokenizing text using
|
|
14
|
+
* Gateway for tokenizing and detokenizing text using gpt-tokenizer.
|
|
7
15
|
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
16
|
+
* A pure-TypeScript BPE tokenizer with no WebAssembly dependency — safe to
|
|
17
|
+
* use in any JavaScript runtime, including Cloudflare Workers, Next.js Edge,
|
|
18
|
+
* and Deno.
|
|
19
|
+
*
|
|
20
|
+
* Implements the {@link Tokenizer} interface so it can be injected wherever
|
|
21
|
+
* a tokenizer is needed (e.g. {@link ChatSession}).
|
|
13
22
|
*
|
|
14
23
|
* @example
|
|
15
24
|
* ```typescript
|
|
16
25
|
* const tokenizer = new TokenizerGateway();
|
|
17
26
|
* const tokens = tokenizer.encode("Hello, world!");
|
|
18
27
|
* const text = tokenizer.decode(tokens);
|
|
28
|
+
* tokenizer.free(); // no-op for pure-JS tokenizers, safe to call
|
|
19
29
|
* ```
|
|
20
30
|
*/
|
|
21
31
|
class TokenizerGateway {
|
|
22
|
-
|
|
32
|
+
encodeText;
|
|
33
|
+
decodeTokens;
|
|
23
34
|
/**
|
|
24
35
|
* Creates a new TokenizerGateway instance.
|
|
25
36
|
*
|
|
26
|
-
* @param
|
|
27
|
-
*
|
|
37
|
+
* @param encoding - The BPE encoding to use. Defaults to `"cl100k_base"`
|
|
38
|
+
* (used by GPT-4 and GPT-3.5-turbo). Other supported values:
|
|
39
|
+
* `"p50k_base"` (GPT-3), `"r50k_base"` (older models),
|
|
40
|
+
* `"o200k_base"` (GPT-4o).
|
|
28
41
|
*/
|
|
29
|
-
constructor(
|
|
30
|
-
|
|
42
|
+
constructor(encoding = 'cl100k_base') {
|
|
43
|
+
const { encodeText, decodeTokens } = TokenizerGateway.selectEncoding(encoding);
|
|
44
|
+
this.encodeText = encodeText;
|
|
45
|
+
this.decodeTokens = decodeTokens;
|
|
46
|
+
}
|
|
47
|
+
static selectEncoding(encoding) {
|
|
48
|
+
switch (encoding) {
|
|
49
|
+
case 'p50k_base':
|
|
50
|
+
return { encodeText: p50k_base_1.encode, decodeTokens: p50k_base_1.decode };
|
|
51
|
+
case 'r50k_base':
|
|
52
|
+
return { encodeText: r50k_base_1.encode, decodeTokens: r50k_base_1.decode };
|
|
53
|
+
case 'o200k_base':
|
|
54
|
+
return { encodeText: o200k_base_1.encode, decodeTokens: o200k_base_1.decode };
|
|
55
|
+
default:
|
|
56
|
+
return { encodeText: cl100k_base_1.encode, decodeTokens: cl100k_base_1.decode };
|
|
57
|
+
}
|
|
31
58
|
}
|
|
32
59
|
/**
|
|
33
60
|
* Encodes text into tokens.
|
|
@@ -43,7 +70,7 @@ class TokenizerGateway {
|
|
|
43
70
|
* ```
|
|
44
71
|
*/
|
|
45
72
|
encode(text) {
|
|
46
|
-
return
|
|
73
|
+
return this.encodeText(text);
|
|
47
74
|
}
|
|
48
75
|
/**
|
|
49
76
|
* Decodes tokens back into text.
|
|
@@ -60,15 +87,15 @@ class TokenizerGateway {
|
|
|
60
87
|
* ```
|
|
61
88
|
*/
|
|
62
89
|
decode(tokens) {
|
|
63
|
-
|
|
64
|
-
return decoder.decode(this.tokenizer.decode(new Uint32Array(tokens)));
|
|
90
|
+
return this.decodeTokens(tokens);
|
|
65
91
|
}
|
|
66
92
|
/**
|
|
67
|
-
*
|
|
68
|
-
*
|
|
93
|
+
* No-op for pure-JavaScript tokenizers — there is no WebAssembly memory
|
|
94
|
+
* to release. Exists so that code written against the {@link Tokenizer}
|
|
95
|
+
* interface continues to work without modification.
|
|
69
96
|
*/
|
|
70
97
|
free() {
|
|
71
|
-
|
|
98
|
+
// pure-JS tokenizer — nothing to release
|
|
72
99
|
}
|
|
73
100
|
}
|
|
74
101
|
exports.TokenizerGateway = TokenizerGateway;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tokenizerGateway.js","sourceRoot":"","sources":["../../../src/llm/gateways/tokenizerGateway.ts"],"names":[],"mappings":";;;AAAA,
|
|
1
|
+
{"version":3,"file":"tokenizerGateway.js","sourceRoot":"","sources":["../../../src/llm/gateways/tokenizerGateway.ts"],"names":[],"mappings":";;;AAAA,+EAA+E;AAC/E,4EAA4E;AAC5E,4EAA4E;AAC5E,8EAA8E;AAC9E,kEAAkE;AAClE,wEAGgD;AAChD,oEAAkG;AAClG,oEAAkG;AAClG,sEAG+C;AAM/C;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAa,gBAAgB;IACV,UAAU,CAA6B;IACvC,YAAY,CAAuC;IAEpE;;;;;;;OAOG;IACH,YAAY,WAAmB,aAAa;QAC1C,MAAM,EAAE,UAAU,EAAE,YAAY,EAAE,GAAG,gBAAgB,CAAC,cAAc,CAClE,QAA6B,CAC9B,CAAC;QACF,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;IACnC,CAAC;IAEO,MAAM,CAAC,cAAc,CAAC,QAA2B;QAIvD,QAAQ,QAAQ,EAAE,CAAC;YACjB,KAAK,WAAW;gBACd,OAAO,EAAE,UAAU,EAAE,kBAAU,EAAE,YAAY,EAAE,kBAAU,EAAE,CAAC;YAC9D,KAAK,WAAW;gBACd,OAAO,EAAE,UAAU,EAAE,kBAAU,EAAE,YAAY,EAAE,kBAAU,EAAE,CAAC;YAC9D,KAAK,YAAY;gBACf,OAAO,EAAE,UAAU,EAAE,mBAAW,EAAE,YAAY,EAAE,mBAAW,EAAE,CAAC;YAChE;gBACE,OAAO,EAAE,UAAU,EAAE,oBAAY,EAAE,YAAY,EAAE,oBAAY,EAAE,CAAC;QACpE,CAAC;IACH,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,MAAM,CAAC,IAAY;QACjB,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,MAAM,CAAC,MAAgB;QACrB,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;IACnC,CAAC;IAED;;;;OAIG;IACH,IAAI;QACF,yCAAyC;IAC3C,CAAC;CACF;AA/ED,4CA+EC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mojentic",
|
|
3
|
-
"version": "1.2.
|
|
3
|
+
"version": "1.2.4",
|
|
4
4
|
"description": "Modern LLM integration framework for TypeScript with tool support and structured output generation",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -44,26 +44,30 @@
|
|
|
44
44
|
"author": "Stacey Vetzal <stacey@mojility.com>",
|
|
45
45
|
"license": "MIT",
|
|
46
46
|
"devDependencies": {
|
|
47
|
+
"@eslint/js": "^10.0.1",
|
|
47
48
|
"@types/jest": "^30.0.0",
|
|
48
|
-
"@types/node": "^25.
|
|
49
|
-
"@typescript-eslint/eslint-plugin": "^8.
|
|
50
|
-
"@typescript-eslint/parser": "^8.
|
|
51
|
-
"eslint": "^
|
|
52
|
-
"eslint-plugin-security": "^
|
|
53
|
-
"jest": "^30.
|
|
49
|
+
"@types/node": "^25.5.0",
|
|
50
|
+
"@typescript-eslint/eslint-plugin": "^8.57.0",
|
|
51
|
+
"@typescript-eslint/parser": "^8.57.0",
|
|
52
|
+
"eslint": "^10.0.3",
|
|
53
|
+
"eslint-plugin-security": "^4.0.0",
|
|
54
|
+
"jest": "^30.3.0",
|
|
54
55
|
"prettier": "^3.8.1",
|
|
55
56
|
"ts-jest": "^29.4.6",
|
|
56
57
|
"ts-node": "^10.9.2",
|
|
57
|
-
"typescript": "^5.
|
|
58
|
-
"
|
|
58
|
+
"typescript": "^5.9.3",
|
|
59
|
+
"typescript-eslint": "^8.57.0",
|
|
60
|
+
"vitepress": "^1.6.4"
|
|
59
61
|
},
|
|
60
62
|
"dependencies": {
|
|
61
|
-
"
|
|
63
|
+
"gpt-tokenizer": "^3.4.0",
|
|
62
64
|
"zod": "^4.3.6"
|
|
63
65
|
},
|
|
64
66
|
"overrides": {
|
|
65
67
|
"js-yaml": "3.14.2",
|
|
66
|
-
"esbuild": "0.27.
|
|
68
|
+
"esbuild": "0.27.3",
|
|
69
|
+
"minimatch": ">=10.2.1",
|
|
70
|
+
"test-exclude": "^8.0.0"
|
|
67
71
|
},
|
|
68
72
|
"engines": {
|
|
69
73
|
"node": ">=18.0.0"
|