@tanstack-ai-browser/wllama 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapter.d.ts +11 -1
- package/dist/adapter.d.ts.map +1 -1
- package/dist/adapter.js +12 -0
- package/package.json +5 -5
- package/src/adapter.ts +16 -1
- package/src/wrapper.test.ts +13 -0
package/dist/adapter.d.ts
CHANGED
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
* thinking switch, the model card's sampling, and audio parts for the
|
|
11
11
|
* models that read them.
|
|
12
12
|
*/
|
|
13
|
-
import type { TextOptions } from "@tanstack/ai";
|
|
13
|
+
import type { TextOptions, TokenUsage } from "@tanstack/ai";
|
|
14
14
|
import { OpenAIBaseChatCompletionsTextAdapter } from "@tanstack/openai-base";
|
|
15
15
|
import type { OpenAIClientLike } from "@tanstack/openai-base";
|
|
16
16
|
import type { WllamaCompat } from "@wllama/wllama/esm/index.js";
|
|
@@ -136,6 +136,16 @@ export declare class WllamaTextAdapter<TModel extends WllamaModel> extends OpenA
|
|
|
136
136
|
/** Unloads the model, freeing its memory. It stays in wllama's storage. */
|
|
137
137
|
destroy(): Promise<void>;
|
|
138
138
|
/** The request, with `modelOptions` as given over the model card's sampling, and the chat template's thinking switch. */
|
|
139
|
+
/**
|
|
140
|
+
* The engine's usage, as TanStack converts it, but with a cached count of 0
|
|
141
|
+
* kept, which TanStack's conversion leaves out: 0 means the whole prompt was
|
|
142
|
+
* read, where no count means the engine didn't say.
|
|
143
|
+
*/
|
|
144
|
+
protected chatCompletionsTokenUsage(usage: {
|
|
145
|
+
prompt_tokens_details?: {
|
|
146
|
+
cached_tokens?: number;
|
|
147
|
+
} | null;
|
|
148
|
+
} | null | undefined, source?: unknown): TokenUsage | undefined;
|
|
139
149
|
protected mapOptionsToRequest(options: TextOptions): CreateParams & {
|
|
140
150
|
stream: true;
|
|
141
151
|
};
|
package/dist/adapter.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"adapter.d.ts","sourceRoot":"","sources":["../src/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AACH,OAAO,KAAK,EAAoB,WAAW,EAAE,MAAM,cAAc,CAAA;
|
|
1
|
+
{"version":3,"file":"adapter.d.ts","sourceRoot":"","sources":["../src/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AACH,OAAO,KAAK,EAAoB,WAAW,EAAE,UAAU,EAAE,MAAM,cAAc,CAAA;AAE7E,OAAO,EAAE,oCAAoC,EAAE,MAAM,uBAAuB,CAAA;AAC5E,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAA;AAC7D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAA;AAI/D,OAAO,KAAK,EAAE,YAAY,EAAE,wBAAwB,EAAgB,MAAM,aAAa,CAAA;AAEvF,OAAO,KAAK,EAAY,eAAe,EAAE,WAAW,EAAE,0BAA0B,EAAE,MAAM,cAAc,CAAA;AAEtG,MAAM,MAAM,YAAY,GAAG;IACzB;;;;;;;OAOG;IACH,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB;;;;;OAKG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB;;;;OAIG;IACH,MAAM,CAAC,EAAE,YAAY,CAAA;IACrB,+DAA+D;IAC/D,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB;;;OAGG;IACH,GAAG,CAAC,EAAE,OAAO,CAAA;IACb,qDAAqD;IACrD,KAAK,CAAC,EAAE,OAAO,CAAA;IACf;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,wBAAwB,CAAA;CAC9C,CAAA;AAED;;;;;GAKG;AACH,MAAM,MAAM,sBAAsB,GAAG;IACnC,0GAA0G;IAC1G,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,2CAA2C;IAC3C,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,gFAAgF;IAChF,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,wEAAwE;IACxE,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,4EAA4E;IAC5E,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,qDAAqD;IACrD,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,+DAA+D;IAC/D,iBAAiB,CAAC,EAAE,MAAM,CAAA;IAC1B,2DAA2D;IAC3D,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,qEAAqE;IACrE,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,8EAA8E;IAC9E,IAAI,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAA;IAC7B,+CAA+C;IAC/C,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IACnC;;;OAGG;IACH,mBAAmB,CAAC,EAAE,OAAO,CAAA;IAC7B,2FAA2F;IAC3F,oBAAoB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAC/C,CAAA;AAED,gHAAgH;AAChH,KAAK,gBAAgB,CAAC,MAAM,SAAS,MAAM,IAAI,MAAM,SAAS,MAAM,0BAA0B,GAC1F,0BAA0B,CAAC,MAAM,CAAC,GAClC,MAAM,SAAS,eAAe,GAC5B,KAAK,GACL;IAAE,MAAM,EAAE,KAAK,GAAG,MAAM,CAAC;IAAC,MAAM,EAAE,IAAI,CAAA;CAAE,CAAA;AAQ9C,KAAK,kBAAkB,GAAG;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,OAAO,CAAC;IAAC,QAAQ,EAAE,OAAO,CAAA;CAAE,CAAA;AAE9G,iFAAiF;AACjF,KAAK,kBAAkB,GAAG,oCAAoC,CAAC,MAAM,CAAC,CAAC,oBAAoB,CAAC,CAAA;AAE5F,iEAAiE;AACjE,KAAK,YAAY,GAAG,UAAU,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,aAAa,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;AA6FpF,qBAAa,iBAAiB,CAAC,MAAM,SAAS,WAAW,CAAE,SAAQ,oCAAoC,CACrG,MAAM,EACN,sBAAsB,EACtB,aAAa,CAAC,MAAM,GAAG,OAAO,GAAG,OAAO,CAAC,EACzC,kBAAkB,EAClB,aAAa,CAAC,MAAM,CAAC,EACrB,gBAAgB,CAAC,MAAM,CAAC,CACzB;IACC,SAAkB,IAAI,YAAW;IACjC,SAAkB,GAAG,YAAW;IAChC,+EAA+E;IAC/E,SAAkB,eAAe,EAAE,aAAa,CAAC,MAAM,GAAG,OAAO,GAAG,OAAO,CAAC,CAAA;IAE5E,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAc;IACvC,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAsC;IAEzE,YAAY,MAAM,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,EAM9C;IAED;;;;;OAKG;IACH,YAAY,IAAI,OAAO,CAAC,YAAY,CAAC,CAEpC;IAED;;;;;OAKG;IACH,MAAM,CAAC,OAAO,GAAE;QAAE,kBAAkB,CAAC,EAAE,wBAAwB,CAAC;QAAC,MAAM,CAAC,EAAE,WAAW,CAAA;KAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAY3G;IAED,2EAA2E;IACrE,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAE7B;IAED,yHAAyH;IACzH;;;;OAIG;IACH,UAAmB,yBAAyB,CAC1C,KAAK,EAAE;QAAE,qBAAqB,CAAC,EAAE;YAAE,aAAa,CAAC,EAAE,MAAM,CAAA;SAAE,GAAG,IAAI,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,EACvF,MAAM,CAAC,EAAE,OAAO,GACf,UAAU,GAAG,SAAS,CAKxB;IAED,UAAmB,mBAAmB,CAAC,OAAO,EAAE,WAAW,GAAG,YAAY,GAAG;QAAE,MAAM,EAAE,IAAI,CAAA;KAAE,CAS5F;IAED,yDAAyD;IACzD,UAAmB,gBAAgB,CAAC,KAAK,EAAE,OAAO;;kBAGjD;IAED;;;;OAIG;IACH,UAAmB,kBAAkB,CAAC,IAAI,EAAE,UAAU,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,kBAAkB,CAAC,CAW7G;IAED,yGAAyG;IACzG,UAAmB,8BAA8B,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,uBAE5E;CACF;AAED;;;;;;;;GAQG;AACH,wBAAgB,UAAU,CAAC,MAAM,SAAS,WAAW,EACnD,KAAK,EAAE,MAAM,EACb,MAAM,GAAE,YAAiB,GACxB,iBAAiB,CAAC,MAAM,CAAC,CAE3B"}
|
package/dist/adapter.js
CHANGED
|
@@ -132,6 +132,18 @@ export class WllamaTextAdapter extends OpenAIBaseChatCompletionsTextAdapter {
|
|
|
132
132
|
await unloadModel(this.model);
|
|
133
133
|
}
|
|
134
134
|
/** The request, with `modelOptions` as given over the model card's sampling, and the chat template's thinking switch. */
|
|
135
|
+
/**
|
|
136
|
+
* The engine's usage, as TanStack converts it, but with a cached count of 0
|
|
137
|
+
* kept, which TanStack's conversion leaves out: 0 means the whole prompt was
|
|
138
|
+
* read, where no count means the engine didn't say.
|
|
139
|
+
*/
|
|
140
|
+
chatCompletionsTokenUsage(usage, source) {
|
|
141
|
+
const result = super.chatCompletionsTokenUsage(usage, source);
|
|
142
|
+
const cachedTokens = usage?.prompt_tokens_details?.cached_tokens;
|
|
143
|
+
if (!result || cachedTokens !== 0)
|
|
144
|
+
return result;
|
|
145
|
+
return { ...result, promptTokensDetails: { ...result.promptTokensDetails, cachedTokens } };
|
|
146
|
+
}
|
|
135
147
|
mapOptionsToRequest(options) {
|
|
136
148
|
const modelOptions = options.modelOptions;
|
|
137
149
|
const settings = {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tanstack-ai-browser/wllama",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.1",
|
|
4
4
|
"description": "TanStack AI adapter for GGUF models running in the browser with wllama (llama.cpp in WebAssembly)",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "daveycodez",
|
|
@@ -55,11 +55,11 @@
|
|
|
55
55
|
"@tanstack/ai": "1.4.0",
|
|
56
56
|
"esbuild": "0.28.2"
|
|
57
57
|
},
|
|
58
|
+
"scripts": {
|
|
59
|
+
"test": "node --import @swc-node/register/esm-register --test \"src/**/*.test.ts\""
|
|
60
|
+
},
|
|
58
61
|
"dependencies": {
|
|
59
62
|
"@tanstack/openai-base": "^1.0.4",
|
|
60
63
|
"@wllama/wllama": "3.13.0"
|
|
61
|
-
},
|
|
62
|
-
"scripts": {
|
|
63
|
-
"test": "node --import @swc-node/register/esm-register --test \"src/**/*.test.ts\""
|
|
64
64
|
}
|
|
65
|
-
}
|
|
65
|
+
}
|
package/src/adapter.ts
CHANGED
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
* thinking switch, the model card's sampling, and audio parts for the
|
|
11
11
|
* models that read them.
|
|
12
12
|
*/
|
|
13
|
-
import type { ReasoningRequest, TextOptions } from "@tanstack/ai"
|
|
13
|
+
import type { ReasoningRequest, TextOptions, TokenUsage } from "@tanstack/ai"
|
|
14
14
|
import { resolveReasoning } from "@tanstack/ai/adapter-internals"
|
|
15
15
|
import { OpenAIBaseChatCompletionsTextAdapter } from "@tanstack/openai-base"
|
|
16
16
|
import type { OpenAIClientLike } from "@tanstack/openai-base"
|
|
@@ -272,6 +272,21 @@ export class WllamaTextAdapter<TModel extends WllamaModel> extends OpenAIBaseCha
|
|
|
272
272
|
}
|
|
273
273
|
|
|
274
274
|
/** The request, with `modelOptions` as given over the model card's sampling, and the chat template's thinking switch. */
|
|
275
|
+
/**
|
|
276
|
+
* The engine's usage, as TanStack converts it, but with a cached count of 0
|
|
277
|
+
* kept, which TanStack's conversion leaves out: 0 means the whole prompt was
|
|
278
|
+
* read, where no count means the engine didn't say.
|
|
279
|
+
*/
|
|
280
|
+
protected override chatCompletionsTokenUsage(
|
|
281
|
+
usage: { prompt_tokens_details?: { cached_tokens?: number } | null } | null | undefined,
|
|
282
|
+
source?: unknown
|
|
283
|
+
): TokenUsage | undefined {
|
|
284
|
+
const result = super.chatCompletionsTokenUsage(usage as never, source)
|
|
285
|
+
const cachedTokens = usage?.prompt_tokens_details?.cached_tokens
|
|
286
|
+
if (!result || cachedTokens !== 0) return result
|
|
287
|
+
return { ...result, promptTokensDetails: { ...result.promptTokensDetails, cachedTokens } }
|
|
288
|
+
}
|
|
289
|
+
|
|
275
290
|
protected override mapOptionsToRequest(options: TextOptions): CreateParams & { stream: true } {
|
|
276
291
|
const modelOptions = options.modelOptions as WllamaTextModelOptions | undefined
|
|
277
292
|
const settings = {
|
package/src/wrapper.test.ts
CHANGED
|
@@ -171,3 +171,16 @@ test("audio kept as an uploaded file is refused: wllama reads bytes", async () =
|
|
|
171
171
|
assert.match(JSON.stringify(failure), /not as uploaded files/)
|
|
172
172
|
assert.equal(control.requests.length, 0)
|
|
173
173
|
})
|
|
174
|
+
|
|
175
|
+
test("usage keeps llama.cpp's cached count, 0 included (TanStack's conversion leaves a 0 out)", async () => {
|
|
176
|
+
for (const cached_tokens of [0, 352]) {
|
|
177
|
+
control.chunks = [
|
|
178
|
+
delta({ role: "assistant", content: "Hi" }),
|
|
179
|
+
{ ...delta({}, "stop"), usage: { prompt_tokens: 371, completion_tokens: 2, total_tokens: 373, prompt_tokens_details: { cached_tokens } } }
|
|
180
|
+
]
|
|
181
|
+
const { events } = await run()
|
|
182
|
+
const usage = events.find((e) => e.type === "RUN_FINISHED")?.usage as { promptTokens: number; promptTokensDetails?: unknown }
|
|
183
|
+
assert.equal(usage.promptTokens, 371)
|
|
184
|
+
assert.deepEqual(usage.promptTokensDetails, { cachedTokens: cached_tokens })
|
|
185
|
+
}
|
|
186
|
+
})
|