@trymirai/uzu 0.2.6 → 0.2.20

package/README.md

@@ -4,8 +4,6 @@
   </picture>
 </p>
 
-<a href="https://artifacts.trymirai.com/social/about_us.mp3"><img src="https://img.shields.io/badge/Listen-Podcast-red" alt="Listen to our podcast"></a>
-<a href="https://docsend.com/v/76bpr/mirai2025"><img src="https://img.shields.io/badge/View-Deck-red" alt="View our deck"></a>
 <a href="https://discord.com/invite/trymirai"><img src="https://img.shields.io/discord/1377764166764462120?label=Discord" alt="Discord"></a>
 <a href="mailto:contact@getmirai.co?subject=Interested%20in%20Mirai"><img src="https://img.shields.io/badge/Send-Email-green" alt="Contact us"></a>
 <a href="https://docs.trymirai.com/app-integration/overview"><img src="https://img.shields.io/badge/Read-Docs-blue" alt="Read docs"></a>
@@ -26,7 +24,7 @@ Add the `uzu` dependency to your project's `package.json`:
 
 ```json
 "dependencies": {
-    "@trymirai/uzu": "0.2.6"
+    "@trymirai/uzu": "0.2.20"
 }
 ```
 
@@ -47,6 +45,7 @@ Place the `API_KEY` you obtained earlier in the corresponding example file, and
 
 ```bash
 pnpm run tsn examples/chat.ts
+pnpm run tsn examples/chatWithSpeculator.ts
 pnpm run tsn examples/chatDynamicContext.ts
 pnpm run tsn examples/chatStaticContext.ts
 pnpm run tsn examples/summarization.ts
@@ -85,6 +84,51 @@ main().catch((error) => {
 });
 ```
 
+### Speedup with speculative decoding
+
+Speculative decoding allows a significant increase in generation speed. For each model, we train a small n-gram model (under 50 MB) tailored to a specific domain or use case. In general chat scenarios, you can use the `Chat` preset, which will automatically use the corresponding speculator:
+
+```ts
+import Engine, { Message, Preset } from '@trymirai/uzu';
+
+async function main() {
+    const engine = Engine.create('API_KEY');
+    const model = engine
+        .chatModel('Qwen/Qwen3-0.6B')
+        .download((update) => {
+            console.log('Progress:', update.progress);
+        });
+
+    const messages = [
+        Message.system('You are a helpful assistant'),
+        Message.user('Tell me a short, funny story about a robot')
+    ];
+
+    const outputGeneral = await model
+        .replyToMessages(
+            messages,
+            (partialOutput) => {
+                return true;
+            },
+        );
+    console.log('Generation speed t/s (general):', outputGeneral.stats.generateStats?.tokensPerSecond ?? 0);
+
+    const outputWithSpeculator = await model
+        .preset(Preset.chat())
+        .replyToMessages(
+            messages,
+            (partialOutput) => {
+                return true;
+            },
+        );
+    console.log('Generation speed t/s (with chat speculator):', outputWithSpeculator.stats.generateStats?.tokensPerSecond ?? 0);
+}
+
+main().catch((error) => {
+    console.error(error);
+});
+```
+
 ### Chat with dynamic context
 
 In this example, we will use the dynamic `ContextMode`, which automatically maintains a continuous conversation history instead of resetting the context with each new input. Every new message is added to the ongoing chat, allowing the model to remember what has already been said and respond with full context.

package/bridging/preset.d.mts

@@ -7,6 +7,8 @@ export declare class Preset implements ToNapi<NapiPreset> {
     static general(): Preset;
     static classification(feature: ClassificationFeature): Preset;
     static summarization(): Preset;
+    static nGramSpeculator(useCase: string, numberOfSpeculatedTokens: number): Preset;
+    static chat(): Preset;
     toNapi(): NapiPreset;
 }
 //# sourceMappingURL=preset.d.mts.map

package/bridging/preset.d.mts.map

@@ -1 +1 @@
-{"version":3,"file":"preset.d.mts","sourceRoot":"","sources":["../src/bridging/preset.ts"],"names":[],"mappings":"OAAO,EAAE,MAAM,IAAI,UAAU,EAAE;OACxB,EAAE,qBAAqB,EAAE;OACzB,EAAE,MAAM,EAAE;AAEjB,qBAAa,MAAO,YAAW,MAAM,CAAC,UAAU,CAAC;IAC7C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAa;IAExC,OAAO;IAIP,MAAM,CAAC,OAAO,IAAI,MAAM;IAKxB,MAAM,CAAC,cAAc,CAAC,OAAO,EAAE,qBAAqB,GAAG,MAAM;IAS7D,MAAM,CAAC,aAAa,IAAI,MAAM;IAK9B,MAAM,IAAI,UAAU;CAGvB"}
+{"version":3,"file":"preset.d.mts","sourceRoot":"","sources":["../src/bridging/preset.ts"],"names":[],"mappings":"OAAO,EAAE,MAAM,IAAI,UAAU,EAAE;OACxB,EAAE,qBAAqB,EAAE;OACzB,EAAE,MAAM,EAAE;AAEjB,qBAAa,MAAO,YAAW,MAAM,CAAC,UAAU,CAAC;IAC7C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAa;IAExC,OAAO;IAIP,MAAM,CAAC,OAAO,IAAI,MAAM;IAKxB,MAAM,CAAC,cAAc,CAAC,OAAO,EAAE,qBAAqB,GAAG,MAAM;IAS7D,MAAM,CAAC,aAAa,IAAI,MAAM;IAK9B,MAAM,CAAC,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,wBAAwB,EAAE,MAAM,GAAG,MAAM;IASjF,MAAM,CAAC,IAAI,IAAI,MAAM;IAKrB,MAAM,IAAI,UAAU;CAGvB"}

package/bridging/preset.d.ts

@@ -7,6 +7,8 @@ export declare class Preset implements ToNapi<NapiPreset> {
     static general(): Preset;
     static classification(feature: ClassificationFeature): Preset;
     static summarization(): Preset;
+    static nGramSpeculator(useCase: string, numberOfSpeculatedTokens: number): Preset;
+    static chat(): Preset;
     toNapi(): NapiPreset;
 }
 //# sourceMappingURL=preset.d.ts.map

package/bridging/preset.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"preset.d.ts","sourceRoot":"","sources":["../src/bridging/preset.ts"],"names":[],"mappings":"OAAO,EAAE,MAAM,IAAI,UAAU,EAAE;OACxB,EAAE,qBAAqB,EAAE;OACzB,EAAE,MAAM,EAAE;AAEjB,qBAAa,MAAO,YAAW,MAAM,CAAC,UAAU,CAAC;IAC7C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAa;IAExC,OAAO;IAIP,MAAM,CAAC,OAAO,IAAI,MAAM;IAKxB,MAAM,CAAC,cAAc,CAAC,OAAO,EAAE,qBAAqB,GAAG,MAAM;IAS7D,MAAM,CAAC,aAAa,IAAI,MAAM;IAK9B,MAAM,IAAI,UAAU;CAGvB"}
+{"version":3,"file":"preset.d.ts","sourceRoot":"","sources":["../src/bridging/preset.ts"],"names":[],"mappings":"OAAO,EAAE,MAAM,IAAI,UAAU,EAAE;OACxB,EAAE,qBAAqB,EAAE;OACzB,EAAE,MAAM,EAAE;AAEjB,qBAAa,MAAO,YAAW,MAAM,CAAC,UAAU,CAAC;IAC7C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAa;IAExC,OAAO;IAIP,MAAM,CAAC,OAAO,IAAI,MAAM;IAKxB,MAAM,CAAC,cAAc,CAAC,OAAO,EAAE,qBAAqB,GAAG,MAAM;IAS7D,MAAM,CAAC,aAAa,IAAI,MAAM;IAK9B,MAAM,CAAC,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,wBAAwB,EAAE,MAAM,GAAG,MAAM;IASjF,MAAM,CAAC,IAAI,IAAI,MAAM;IAKrB,MAAM,IAAI,UAAU;CAGvB"}

package/bridging/preset.js

@@ -21,6 +21,18 @@ class Preset {
         const napiPreset = { type: 'Summarization' };
         return new Preset(napiPreset);
     }
+    static nGramSpeculator(useCase, numberOfSpeculatedTokens) {
+        const napiPreset = {
+            type: 'NGramSpeculator',
+            useCase,
+            numberOfSpeculatedTokens,
+        };
+        return new Preset(napiPreset);
+    }
+    static chat() {
+        const napiPreset = { type: 'Chat' };
+        return new Preset(napiPreset);
+    }
     toNapi() {
         return this.napiPreset;
     }

package/bridging/preset.js.map

@@ -1 +1 @@
-{"version":3,"file":"preset.js","sourceRoot":"","sources":["../src/bridging/preset.ts"],"names":[],"mappings":";;;AAIA,MAAa,MAAM;IAGf,YAAoB,UAAsB;QACtC,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IACjC,CAAC;IAED,MAAM,CAAC,OAAO;QACV,MAAM,UAAU,GAAe,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;QACnD,OAAO,IAAI,MAAM,CAAC,UAAU,CAAC,CAAC;IAClC,CAAC;IAED,MAAM,CAAC,cAAc,CAAC,OAA8B;QAChD,MAAM,yBAAyB,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;QACnD,MAAM,UAAU,GAAe;YAC3B,IAAI,EAAE,gBAAgB;YACtB,OAAO,EAAE,yBAAyB;SACrC,CAAC;QACF,OAAO,IAAI,MAAM,CAAC,UAAU,CAAC,CAAC;IAClC,CAAC;IAED,MAAM,CAAC,aAAa;QAChB,MAAM,UAAU,GAAe,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC;QACzD,OAAO,IAAI,MAAM,CAAC,UAAU,CAAC,CAAC;IAClC,CAAC;IAED,MAAM;QACF,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;CACJ;AA7BD,wBA6BC"}
+{"version":3,"file":"preset.js","sourceRoot":"","sources":["../src/bridging/preset.ts"],"names":[],"mappings":";;;AAIA,MAAa,MAAM;IAGf,YAAoB,UAAsB;QACtC,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IACjC,CAAC;IAED,MAAM,CAAC,OAAO;QACV,MAAM,UAAU,GAAe,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;QACnD,OAAO,IAAI,MAAM,CAAC,UAAU,CAAC,CAAC;IAClC,CAAC;IAED,MAAM,CAAC,cAAc,CAAC,OAA8B;QAChD,MAAM,yBAAyB,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;QACnD,MAAM,UAAU,GAAe;YAC3B,IAAI,EAAE,gBAAgB;YACtB,OAAO,EAAE,yBAAyB;SACrC,CAAC;QACF,OAAO,IAAI,MAAM,CAAC,UAAU,CAAC,CAAC;IAClC,CAAC;IAED,MAAM,CAAC,aAAa;QAChB,MAAM,UAAU,GAAe,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC;QACzD,OAAO,IAAI,MAAM,CAAC,UAAU,CAAC,CAAC;IAClC,CAAC;IAED,MAAM,CAAC,eAAe,CAAC,OAAe,EAAE,wBAAgC;QACpE,MAAM,UAAU,GAAe;YAC3B,IAAI,EAAE,iBAAiB;YACvB,OAAO;YACP,wBAAwB;SAC3B,CAAC;QACF,OAAO,IAAI,MAAM,CAAC,UAAU,CAAC,CAAC;IAClC,CAAC;IAED,MAAM,CAAC,IAAI;QACP,MAAM,UAAU,GAAe,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;QAChD,OAAO,IAAI,MAAM,CAAC,UAAU,CAAC,CAAC;IAClC,CAAC;IAED,MAAM;QACF,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;CACJ;AA3CD,wBA2CC"}

package/bridging/preset.mjs

@@ -18,6 +18,18 @@ export class Preset {
         const napiPreset = { type: 'Summarization' };
         return new Preset(napiPreset);
     }
+    static nGramSpeculator(useCase, numberOfSpeculatedTokens) {
+        const napiPreset = {
+            type: 'NGramSpeculator',
+            useCase,
+            numberOfSpeculatedTokens,
+        };
+        return new Preset(napiPreset);
+    }
+    static chat() {
+        const napiPreset = { type: 'Chat' };
+        return new Preset(napiPreset);
+    }
     toNapi() {
         return this.napiPreset;
     }

package/bridging/preset.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"preset.mjs","sourceRoot":"","sources":["../src/bridging/preset.ts"],"names":[],"mappings":"AAIA,MAAM,OAAO,MAAM;IAGf,YAAoB,UAAsB;QACtC,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IACjC,CAAC;IAED,MAAM,CAAC,OAAO;QACV,MAAM,UAAU,GAAe,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;QACnD,OAAO,IAAI,MAAM,CAAC,UAAU,CAAC,CAAC;IAClC,CAAC;IAED,MAAM,CAAC,cAAc,CAAC,OAA8B;QAChD,MAAM,yBAAyB,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;QACnD,MAAM,UAAU,GAAe;YAC3B,IAAI,EAAE,gBAAgB;YACtB,OAAO,EAAE,yBAAyB;SACrC,CAAC;QACF,OAAO,IAAI,MAAM,CAAC,UAAU,CAAC,CAAC;IAClC,CAAC;IAED,MAAM,CAAC,aAAa;QAChB,MAAM,UAAU,GAAe,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC;QACzD,OAAO,IAAI,MAAM,CAAC,UAAU,CAAC,CAAC;IAClC,CAAC;IAED,MAAM;QACF,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;CACJ"}
+{"version":3,"file":"preset.mjs","sourceRoot":"","sources":["../src/bridging/preset.ts"],"names":[],"mappings":"AAIA,MAAM,OAAO,MAAM;IAGf,YAAoB,UAAsB;QACtC,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IACjC,CAAC;IAED,MAAM,CAAC,OAAO;QACV,MAAM,UAAU,GAAe,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;QACnD,OAAO,IAAI,MAAM,CAAC,UAAU,CAAC,CAAC;IAClC,CAAC;IAED,MAAM,CAAC,cAAc,CAAC,OAA8B;QAChD,MAAM,yBAAyB,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;QACnD,MAAM,UAAU,GAAe;YAC3B,IAAI,EAAE,gBAAgB;YACtB,OAAO,EAAE,yBAAyB;SACrC,CAAC;QACF,OAAO,IAAI,MAAM,CAAC,UAAU,CAAC,CAAC;IAClC,CAAC;IAED,MAAM,CAAC,aAAa;QAChB,MAAM,UAAU,GAAe,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC;QACzD,OAAO,IAAI,MAAM,CAAC,UAAU,CAAC,CAAC;IAClC,CAAC;IAED,MAAM,CAAC,eAAe,CAAC,OAAe,EAAE,wBAAgC;QACpE,MAAM,UAAU,GAAe;YAC3B,IAAI,EAAE,iBAAiB;YACvB,OAAO;YACP,wBAAwB;SAC3B,CAAC;QACF,OAAO,IAAI,MAAM,CAAC,UAAU,CAAC,CAAC;IAClC,CAAC;IAED,MAAM,CAAC,IAAI;QACP,MAAM,UAAU,GAAe,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;QAChD,OAAO,IAAI,MAAM,CAAC,UAAU,CAAC,CAAC;IAClC,CAAC;IAED,MAAM;QACF,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;CACJ"}

package/napi/uzu.d.ts

@@ -10,7 +10,7 @@ export declare class Engine {
   getChatModels(types: Array<ModelType>): Promise<Array<ChatModel>>
   getModelDownloadState(repoId: string): ModelDownloadState
   createModelDownloadHandle(repoId: string): ModelDownloadHandle
-  benchmark(task: BenchmarksTask): Promise<Array<BenchmarksResult>>
+  benchmark(task: BenchmarksTask, prefillStepSize?: number | undefined | null): Promise<Array<BenchmarksResult>>
   constructor()
   createChatSession(model: ChatModel, config: Config): ChatSession
   registerLicenseStatusHandler(callback?: ((arg: LicenseStatus) => void) | undefined | null): void
@@ -203,6 +203,8 @@ export type Preset =
   | { type: 'General' }
   | { type: 'Classification', feature: ClassificationFeature }
   | { type: 'Summarization' }
+  | { type: 'NGramSpeculator', useCase: string, numberOfSpeculatedTokens: number }
+  | { type: 'Chat' }
 
 export declare const enum Role {
   System = 0,

package/package.json

@@ -101,5 +101,5 @@
   },
   "type": "commonjs",
   "types": "./index.d.ts",
-  "version": "0.2.6"
+  "version": "0.2.20"
 }

package/src/bridging/preset.ts

@@ -28,6 +28,20 @@ export class Preset implements ToNapi<NapiPreset> {
         return new Preset(napiPreset);
     }
 
+    static nGramSpeculator(useCase: string, numberOfSpeculatedTokens: number): Preset {
+        const napiPreset: NapiPreset = {
+            type: 'NGramSpeculator',
+            useCase,
+            numberOfSpeculatedTokens,
+        };
+        return new Preset(napiPreset);
+    }
+
+    static chat(): Preset {
+        const napiPreset: NapiPreset = { type: 'Chat' };
+        return new Preset(napiPreset);
+    }
+
     toNapi(): NapiPreset {
         return this.napiPreset;
     }

package/src/napi/uzu.d.ts

@@ -10,7 +10,7 @@ export declare class Engine {
   getChatModels(types: Array<ModelType>): Promise<Array<ChatModel>>
   getModelDownloadState(repoId: string): ModelDownloadState
   createModelDownloadHandle(repoId: string): ModelDownloadHandle
-  benchmark(task: BenchmarksTask): Promise<Array<BenchmarksResult>>
+  benchmark(task: BenchmarksTask, prefillStepSize?: number | undefined | null): Promise<Array<BenchmarksResult>>
   constructor()
   createChatSession(model: ChatModel, config: Config): ChatSession
   registerLicenseStatusHandler(callback?: ((arg: LicenseStatus) => void) | undefined | null): void
@@ -203,6 +203,8 @@ export type Preset =
   | { type: 'General' }
   | { type: 'Classification', feature: ClassificationFeature }
   | { type: 'Summarization' }
+  | { type: 'NGramSpeculator', useCase: string, numberOfSpeculatedTokens: number }
+  | { type: 'Chat' }
 
 export declare const enum Role {
   System = 0,

package/src/version.ts

@@ -1 +1 @@
-export const VERSION = '0.2.6';
+export const VERSION = '0.2.20';

package/version.d.mts

@@ -1,2 +1,2 @@
-export declare const VERSION = "0.2.6";
+export declare const VERSION = "0.2.20";
 //# sourceMappingURL=version.d.mts.map

package/version.d.mts.map

@@ -1 +1 @@
-{"version":3,"file":"version.d.mts","sourceRoot":"","sources":["src/version.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,OAAO,UAAU,CAAC"}
+{"version":3,"file":"version.d.mts","sourceRoot":"","sources":["src/version.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,OAAO,WAAW,CAAC"}

package/version.d.ts

@@ -1,2 +1,2 @@
-export declare const VERSION = "0.2.6";
+export declare const VERSION = "0.2.20";
 //# sourceMappingURL=version.d.ts.map

package/version.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"version.d.ts","sourceRoot":"","sources":["src/version.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,OAAO,UAAU,CAAC"}
+{"version":3,"file":"version.d.ts","sourceRoot":"","sources":["src/version.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,OAAO,WAAW,CAAC"}

package/version.js

@@ -1,5 +1,5 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.VERSION = void 0;
-exports.VERSION = '0.2.6';
+exports.VERSION = '0.2.20';
 //# sourceMappingURL=version.js.map

package/version.js.map

@@ -1 +1 @@
-{"version":3,"file":"version.js","sourceRoot":"","sources":["src/version.ts"],"names":[],"mappings":";;;AAAa,QAAA,OAAO,GAAG,OAAO,CAAC"}
+{"version":3,"file":"version.js","sourceRoot":"","sources":["src/version.ts"],"names":[],"mappings":";;;AAAa,QAAA,OAAO,GAAG,QAAQ,CAAC"}

package/version.mjs

@@ -1,2 +1,2 @@
-export const VERSION = '0.2.6';
+export const VERSION = '0.2.20';
 //# sourceMappingURL=version.mjs.map

package/version.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"version.mjs","sourceRoot":"","sources":["src/version.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,OAAO,GAAG,OAAO,CAAC"}
+{"version":3,"file":"version.mjs","sourceRoot":"","sources":["src/version.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,OAAO,GAAG,QAAQ,CAAC"}