@degreesign/chat 0.2.0 → 0.3.1

package/dist/node/engine-litert.node.min.mjs

@@ -0,0 +1,2 @@
+/*! For license information please see engine-litert.node.min.mjs.LICENSE.txt */
+export const __webpack_esm_ids__=[520];export const __webpack_esm_modules__={507(e,t,s){s.r(t),s.d(t,{AutoToolChat:()=>R,Backend:()=>U,Conversation:()=>$,EmbeddingEngine:()=>ce,Engine:()=>te,InputOverflowStrategy:()=>N,LiteRtLm:()=>g,LiteRtLmNotLoadedError:()=>i,SUPPORTED_AUDIO_MIME_TYPES:()=>ne,SUPPORTED_IMAGE_MIME_TYPES:()=>se,SamplerType:()=>G,Session:()=>Q,Wasm:()=>n,fillWasmEmbeddingEngineSettingsFromEmbeddingEngineSettings:()=>ie,fillWasmEngineSettingsFromEngineSettings:()=>W,getGlobalLiteRtLm:()=>o,getGlobalLiteRtLmPromise:()=>u,getOrLoadGlobalLiteRtLm:()=>T,hasGlobalLiteRtLm:()=>c,hasGlobalLiteRtLmPromise:()=>d,isWebMcpTool:()=>B,loadLiteRtLm:()=>E,sessionConfigToWasmSessionConfig:()=>K,setGlobalLiteRtLm:()=>l,setGlobalLiteRtLmPromise:()=>h,setupDefaultWebGpuDevice:()=>m,toTool:()=>C,unloadLiteRtLm:()=>_,wasmEngineSettingsToEngineSettings:()=>q,wasmSessionConfigToSessionConfig:()=>Y});var n={};s.r(n),s.d(n,{Backend:()=>U,InputOverflowStrategy:()=>N,ReadableStreamDataStreamWrapper:()=>I,SamplerType:()=>G});class i extends Error{constructor(){super("LiteRT-LM is not initialized yet. Please call loadLiteRtLm() and wait for its promise to resolve to load the LiteRT-LM WASM module.")}}let a,r;function o(){if(!a)throw new i;return a}function c(){return Boolean(a)}function l(e){a=e}function u(){return r}function d(){return Boolean(r)}function h(e){r=e}async function m(){await o().setupDefaultWebGpuDevice()}class g{liteRtLmWasm;static DEFAULT_WASM_PATH="https://cdn.jsdelivr.net/npm/@litert-lm/core@0.19.0-nightly.20261024/wasm";constructor(e){this.liteRtLmWasm=e,this.liteRtLmWasm.setupLogging()}async setupDefaultWebGpuDevice(){this.liteRtLmWasm.preinitializedWebGPUDevice||(this.liteRtLmWasm.preinitializedWebGPUDevice=await async function(){const e={powerPreference:"high-performance"},t=await navigator.gpu.requestAdapter(e);if(!t)throw new Error("No GPU adapter found.");const s={maxBufferSize:t.limits.maxBufferSize,maxStorageBufferBindingSize:t.limits.maxStorageBufferBindingSize,maxStorageBuffersPerShaderStage:t.limits.maxStorageBuffersPerShaderStage,maxTextureDimension2D:t.limits.maxTextureDimension2D},n=[];for(const e of f)t.features.has(e)&&n.push(e);return await t.requestDevice({requiredFeatures:n,requiredLimits:s})}())}delete(){}}const f=["shader-f16","subgroups"];async function p(e){if("function"!=typeof importScripts){const t=document.createElement("script");return t.src=e.toString(),t.crossOrigin="anonymous",new Promise((e,s)=>{t.addEventListener("load",()=>{e()},!1),t.addEventListener("error",e=>{s(e)},!1),document.body.appendChild(t)})}importScripts(e.toString())}const y=new Uint8Array([0,97,115,109,1,0,0,0,1,5,1,96,0,1,123,3,2,1,0,10,15,1,13,0,65,1,253,15,65,2,253,15,253,128,2,11]),w=new Uint8Array([0,97,115,109,1,0,0,0,1,4,1,96,0,0,3,2,1,0,5,4,1,3,1,1,10,11,1,9,0,65,0,254,16,2,0,26,11]),b={relaxedSimd:void 0,threads:void 0,jspi:void 0};async function v(e){try{return await WebAssembly.instantiate(e),{supported:!0}}catch(e){return{supported:!1,error:e}}}const S={relaxedSimd:()=>(void 0===b.relaxedSimd&&(b.relaxedSimd=v(y)),b.relaxedSimd),threads:()=>{if(void 0===b.threads)try{"undefined"!=typeof MessageChannel&&(new MessageChannel).port1.postMessage(new SharedArrayBuffer(1)),b.threads=v(w)}catch(e){b.threads=Promise.resolve({supported:!1,error:e})}return b.threads},jspi:()=>{if(void 0===b.jspi)try{const e="Suspending"in WebAssembly;b.jspi=Promise.resolve({supported:e,error:e?void 0:new Error("JSPI is not supported")})}catch(e){b.jspi=Promise.resolve({supported:!1,error:e})}return b.jspi}};async function k(e){const t=S[e]?.();if(!t)throw new Error(`Unknown feature: ${e}`);return(await t).supported}async function x(e,t){const s=e,n=await k("relaxedSimd"),i=await k("jspi");let a="";a=n?i?"litertlm_wasm_internal.js":"litertlm_wasm_asyncify_internal.js":i?"litertlm_wasm_compat_internal.js":"litertlm_wasm_compat_asyncify_internal.js";let r=e;if(s.endsWith(".wasm"))throw new Error("Please load the `.js` file corresponding to the `.wasm` file, or load the directory containing it.");return s.endsWith(".js")||(r=function(e,t){return e?t?(e.endsWith("/")?e:e+"/")+(t.startsWith("/")?t.substring(1):t):e:t}(e,a)),(async(e,t,s,n,i)=>{if(t&&await p(t),!self.ModuleFactory)throw new Error("ModuleFactory not set.");if(s&&(await p(s),!self.ModuleFactory))throw new Error("ModuleFactory not set.");if(self.Module&&i){const e=self.Module;e.locateFile=i.locateFile,i.mainScriptUrlOrBlob&&(e.mainScriptUrlOrBlob=i.mainScriptUrlOrBlob)}const a=await self.ModuleFactory(self.Module||i);return self.ModuleFactory=self.Module=void 0,new e(a,n)})(g,r)}let P;function E(e,t){if(d())throw new Error("LiteRT-LM is already loading / loaded.");return h(x(e).then(async e=>(l(e),e)).catch(e=>{throw h(void 0),e})),u()}function _(){if(d()&&!c())throw new Error("LiteRT-LM is loading and can not be unloaded or canceled until it is finished loading.");c()&&(o().delete(),l(void 0),P=void 0),h(void 0)}function T(e){const t=u();if(t){if(e&&P!==e)throw new Error(`LiteRT-LM is already loading / loaded with a different path ${P} but was requested to load ${e}. To reload with a different path, call unloadLiteRtLm() first.`);return t}return P=e??g.DEFAULT_WASM_PATH,t??E(P)}function B(e){return"inputSchema"in e}function C(e){const{inputSchema:t,execute:s,...n}=e;return{type:"function",function:{...n,parameters:t}}}const A="Conversation is busy. A generation is already in progress.";class R{options;baseConversation=null;isBusy=!1;isCancelled=!1;tools=new Map;nextToolCallId=1;currentBatch=null;constructor(e){if(this.options=e,e.config?.preface?.tools&&e.config.preface.tools.length>0)throw new Error("AutoToolChat handles tools itself. Do not provide tools via ConversationConfig.preface.tools.");const t=e.tools||[];for(const e of t){let t;if(B(e))t={...C(e),execute:e.execute};else if("function"in e)t=e;else{const{execute:s,...n}=e;t={type:"function",function:n,execute:s}}const s=t.function.name;if(!s)throw new Error("Tool must have a name.");if(!t.execute)throw new Error(`Tool ${s} must have an execute function.`);if(this.tools.has(s))throw new Error(`Duplicate tool name ${s} provided.`);this.tools.set(s,t)}}async getConversation(){if(!this.baseConversation){const e=this.options.config?{...this.options.config}:{};this.tools.size>0&&(e.preface||(e.preface={}),e.preface.tools=Array.from(this.tools.values()).map(e=>{const{execute:t,...s}=e;return s})),this.baseConversation=await this.options.engine.createConversation(e)}return this.baseConversation}async sendMessage(e){if(this.isBusy)throw new Error(A);this.isBusy=!0,this.isCancelled=!1;try{const t=await this.getConversation();let s=e;const n=this.options.recurringToolCallLimit??25;let i=0;for(;i<n;){i++;const e=await t.sendMessage(s);if(e.tool_calls&&e.tool_calls.length>0){const t=await this.executeToolCalls(e.tool_calls);if(this.isCancelled)throw new Error("Conversation cancelled");s={role:"tool",content:t};continue}return e}throw new Error("Tool calling exceeded the recurring limit")}finally{this.isBusy=!1}}sendMessageStreaming(e){if(this.isBusy)throw new Error(A);this.isBusy=!0,this.isCancelled=!1;let t=!1;return new ReadableStream({start:s=>{let n=e,i=0;const a=this.options.recurringToolCallLimit??25,r=async()=>{if(i>=a)return this.isBusy=!1,void s.error(new Error("Tool calling exceeded the recurring limit"));i++;let e=[];try{const i=(await this.getConversation()).sendMessageStreaming(n);for await(const n of i){if(t||this.isCancelled)break;if(n.tool_calls&&n.tool_calls.length>0){e=e.concat(n.tool_calls);const t={...n};delete t.tool_calls,t.content&&s.enqueue(t)}else s.enqueue(n)}}catch(e){return this.isBusy=!1,void s.error(e)}if(!t&&!this.isCancelled){if(e.length>0){const s=await this.executeToolCalls(e);if(t||this.isCancelled)return;return n={role:"tool",content:s},void await r()}this.isBusy=!1,s.close()}};r().catch(e=>{t||this.isCancelled||(this.isBusy=!1,s.error(e))})},cancel:()=>{t=!0,this.cancel()}})}cancel(){this.isCancelled=!0,this.currentBatch&&this.currentBatch.cancel(),this.baseConversation&&this.baseConversation.cancel()}async getHistory(){return(await this.getConversation()).getHistory()}async getTokenCount(){return(await this.getConversation()).getTokenCount()}async getBenchmarkInfo(){return(await this.getConversation()).getBenchmarkInfo()}async delete(){this.baseConversation&&(await this.baseConversation.delete(),this.baseConversation=null)}async executeToolCalls(e){this.currentBatch=new M(this.tools,this.options.onToolProgress,()=>this.nextToolCallId++);try{return await this.currentBatch.executeAll(e)}finally{this.currentBatch=null}}}class M{tools;onToolProgress;getNextId;isCancelled=!1;constructor(e,t,s){this.tools=e,this.onToolProgress=t,this.getNextId=s}cancel(){this.isCancelled=!0}async executeAll(e){return Promise.all(e.map(async e=>{const t=this.tools.get(e.function.name),s=this.getNextId();let n;if(this.onToolProgress&&this.onToolProgress({id:s,name:e.function.name,arguments:e.function.arguments,status:"started"}),t)try{n=await t.execute(e.function.arguments),!this.isCancelled&&this.onToolProgress&&this.onToolProgress({id:s,name:e.function.name,arguments:e.function.arguments,status:"completed",result:n})}catch(t){const i=String(t);n={error:`Error executing tool ${e.function.name}: ${i}`},!this.isCancelled&&this.onToolProgress&&this.onToolProgress({id:s,name:e.function.name,arguments:e.function.arguments,status:"error",error:i})}else{const t=`Tool ${e.function.name} not found.`;n={error:t},!this.isCancelled&&this.onToolProgress&&this.onToolProgress({id:s,name:e.function.name,arguments:e.function.arguments,status:"error",error:t})}return{type:"tool_response",name:e.function.name,response:n}}))}}class L{size;data;fullyDiscarded=!1;discardedRegions=[];constructor(e){this.size=e}discard(e,t){if(this.fullyDiscarded)return;if(0===this.discardedRegions.length)return this.discardedRegions.push([e,t]),void this.checkFullyDiscarded();this.discardedRegions.push([e,t]),this.discardedRegions.sort((e,t)=>e[0]-t[0]);const s=[];for(const e of this.discardedRegions){const t=s[s.length-1];!t||t[1]<e[0]?s.push(e):t[1]=Math.max(t[1],e[1])}this.discardedRegions=s,this.checkFullyDiscarded()}getDiscardedRegions(e=0,t=this.size){const s=[];for(const[n,i]of this.discardedRegions){if(n>=t)break;i<=e||s.push([n,i])}return s}checkFullyDiscarded(){1===this.discardedRegions.length&&0===this.discardedRegions[0][0]&&this.discardedRegions[0][1]===this.size&&(this.fullyDiscarded=!0,this.data=void 0)}}class I{stream;getWasmHeap;bytesPerSlice;slices=[];streamDone=!1;globalPosition=0;reader;leftoverData;isReading=!1;constructor(e,t,s=1e7){this.stream=e,this.getWasmHeap=t,this.bytesPerSlice=s;try{this.reader={byobReader:this.stream.getReader({mode:"byob"})}}catch(e){this.reader={defaultReader:this.stream.getReader()}}}*getViews(e,t){if(e>=t)return;let s,n,i=e;for(;i<t;){const e=Math.floor(i/this.bytesPerSlice),a=i%this.bytesPerSlice,r=e*this.bytesPerSlice;let o=this.slices[e];o||(s&&n&&n.fullyDiscarded||(s=new Uint8Array(this.bytesPerSlice)),o=new L(this.bytesPerSlice),o.data=s,this.slices[e]=o);const c=Math.min(this.bytesPerSlice,t-r),l=()=>{o.discard(a,c)},u=c-a,d=e=>{o.data=new Uint8Array(e.buffer,0,e.buffer.byteLength),s=o.data},h=()=>{const t=o.getDiscardedRegions(a,c);if(t.length>0){const s=t.map(([e,t])=>`[${e+r}, ${t+r}]`);throw new Error(`Slice ${e} for data at ${i} overlaps discarded regions: [${s.join(", ")}]`)}if(!o.data)throw new Error(`No ArrayBuffer for slice ${e} at position ${i}`)},m=()=>(h(),new Uint8Array(o.data.buffer,o.data.byteOffset+a,o.data.byteOffset+u)),g=()=>{if(o.data)return new Uint8Array(o.data.buffer,o.data.byteOffset+a,u)};yield{getView:m,getUncheckedView:g,discardView:l,setView:d,checkReadable:h,viewLength:u},n=o,i+=u}}async readInternal(e,t,s,n){if(this.isReading)throw new Error("Concurrent reads are not supported");this.isReading=!0;try{let i;for(const{getUncheckedView:e,setView:s,viewLength:n}of this.getViews(this.globalPosition,t)){const t=e();let a;t?a=t:((!i||i.byteLength<n)&&(i=new ArrayBuffer(this.bytesPerSlice)),a=new Uint8Array(i,0,n));const r=await this.fillBufferFromStream(a);if(t?s(r):i=r.buffer,0===r.length)break}const a=Math.min(this.globalPosition,t+s);for(const{getView:s,discardView:i}of this.getViews(t,a)){const t=s();this.getWasmHeap().set(t,e),n&&i(),e+=t.length}const r=Math.max(a,t);for(const{getView:i,setView:a,discardView:o}of this.getViews(r,t+s)){if(this.streamDone&&this.globalPosition<=r)break;const t=await this.fillBufferFromStream(i());a(t),this.getWasmHeap().set(t,e),n&&o(),e+=t.length}}finally{this.isReading=!1}}async fillBufferFromStream(e){const t=e.byteLength;if(0===t)return e;const s=e.byteOffset;let n=0,i=e.buffer;for(;n<t;){if(this.streamDone&&(!this.leftoverData||0===this.leftoverData.length))throw new Error(`Read from stream returned early EOF at position ${this.globalPosition}. Expected ${t-n} more bytes.`);if(this.reader.byobReader){const e=new Uint8Array(i,s+n,t-n),a=await this.reader.byobReader.read(e);if(a.done&&(this.streamDone=!0),!a.value)throw new Error("Read from stream returned null value");this.globalPosition+=a.value.byteLength,n+=a.value.byteLength,i=a.value.buffer}else{let e;if(this.leftoverData&&this.leftoverData.length>0)e=this.leftoverData,this.leftoverData=void 0;else{const t=await this.reader.defaultReader.read();if(t.done&&(this.streamDone=!0,!t.value))continue;if(!t.value)throw new Error("Read from stream returned null value");e=t.value}const a=t-n,r=Math.min(e.length,a);new Uint8Array(i,s+n,r).set(e.subarray(0,r)),e.length>r&&(this.leftoverData=e.subarray(r)),this.globalPosition+=r,n+=r}}const a=new Uint8Array(i,s,n);if(a.length!==t)throw new Error(`Read from stream returned ${a.length} bytes, expected ${t}`);return a}doDiscard(e,t){for(const{discardView:s}of this.getViews(e,e+t))s()}async readAndDiscard(e,t,s){try{return await this.readInternal(e>>>0,D(t),D(s),!0),{}}catch(e){return{error:e}}}async readAndPreserve(e,t,s){try{return await this.readInternal(e>>>0,D(t),D(s),!1),{}}catch(e){return{error:e}}}discard(e,t){try{return this.doDiscard(D(e),D(t)),{}}catch(e){return{error:e}}}}function D(e){if("number"==typeof e)return e;if(e>Number.MAX_SAFE_INTEGER)throw new Error(`BigInt value ${e} too large to be represented as a number`);return Number(e)}const U={UNSPECIFIED:0,CPU_ARTISAN:1,GPU_ARTISAN:2,CPU:3,GPU:4,GOOGLE_TENSOR_ARTISAN:5,NPU:6},G={TYPE_UNSPECIFIED:0,TOP_K:1,TOP_P:2,GREEDY:3},N={CHUNK_AND_AVERAGE:0,TRUNCATE:1,ERROR:2};function O(e){const t=new Array(e.size());for(let s=0;s<e.size();++s)t[s]=e.get(s);return e.delete(),t}function V(e,t){for(const s of e)t.push_back(s)}function W(e,t,s,n=o().liteRtLmWasm){const i=e.getMutableMainExecutorSettings();if(i.setCacheDir(":nocache"),t.benchmarkEnabled&&e.enableBenchmark(),void 0!==t.maxVisionTokensPerImage&&e.setMaxVisionTokensPerImage(t.maxVisionTokensPerImage),void 0!==t.minVisionTokensPerImage&&e.setMinVisionTokensPerImage(t.minVisionTokensPerImage),t.mainExecutorSettings){const e=t.mainExecutorSettings;if(void 0!==e.maxNumTokens&&i.setMaxNumTokens(e.maxNumTokens),void 0!==e.maxNumImages&&i.setMaxNumImages(e.maxNumImages),void 0!==e.samplerBackend&&i.setSamplerBackend({value:e.samplerBackend}),void 0!==e.backendConfig){const t=e.backendConfig;if(s===U.CPU){const e=t,s={kv_increment_size:e.kv_increment_size??0,prefill_chunk_size:e.prefill_chunk_size??0,number_of_threads:e.number_of_threads??0};i.setBackendConfigCpu(s)}else if(s===U.GPU){const e=t,s={max_top_k:e.max_top_k??1,external_tensor_mode:e.external_tensor_mode??!1};i.setBackendConfigGpu(s)}else if(s===U.GPU_ARTISAN){const e=t,s=new n.VectorUint32;V(e.supported_lora_ranks??[],s);const a={num_output_candidates:e.num_output_candidates??1,wait_for_weight_uploads:e.wait_for_weight_uploads??!0,num_decode_steps_per_sync:e.num_decode_steps_per_sync??1,sequence_batch_size:e.sequence_batch_size??0,supported_lora_ranks:s,max_top_k:e.max_top_k??1,enable_decode_logits:e.enable_decode_logits??!1,enable_external_embeddings:e.enable_external_embeddings??!1,use_submodel:e.use_submodel??!0,use_autosized_ringbuffers:e.use_autosized_ringbuffers??!0};i.setBackendConfigGpuArtisan(a),s.delete()}}if(void 0!==e.advancedSettings){const t={...i.getAdvancedSettings(),...e.advancedSettings};i.setAdvancedSettings(t)}}}function q(e){const t=e.getMutableMainExecutorSettings();let s,n;try{s=t.getBackendConfigCpu(),n=U.CPU}catch(e){try{s=t.getBackendConfigGpu(),n=U.GPU}catch(e){try{const e=t.getBackendConfigGpuArtisan();s={...e,supported_lora_ranks:O(e.supported_lora_ranks)},n=U.GPU_ARTISAN}catch(e){throw new Error("Unsupported backend config")}}}return{backend:n,visionBackend:e.getVisionBackend()?.value,maxVisionTokensPerImage:e.getMaxVisionTokensPerImage(),minVisionTokensPerImage:e.getMinVisionTokensPerImage(),audioBackend:e.getAudioBackend()?.value,mainExecutorSettings:{maxNumTokens:t.getMaxNumTokens(),maxNumImages:t.getMaxNumImages(),samplerBackend:t.getSamplerBackend().value,backendConfig:s,advancedSettings:t.getAdvancedSettings()},benchmarkEnabled:!1}}class z{callbacks=new Set;add(e){return this.callbacks.add(e),()=>{this.callbacks.delete(e),e()}}run(){for(const e of this.callbacks)e();this.callbacks.clear()}}const F="Conversation is busy. A generation is already in progress.";class ${conversation;engine;mutexes;visualTokenBudget;isBusy=!1;constructor(e,t,s,n=0){this.conversation=e,this.engine=t,this.mutexes=s,this.visualTokenBudget=n}async sendMessage(e){if(this.isBusy)throw new Error(F);this.isBusy=!0;try{const{json:t,blobs:s}=await j(e);return await this.mutexes.executor.acquireAndRun(async()=>{const e=await this.conversation.sendMessage(t,s,this.visualTokenBudget);return JSON.parse(e)})}finally{this.isBusy=!1}}sendMessageStreaming(e){if(this.isBusy)throw new Error(F);this.isBusy=!0;let t=!1,s=!1;return new ReadableStream({start:n=>{(async()=>{const{json:i,blobs:a}=await j(e);await this.mutexes.executor.acquireAndRun(async()=>{await this.conversation.sendMessageAsync(i,a,(e,i,a)=>{if(!t&&!s){if(a)return s=!0,this.isBusy=!1,void n.error(new Error(a));if(e)try{const t=JSON.parse(e);n.enqueue(t)}catch(e){s=!0,this.isBusy=!1,n.error(e)}}},this.visualTokenBudget),await this.engine.waitUntilDone()}),t||s||(this.isBusy=!1,n.close())})().catch(e=>{t||s||(this.isBusy=!1,n.error(e))})},cancel:()=>{t=!0,this.isBusy=!1,this.cancel()}})}cancel(){this.conversation.cancelProcess()}async getHistory(){return this.mutexes.executor.acquireAndRun(()=>{const e=this.conversation.getHistory();return JSON.parse(e)})}async getTokenCount(){return this.mutexes.executor.acquireAndRun(()=>this.conversation.getTokenCount())}async getBenchmarkInfo(){return this.mutexes.executor.acquireAndRun(()=>this.conversation.getBenchmarkInfo())}async clone(){if(this.isBusy)throw new Error(F);this.isBusy=!0;try{return await this.mutexes.executor.acquireAndRun(()=>{const e=this.conversation.clone();return new $(e,this.engine,this.mutexes,this.visualTokenBudget)})}finally{this.isBusy=!1}}async delete(){await this.mutexes.executor.acquireAndRun(()=>{this.conversation.delete()})}}async function j(e){const t=[],s=await async function(e,t){const s=function(e){if("string"==typeof e)return{role:"user",content:e};return e}(e);if(!Array.isArray(s.content))return s;const n=[];for(const e of s.content){if("image"!==e.type&&"audio"!==e.type||!("data"in e)||void 0===e.data){n.push(e);continue}const{data:s,...i}=e;if("string"==typeof s){n.push({...i,blob:s});continue}const a=await H(s),r=t.length;t.push(a),n.push({...i,blob_index:r})}return{...s,content:n}}(e,t);return{json:JSON.stringify(s),blobs:t}}async function H(e){return e instanceof Blob?new Uint8Array(await e.arrayBuffer()):e instanceof ArrayBuffer?new Uint8Array(e):e}class J{mutexPromise=Promise.resolve();queueCount=0;get isLocked(){return this.queueCount>0}async acquireAndRun(e){this.queueCount++;const t=this.mutexPromise;let s;this.mutexPromise=new Promise(e=>{s=e});try{return await t,await e()}finally{this.queueCount--,s()}}}function K(e,t=o().liteRtLmWasm){const s=t.SessionConfig.createDefault();if(void 0!==e.audioModalityEnabled&&s.setAudioModalityEnabled(e.audioModalityEnabled),void 0!==e.visionModalityEnabled&&s.setVisionModalityEnabled(e.visionModalityEnabled),void 0!==e.samplerParams){const t=s.getMutableSamplerParams();void 0!==e.samplerParams.type&&t.setType({value:e.samplerParams.type}),void 0!==e.samplerParams.k&&t.setK(e.samplerParams.k),void 0!==e.samplerParams.p&&t.setP(e.samplerParams.p),void 0!==e.samplerParams.temperature&&t.setTemperature(e.samplerParams.temperature),void 0!==e.samplerParams.seed&&t.setSeed(e.samplerParams.seed)}if(void 0!==e.startTokenId&&s.setStartTokenId(e.startTokenId),void 0!==e.numOutputCandidates&&s.setNumOutputCandidates(e.numOutputCandidates),void 0!==e.samplerBackend&&s.setSamplerBackend({value:e.samplerBackend}),void 0!==e.applyPromptTemplateInSession&&s.setApplyPromptTemplateInSession(e.applyPromptTemplateInSession),void 0!==e.useExternalSampler&&s.setUseExternalSampler(e.useExternalSampler),void 0!==e.maxOutputTokens&&s.setMaxOutputTokens(e.maxOutputTokens),void 0!==e.stopTokenIds){const n=new t.VectorVectorInt;for(const s of e.stopTokenIds){const e=new t.VectorInt;V(s,e),n.push_back(e),e.delete()}s.setStopTokenIds(n),n.delete()}return s}function Y(e){const t=e.getMutableSamplerParams();return{audioModalityEnabled:e.getAudioModalityEnabled(),visionModalityEnabled:e.getVisionModalityEnabled(),samplerParams:{type:t.type().value,k:t.k(),p:t.p(),temperature:t.temperature(),seed:t.seed()},stopTokenIds:O(e.getStopTokenIds()).map(O),startTokenId:e.getStartTokenId(),numOutputCandidates:e.getNumOutputCandidates(),samplerBackend:e.getSamplerBackend().value,applyPromptTemplateInSession:e.getApplyPromptTemplateInSession(),useExternalSampler:e.getUseExternalSampler(),maxOutputTokens:e.getMaxOutputTokens()}}const X="Session is busy. An operation is already in progress.";class Q{session;mutexes;config;isBusy=!1;constructor(e,t){this.session=e,this.mutexes=t;const s=e.getSessionConfig();this.config=Y(s),s.delete()}async runPrefill(e){if(this.isBusy)throw new Error(X);this.isBusy=!0;try{await this.mutexes.executor.acquireAndRun(async()=>{await this.session.runPrefill(e)})}finally{this.isBusy=!1}}async runDecode(){if(this.isBusy)throw new Error(X);this.isBusy=!0;try{return await this.mutexes.executor.acquireAndRun(async()=>{const e=await this.session.runDecode();return new Z(e)})}finally{this.isBusy=!1}}cancel(){this.session.cancelProcess()}async clone(){if(this.isBusy)throw new Error(X);this.isBusy=!0;try{return await this.mutexes.executor.acquireAndRun(()=>{const e=this.session.clone();return new Q(e,this.mutexes)})}finally{this.isBusy=!1}}async delete(){await this.mutexes.executor.acquireAndRun(()=>{this.session.delete()})}}class Z{responses;constructor(e){this.responses=e}getTexts(){return O(this.responses.getTexts())}delete(){this.responses.delete()}}async function ee(e,t,s){const n=await async function(e){if(e instanceof ReadableStream)return e;if(e instanceof Blob)return e.stream();const t=e,s=await fetch(t,{credentials:"same-origin"});if(!s.ok)throw new Error(`Failed to fetch model file from ${t}`);return s.body}(e),i=new I(n,()=>t.HEAPU8),a=t.ReadableStreamDataStream.create(i);s.add(()=>{a.delete()});const r=t.ModelAssets.createStreaming(a),o=s.add(()=>{r.delete()});return{modelAssets:r,cleanupModelAssets:o}}class te{wasm;engine;deleteCallback;settings;mutexes={executor:new J};constructor(e,t,s,n){this.wasm=e,this.engine=t,this.deleteCallback=n;const i=t.getEngineSettings(),a=q(i);this.settings={...a,model:s},i.delete()}static async create(e,t=""){const s=await T(),n=s.liteRtLmWasm,i=e.backend??U.GPU;e={...e,backend:i};const a=e.mainExecutorSettings?.samplerBackend,r=e.visionBackend,o=e.audioBackend;i!==U.GPU&&i!==U.GPU_ARTISAN&&a!==U.GPU&&a!==U.GPU_ARTISAN&&r!==U.GPU&&o!==U.GPU||await s.setupDefaultWebGpuDevice();const c=new z;let l;try{const{modelAssets:s,cleanupModelAssets:a}=await ee(e.model,n,c),u=void 0===r&&void 0===o?await n.EngineSettings.createDefault(s,{value:i}):await n.EngineSettings.createDefaultMultimodal(s,{value:i},void 0===r?void 0:{value:r},void 0===o?void 0:{value:o});a();const d=c.add(()=>{u.delete()}),h=u.getMutableMainExecutorSettings().getBackend().value;W(u,e,h,n),u.setParallelFileSectionLoading(!1),u.setSingleThreadedExecution(!0);try{l=h===U.GPU_ARTISAN?await n.Engine.createLegacyEngine(u,t):await n.Engine.createEngine(u,t)}finally{if(h===U.GPU||r===U.GPU||o===U.GPU)try{await n.clearStoredWeightsStreams()}catch(e){console.error("Error clearing stored weights streams:",e)}}return d(),c.add(()=>{l.delete()}),new te(n,l,e.model,()=>c.run())}catch(e){throw c.run(),e}}withDefaultModalities(e){const t={...e};return void 0!==this.settings.visionBackend&&void 0===t.visionModalityEnabled&&(t.visionModalityEnabled=!0),void 0!==this.settings.audioBackend&&void 0===t.audioModalityEnabled&&(t.audioModalityEnabled=!0),t}async createSession(e={}){return this.mutexes.executor.acquireAndRun(()=>{const t=K(this.withDefaultModalities(e),this.wasm),s=this.engine.createSession(t);return t.delete(),new Q(s,this.mutexes)})}async createConversation(e){return e||void 0===this.settings.visionBackend&&void 0===this.settings.audioBackend||(e={filterChannelContentFromKvCache:!0}),this.mutexes.executor.acquireAndRun(async()=>{let t,s;if(e){s=K(this.withDefaultModalities(e.sessionConfig||{}),this.wasm);const n=e.preface?JSON.stringify(e.preface):"",i=this.settings.backend===U.GPU_ARTISAN&&Boolean(this.settings.mainExecutorSettings?.backendConfig?.use_autosized_ringbuffers),a=e.filterChannelContentFromKvCache??i;t=this.wasm.ConversationConfig.createCustom(this.engine,s,!!e.enableConstrainedDecoding,!!e.prefillPrefaceOnInit,a,n)}else t=this.wasm.ConversationConfig.createDefault(this.engine);const n=await this.wasm.Conversation.create(this.engine,t);return t.delete(),s&&s.delete(),new $(n,this.engine,this.mutexes,this.settings.maxVisionTokensPerImage??0)})}async delete(){await this.mutexes.executor.acquireAndRun(()=>{this.deleteCallback()})}}const se=new Set(["image/jpeg","image/png","image/bmp","image/gif","image/x-tga","image/vnd.adobe.photoshop","image/vnd.radiance","image/x-portable-pixmap","image/x-portable-graymap"]),ne=new Set(["audio/wav","audio/x-wav","audio/wave","audio/mp3","audio/mpeg","audio/flac","audio/x-flac"]);function ie(e,t){const s=e.getMutableMainExecutorSettings();s.setCacheDir(":nocache");const n=t.maxInputLength??1024;if(e.setMaxInputLength(n),e.setMinInputLength(t.minInputLength??n),void 0!==t.visionTokensPerImage&&e.setVisionTokensPerImage(t.visionTokensPerImage),t.mainExecutorSettings){const e=t.mainExecutorSettings;void 0!==e.numThreads&&s.setNumThreads(e.numThreads)}}function ae(e){return{normalize:e?.normalize??!0,insertSpecialTokens:e?.insertSpecialTokens??!0,inputOverflowStrategy:e?.inputOverflowStrategy??N.ERROR,visionTokensPerImage:e?.visionTokensPerImage,outputSize:e?.outputSize}}async function re(e){if("string"==typeof e)return{type:"text",text:e};if(e instanceof Blob){const t=e.type.toLowerCase().split(";")[0].trim();if(se.has(t)){return{type:"image",data:await e.arrayBuffer()}}if(ne.has(t)){return{type:"audio",data:await e.arrayBuffer()}}throw new Error(`Unsupported Blob mime type: "${e.type}". Supported image types: ${[...se].join(", ")}. Supported audio types: ${[...ne].join(", ")}.`)}if("object"==typeof e&&null!==e){if("image"===e.type){if(e.data instanceof Blob){return{type:"image",data:await e.data.arrayBuffer()}}return e}if("audio"===e.type){if(e.data instanceof Blob){return{type:"audio",data:await e.data.arrayBuffer()}}return e}return e}throw new Error(`Unsupported input part: ${JSON.stringify(e)}`)}async function oe(e){return Array.isArray(e)?Promise.all(e.map(e=>re(e))):re(e)}class ce{engine;deleteCallback;settings;mutex=new J;constructor(e,t,s){this.engine=e,this.deleteCallback=s,this.settings=t}static async create(e){const t=await T(),s=t.liteRtLmWasm,n=e.backend??U.GPU;e={...e,backend:n},n!==U.GPU&&n!==U.GPU_ARTISAN&&e.visionBackend!==U.GPU&&e.audioBackend!==U.GPU||await t.setupDefaultWebGpuDevice();const i=new z;let a;try{const{modelAssets:t,cleanupModelAssets:r}=await ee(e.model,s,i),o=s.EmbeddingEngineSettings.createDefaultMultimodal(t,{value:n},void 0!==e.visionBackend?{value:e.visionBackend}:void 0,void 0!==e.audioBackend?{value:e.audioBackend}:void 0);r();const c=i.add(()=>{o.delete()}),l=o.getMutableMainExecutorSettings().getBackend().value;ie(o,e);try{a=await s.EmbeddingEngine.createEngine(o)}finally{if(l===U.GPU||e.visionBackend===U.GPU||e.audioBackend===U.GPU)try{await s.clearStoredWeightsStreams()}catch(e){console.error("Error clearing stored weights streams:",e)}}return c(),i.add(()=>{a.delete()}),new ce(a,e,()=>i.run())}catch(e){throw i.run(),e}}async computeEmbedding(e,t={}){const s=await oe(e);return this.mutex.acquireAndRun(async()=>{const e=await this.engine.computeEmbedding(s,ae(t));return{embedding:O(e.embedding),inputLength:e.inputLength,truncatedLength:e.truncatedLength,numChunks:e.numChunks}})}async computeEmbeddingBatch(e,t={}){const s=await Promise.all(e.map(oe));return this.mutex.acquireAndRun(async()=>{const e=await this.engine.computeEmbeddingBatch(s,ae(t));try{const t=[],s=e.size();for(let n=0;n<s;n++){const s=e.get(n),i=O(s.embedding);t.push({embedding:i,inputLength:s.inputLength,truncatedLength:s.truncatedLength,numChunks:s.numChunks})}return t}finally{e.delete()}})}async delete(){await this.mutex.acquireAndRun(()=>{this.deleteCallback()})}}}};

package/dist/node/engine-litert.node.min.mjs.LICENSE.txt

@@ -0,0 +1 @@
+/*! MIT License. DegreeSign Chat SDK */

package/dist/types.d.ts

@@ -13,7 +13,7 @@ interface ChatModel {
     /** Whether the model supports chain-of-thought reasoning */
     reasoning: boolean;
     /** wllama (CPU) artifact */
-    cpu: {
+    cpu?: {
         repo: string;
         file: string;
         /** Vision projector, present when the CPU build accepts images */
@@ -21,6 +21,13 @@ interface ChatModel {
             file: string;
         };
     };
+    /** LiteRT-LM (WebGPU) artifact */
+    litert?: {
+        repo: string;
+        file: string;
+        /** Whether the build accepts images */
+        vision: boolean;
+    };
 }
 /** Token, prompt and thread budgets of an on-device generation */
 interface LlmLimits {
@@ -89,6 +96,17 @@ interface WllamaAssets {
     /** URL of the wllama compat worker script */
     compatWorker: string;
 }
+/** LiteRT-LM wasm assets used by the GPU engine */
+interface LitertAssets {
+    /** URL of the LiteRT-LM wasm glue script */
+    script: string;
+    /** URL of the LiteRT-LM wasm binary */
+    wasm: string;
+}
+/** Navigator with the WebGPU entry point, absent in Node */
+interface GpuNavigator extends Navigator {
+    gpu?: object;
+}
 /** Request state that builds one status snapshot */
 interface ChatState {
     /** Whether a reply is streaming right now */
@@ -110,6 +128,7 @@ interface ActiveCall extends ChatState {
 interface RuntimeDeps {
     engines?: {
         cpu?: LlmEngine;
+        litert?: LlmEngine;
     };
     backend?: LlmBackend;
 }
@@ -120,8 +139,17 @@ interface RuntimeLoadParams {
     assets?: Partial<WllamaAssets>;
     /** Whether engine logs reach the console */
     logging?: boolean;
+    /** Stop waiting for this load, the download continues while another caller waits */
+    signal?: AbortSignal;
     onProgress: (progress: number, phase: LlmPhase) => void;
 }
+/** One caller waiting on an in-flight model load */
+interface RuntimeLoadWaiter {
+    /** Report load progress to this caller */
+    report: RuntimeLoadParams[`onProgress`];
+    /** Resolve this caller, early when it leaves the load */
+    settle: (loaded: boolean) => void;
+}
 /** One generation request */
 interface RuntimeChatParams {
     messages: ChatMessage[];
@@ -135,10 +163,10 @@ interface RuntimeChatParams {
 }
 /** On-device runtime shared by every request */
 interface Runtime {
+    /** Load a model, or join a load of the same model already in flight */
     load: (params: RuntimeLoadParams) => Promise<boolean>;
     chat: (params: RuntimeChatParams) => Promise<string | undefined>;
     unload: () => Promise<void>;
-    cancelLoad: () => void;
     ready: () => boolean;
     loading: () => boolean;
     status: () => LlmStatus;
@@ -239,6 +267,28 @@ type EngineMessage = {
     role: `assistant`;
     content: string;
 };
+/** One LiteRT-LM content segment */
+type LitertContentPart = import('@litert-lm/core').ContentPart;
+/** One LiteRT-LM chat turn */
+type LitertMessage = import('@litert-lm/core').Message;
+/** One LiteRT-LM engine */
+type LitertEngine = import('@litert-lm/core').Engine;
+/** The LiteRT-LM module namespace */
+type LitertModule = typeof import('@litert-lm/core');
+/** Params of one LiteRT-LM chat call */
+interface LitertChatParams {
+    engine: LitertEngine;
+    model: ChatModel;
+    mod: LitertModule;
+    params: EngineChatParams;
+}
+/** Params of one LiteRT-LM stream */
+interface StreamLitertParams {
+    engine: LitertEngine;
+    mod: LitertModule;
+    params: EngineChatParams;
+    messages: EngineMessage[];
+}
 /** Streamed delta, reasoning may be inline or structured */
 interface EngineDelta {
     content?: string;
@@ -275,4 +325,4 @@ interface EngineRunParams {
     /** Whether the caller stopped the generation */
     stopped: () => boolean;
 }
-export type { ChatMessage, ChatModel, LlmLimits, ChatStatusSnapshot, ChatInput, ChatPreloadInput, ChatState, CancelChat, WllamaAssets, ActiveCall, RuntimeDeps, RuntimeLoadParams, RuntimeChatParams, Runtime, EngineChatParams, StartLlmParams, LlmRuntime, LlmEngine, EngineMessage, EngineContentPart, WllamaContentPart, WllamaMessage, EngineDelta, EngineChunk, StreamReplyParams, EngineRunParams, };
+export type { ChatMessage, ChatModel, LlmLimits, ChatStatusSnapshot, ChatInput, ChatPreloadInput, ChatState, CancelChat, WllamaAssets, LitertAssets, GpuNavigator, ActiveCall, RuntimeDeps, RuntimeLoadParams, RuntimeLoadWaiter, RuntimeChatParams, Runtime, EngineChatParams, StartLlmParams, LlmRuntime, LlmEngine, EngineMessage, EngineContentPart, WllamaContentPart, WllamaMessage, LitertContentPart, LitertMessage, LitertEngine, LitertModule, LitertChatParams, StreamLitertParams, EngineDelta, EngineChunk, StreamReplyParams, EngineRunParams, };

package/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@degreesign/chat",
-    "version": "0.2.0",
-    "description": "Headless on-device AI chat for browsers, workers and Node with a wllama CPU engine",
+    "version": "0.3.1",
+    "description": "Headless on-device AI chat for browsers and Node with wllama CPU and LiteRT WebGPU engines",
     "main": "dist/node/chat.node.min.js",
     "module": "dist/browser/chat.min.mjs",
     "browser": "dist/browser/chat.min.mjs",
@@ -36,6 +36,9 @@
         "on-device",
         "local inference",
         "wllama",
+        "litert",
+        "webgpu",
+        "gemma",
         "typescript",
         "browser",
         "node",
@@ -58,6 +61,7 @@
         "dist/"
     ],
     "dependencies": {
+        "@litert-lm/core": "0.19.0-nightly.20261024",
         "@wllama/wllama": "^3.8.1",
         "@wllama/wllama-compat": "^3.8.1"
     },
@@ -70,4 +74,4 @@
         "webpack": "^5.97.1",
         "webpack-cli": "^6.0.1"
     }
-}
+}