@seanhogg/builderforce-webcontainers 2026.10.12 → 2026.10.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (372) hide show
  1. package/README.md +22 -0
  2. package/dist/adaptive/controller.d.ts +39 -0
  3. package/dist/adaptive/controller.d.ts.map +1 -0
  4. package/dist/adaptive/controller.js +108 -0
  5. package/dist/adaptive/controller.js.map +1 -0
  6. package/dist/adaptive/costModel.d.ts +37 -0
  7. package/dist/adaptive/costModel.d.ts.map +1 -0
  8. package/dist/adaptive/costModel.js +102 -0
  9. package/dist/adaptive/costModel.js.map +1 -0
  10. package/dist/adaptive/planner.d.ts +22 -0
  11. package/dist/adaptive/planner.d.ts.map +1 -0
  12. package/dist/adaptive/planner.js +114 -0
  13. package/dist/adaptive/planner.js.map +1 -0
  14. package/dist/adaptive/types.d.ts +138 -0
  15. package/dist/adaptive/types.d.ts.map +1 -0
  16. package/dist/adaptive/types.js +12 -0
  17. package/dist/adaptive/types.js.map +1 -0
  18. package/dist/assets.js +2 -2
  19. package/dist/check/cachedFetch.js.map +1 -1
  20. package/dist/check/index.d.ts +6 -1
  21. package/dist/check/index.d.ts.map +1 -1
  22. package/dist/check/index.js +6 -1
  23. package/dist/check/index.js.map +1 -1
  24. package/dist/componentCompilers.d.ts.map +1 -1
  25. package/dist/componentCompilers.js.map +1 -1
  26. package/dist/esbuildWasm.js.map +1 -1
  27. package/dist/host.d.ts +11 -5
  28. package/dist/host.d.ts.map +1 -1
  29. package/dist/host.js +19 -8
  30. package/dist/host.js.map +1 -1
  31. package/dist/inference/backends/mock.d.ts +24 -0
  32. package/dist/inference/backends/mock.d.ts.map +1 -0
  33. package/dist/inference/backends/mock.js +55 -0
  34. package/dist/inference/backends/mock.js.map +1 -0
  35. package/dist/inference/backends/onnx.d.ts +33 -0
  36. package/dist/inference/backends/onnx.d.ts.map +1 -0
  37. package/dist/inference/backends/onnx.js +60 -0
  38. package/dist/inference/backends/onnx.js.map +1 -0
  39. package/dist/inference/backends/webllm.d.ts +27 -0
  40. package/dist/inference/backends/webllm.d.ts.map +1 -0
  41. package/dist/inference/backends/webllm.js +52 -0
  42. package/dist/inference/backends/webllm.js.map +1 -0
  43. package/dist/inference/backends/wllama.d.ts +29 -0
  44. package/dist/inference/backends/wllama.d.ts.map +1 -0
  45. package/dist/inference/backends/wllama.js +51 -0
  46. package/dist/inference/backends/wllama.js.map +1 -0
  47. package/dist/inference/capabilities.d.ts +20 -0
  48. package/dist/inference/capabilities.d.ts.map +1 -0
  49. package/dist/inference/capabilities.js +33 -0
  50. package/dist/inference/capabilities.js.map +1 -0
  51. package/dist/inference/client.d.ts +45 -0
  52. package/dist/inference/client.d.ts.map +1 -0
  53. package/dist/inference/client.js +201 -0
  54. package/dist/inference/client.js.map +1 -0
  55. package/dist/inference/modelCache.d.ts +71 -0
  56. package/dist/inference/modelCache.d.ts.map +1 -0
  57. package/dist/inference/modelCache.js +91 -0
  58. package/dist/inference/modelCache.js.map +1 -0
  59. package/dist/inference/protocol.d.ts +94 -0
  60. package/dist/inference/protocol.d.ts.map +1 -0
  61. package/dist/inference/protocol.js +19 -0
  62. package/dist/inference/protocol.js.map +1 -0
  63. package/dist/inference/tokenStream.d.ts +26 -0
  64. package/dist/inference/tokenStream.d.ts.map +1 -0
  65. package/dist/inference/tokenStream.js +79 -0
  66. package/dist/inference/tokenStream.js.map +1 -0
  67. package/dist/inference/types.d.ts +108 -0
  68. package/dist/inference/types.d.ts.map +1 -0
  69. package/dist/inference/types.js +8 -0
  70. package/dist/inference/types.js.map +1 -0
  71. package/dist/inference/worker.bundle.js +5 -0
  72. package/dist/inference/worker.d.ts +35 -0
  73. package/dist/inference/worker.d.ts.map +1 -0
  74. package/dist/inference/worker.js +157 -0
  75. package/dist/inference/worker.js.map +1 -0
  76. package/dist/inference/workerEntry.d.ts +15 -0
  77. package/dist/inference/workerEntry.d.ts.map +1 -0
  78. package/dist/inference/workerEntry.js +17 -0
  79. package/dist/inference/workerEntry.js.map +1 -0
  80. package/dist/kvcache/advanced.d.ts +39 -0
  81. package/dist/kvcache/advanced.d.ts.map +1 -0
  82. package/dist/kvcache/advanced.js +59 -0
  83. package/dist/kvcache/advanced.js.map +1 -0
  84. package/dist/kvcache/compatibility.d.ts +14 -0
  85. package/dist/kvcache/compatibility.d.ts.map +1 -0
  86. package/dist/kvcache/compatibility.js +44 -0
  87. package/dist/kvcache/compatibility.js.map +1 -0
  88. package/dist/kvcache/lru.d.ts +45 -0
  89. package/dist/kvcache/lru.d.ts.map +1 -0
  90. package/dist/kvcache/lru.js +91 -0
  91. package/dist/kvcache/lru.js.map +1 -0
  92. package/dist/kvcache/paged.d.ts +86 -0
  93. package/dist/kvcache/paged.d.ts.map +1 -0
  94. package/dist/kvcache/paged.js +238 -0
  95. package/dist/kvcache/paged.js.map +1 -0
  96. package/dist/kvcache/quant.d.ts +30 -0
  97. package/dist/kvcache/quant.d.ts.map +1 -0
  98. package/dist/kvcache/quant.js +73 -0
  99. package/dist/kvcache/quant.js.map +1 -0
  100. package/dist/kvcache/radix.d.ts +28 -0
  101. package/dist/kvcache/radix.d.ts.map +1 -0
  102. package/dist/kvcache/radix.js +58 -0
  103. package/dist/kvcache/radix.js.map +1 -0
  104. package/dist/kvcache/runtime.d.ts +11 -0
  105. package/dist/kvcache/runtime.d.ts.map +1 -0
  106. package/dist/kvcache/runtime.js +204 -0
  107. package/dist/kvcache/runtime.js.map +1 -0
  108. package/dist/kvcache/types.d.ts +151 -0
  109. package/dist/kvcache/types.d.ts.map +1 -0
  110. package/dist/kvcache/types.js +10 -0
  111. package/dist/kvcache/types.js.map +1 -0
  112. package/dist/learning/fingerprint.d.ts +17 -0
  113. package/dist/learning/fingerprint.d.ts.map +1 -0
  114. package/dist/learning/fingerprint.js +46 -0
  115. package/dist/learning/fingerprint.js.map +1 -0
  116. package/dist/learning/index.d.ts +5 -0
  117. package/dist/learning/index.d.ts.map +1 -0
  118. package/dist/learning/index.js +3 -0
  119. package/dist/learning/index.js.map +1 -0
  120. package/dist/learning/learner.d.ts +21 -0
  121. package/dist/learning/learner.d.ts.map +1 -0
  122. package/dist/learning/learner.js +272 -0
  123. package/dist/learning/learner.js.map +1 -0
  124. package/dist/learning/ledger.d.ts +26 -0
  125. package/dist/learning/ledger.d.ts.map +1 -0
  126. package/dist/learning/ledger.js +40 -0
  127. package/dist/learning/ledger.js.map +1 -0
  128. package/dist/learning/selection.d.ts +44 -0
  129. package/dist/learning/selection.d.ts.map +1 -0
  130. package/dist/learning/selection.js +111 -0
  131. package/dist/learning/selection.js.map +1 -0
  132. package/dist/learning/stats.d.ts +19 -0
  133. package/dist/learning/stats.d.ts.map +1 -0
  134. package/dist/learning/stats.js +31 -0
  135. package/dist/learning/stats.js.map +1 -0
  136. package/dist/learning/types.d.ts +133 -0
  137. package/dist/learning/types.d.ts.map +1 -0
  138. package/dist/learning/types.js +8 -0
  139. package/dist/learning/types.js.map +1 -0
  140. package/dist/memory/envProbe.d.ts +16 -0
  141. package/dist/memory/envProbe.d.ts.map +1 -0
  142. package/dist/memory/envProbe.js +28 -0
  143. package/dist/memory/envProbe.js.map +1 -0
  144. package/dist/memory/estimates.d.ts +46 -0
  145. package/dist/memory/estimates.d.ts.map +1 -0
  146. package/dist/memory/estimates.js +72 -0
  147. package/dist/memory/estimates.js.map +1 -0
  148. package/dist/memory/eviction.d.ts +42 -0
  149. package/dist/memory/eviction.d.ts.map +1 -0
  150. package/dist/memory/eviction.js +143 -0
  151. package/dist/memory/eviction.js.map +1 -0
  152. package/dist/memory/manager.d.ts +34 -0
  153. package/dist/memory/manager.d.ts.map +1 -0
  154. package/dist/memory/manager.js +346 -0
  155. package/dist/memory/manager.js.map +1 -0
  156. package/dist/memory/managerTypes.d.ts +49 -0
  157. package/dist/memory/managerTypes.d.ts.map +1 -0
  158. package/dist/memory/managerTypes.js +7 -0
  159. package/dist/memory/managerTypes.js.map +1 -0
  160. package/dist/memory/pressure.d.ts +56 -0
  161. package/dist/memory/pressure.d.ts.map +1 -0
  162. package/dist/memory/pressure.js +80 -0
  163. package/dist/memory/pressure.js.map +1 -0
  164. package/dist/memory/spillTier.d.ts +47 -0
  165. package/dist/memory/spillTier.d.ts.map +1 -0
  166. package/dist/memory/spillTier.js +128 -0
  167. package/dist/memory/spillTier.js.map +1 -0
  168. package/dist/memory/storageSink.d.ts +69 -0
  169. package/dist/memory/storageSink.d.ts.map +1 -0
  170. package/dist/memory/storageSink.js +221 -0
  171. package/dist/memory/storageSink.js.map +1 -0
  172. package/dist/memory/types.d.ts +202 -0
  173. package/dist/memory/types.d.ts.map +1 -0
  174. package/dist/memory/types.js +15 -0
  175. package/dist/memory/types.js.map +1 -0
  176. package/dist/memory/victims.d.ts +32 -0
  177. package/dist/memory/victims.d.ts.map +1 -0
  178. package/dist/memory/victims.js +74 -0
  179. package/dist/memory/victims.js.map +1 -0
  180. package/dist/memory/workspace.d.ts +34 -0
  181. package/dist/memory/workspace.d.ts.map +1 -0
  182. package/dist/memory/workspace.js +55 -0
  183. package/dist/memory/workspace.js.map +1 -0
  184. package/dist/memoryIntegration/index.d.ts +8 -0
  185. package/dist/memoryIntegration/index.d.ts.map +1 -0
  186. package/dist/memoryIntegration/index.js +4 -0
  187. package/dist/memoryIntegration/index.js.map +1 -0
  188. package/dist/memoryIntegration/knowledgeMemory.d.ts +43 -0
  189. package/dist/memoryIntegration/knowledgeMemory.d.ts.map +1 -0
  190. package/dist/memoryIntegration/knowledgeMemory.js +285 -0
  191. package/dist/memoryIntegration/knowledgeMemory.js.map +1 -0
  192. package/dist/memoryIntegration/persistence.d.ts +15 -0
  193. package/dist/memoryIntegration/persistence.d.ts.map +1 -0
  194. package/dist/memoryIntegration/persistence.js +43 -0
  195. package/dist/memoryIntegration/persistence.js.map +1 -0
  196. package/dist/memoryIntegration/text.d.ts +57 -0
  197. package/dist/memoryIntegration/text.d.ts.map +1 -0
  198. package/dist/memoryIntegration/text.js +179 -0
  199. package/dist/memoryIntegration/text.js.map +1 -0
  200. package/dist/memoryIntegration/types.d.ts +131 -0
  201. package/dist/memoryIntegration/types.d.ts.map +1 -0
  202. package/dist/memoryIntegration/types.js +11 -0
  203. package/dist/memoryIntegration/types.js.map +1 -0
  204. package/dist/ml.d.ts +52 -0
  205. package/dist/ml.d.ts.map +1 -0
  206. package/dist/ml.js +33 -0
  207. package/dist/ml.js.map +1 -0
  208. package/dist/node/childRelay.d.ts +10 -0
  209. package/dist/node/childRelay.d.ts.map +1 -1
  210. package/dist/node/childRelay.js +12 -2
  211. package/dist/node/childRelay.js.map +1 -1
  212. package/dist/node/index.js.map +1 -1
  213. package/dist/node/portResponder.d.ts +2 -0
  214. package/dist/node/portResponder.d.ts.map +1 -1
  215. package/dist/node/portResponder.js +3 -3
  216. package/dist/node/portResponder.js.map +1 -1
  217. package/dist/node/processHost.d.ts +4 -1
  218. package/dist/node/processHost.d.ts.map +1 -1
  219. package/dist/node/processHost.js +19 -16
  220. package/dist/node/processHost.js.map +1 -1
  221. package/dist/node/protocol.d.ts +1 -0
  222. package/dist/node/protocol.d.ts.map +1 -1
  223. package/dist/node/responseWire.d.ts.map +1 -1
  224. package/dist/node/worker.js +127 -120
  225. package/dist/node/workerProcess.js +1 -1
  226. package/dist/node/workerProcess.js.map +1 -1
  227. package/dist/prefetch/admission.d.ts +41 -0
  228. package/dist/prefetch/admission.d.ts.map +1 -0
  229. package/dist/prefetch/admission.js +49 -0
  230. package/dist/prefetch/admission.js.map +1 -0
  231. package/dist/prefetch/loadRegistry.d.ts +56 -0
  232. package/dist/prefetch/loadRegistry.d.ts.map +1 -0
  233. package/dist/prefetch/loadRegistry.js +108 -0
  234. package/dist/prefetch/loadRegistry.js.map +1 -0
  235. package/dist/prefetch/predictor.d.ts +40 -0
  236. package/dist/prefetch/predictor.d.ts.map +1 -0
  237. package/dist/prefetch/predictor.js +178 -0
  238. package/dist/prefetch/predictor.js.map +1 -0
  239. package/dist/prefetch/prefetcher.d.ts +37 -0
  240. package/dist/prefetch/prefetcher.d.ts.map +1 -0
  241. package/dist/prefetch/prefetcher.js +339 -0
  242. package/dist/prefetch/prefetcher.js.map +1 -0
  243. package/dist/prefetch/residentStore.d.ts +49 -0
  244. package/dist/prefetch/residentStore.d.ts.map +1 -0
  245. package/dist/prefetch/residentStore.js +87 -0
  246. package/dist/prefetch/residentStore.js.map +1 -0
  247. package/dist/prefetch/selfTuner.d.ts +40 -0
  248. package/dist/prefetch/selfTuner.d.ts.map +1 -0
  249. package/dist/prefetch/selfTuner.js +78 -0
  250. package/dist/prefetch/selfTuner.js.map +1 -0
  251. package/dist/prefetch/stats.d.ts +52 -0
  252. package/dist/prefetch/stats.d.ts.map +1 -0
  253. package/dist/prefetch/stats.js +72 -0
  254. package/dist/prefetch/stats.js.map +1 -0
  255. package/dist/prefetch/streaming.d.ts +132 -0
  256. package/dist/prefetch/streaming.d.ts.map +1 -0
  257. package/dist/prefetch/streaming.js +141 -0
  258. package/dist/prefetch/streaming.js.map +1 -0
  259. package/dist/prefetch/targetIndex.d.ts +18 -0
  260. package/dist/prefetch/targetIndex.d.ts.map +1 -0
  261. package/dist/prefetch/targetIndex.js +32 -0
  262. package/dist/prefetch/targetIndex.js.map +1 -0
  263. package/dist/prefetch/types.d.ts +283 -0
  264. package/dist/prefetch/types.d.ts.map +1 -0
  265. package/dist/prefetch/types.js +19 -0
  266. package/dist/prefetch/types.js.map +1 -0
  267. package/dist/previewBridge.d.ts +6 -0
  268. package/dist/previewBridge.d.ts.map +1 -1
  269. package/dist/previewBridge.js +25 -5
  270. package/dist/previewBridge.js.map +1 -1
  271. package/dist/previewMessages.d.ts +4 -3
  272. package/dist/previewMessages.d.ts.map +1 -1
  273. package/dist/previewMessages.js +12 -6
  274. package/dist/previewMessages.js.map +1 -1
  275. package/dist/process-worker.js +127 -120
  276. package/dist/relay.html +1 -1
  277. package/dist/shared/clock.d.ts +7 -0
  278. package/dist/shared/clock.d.ts.map +1 -0
  279. package/dist/shared/clock.js +10 -0
  280. package/dist/shared/clock.js.map +1 -0
  281. package/dist/shared/hardwareProbe.d.ts +26 -0
  282. package/dist/shared/hardwareProbe.d.ts.map +1 -0
  283. package/dist/shared/hardwareProbe.js +62 -0
  284. package/dist/shared/hardwareProbe.js.map +1 -0
  285. package/dist/shared/idb.d.ts +41 -0
  286. package/dist/shared/idb.d.ts.map +1 -0
  287. package/dist/shared/idb.js +184 -0
  288. package/dist/shared/idb.js.map +1 -0
  289. package/dist/shared/tokens.d.ts +7 -0
  290. package/dist/shared/tokens.d.ts.map +1 -0
  291. package/dist/shared/tokens.js +9 -0
  292. package/dist/shared/tokens.js.map +1 -0
  293. package/dist/snapshotStore.d.ts +2 -1
  294. package/dist/snapshotStore.d.ts.map +1 -1
  295. package/dist/snapshotStore.js +27 -12
  296. package/dist/snapshotStore.js.map +1 -1
  297. package/dist/styleCompilers.d.ts.map +1 -1
  298. package/dist/styleCompilers.js.map +1 -1
  299. package/dist/transport.js.map +1 -1
  300. package/package.json +11 -5
  301. package/src/adaptive/controller.ts +151 -0
  302. package/src/adaptive/costModel.ts +131 -0
  303. package/src/adaptive/planner.ts +127 -0
  304. package/src/adaptive/types.ts +137 -0
  305. package/src/check/index.ts +6 -1
  306. package/src/host.ts +33 -11
  307. package/src/inference/backends/mock.ts +77 -0
  308. package/src/inference/backends/onnx.ts +91 -0
  309. package/src/inference/backends/webllm.ts +79 -0
  310. package/src/inference/backends/wllama.ts +79 -0
  311. package/src/inference/capabilities.ts +46 -0
  312. package/src/inference/client.ts +276 -0
  313. package/src/inference/modelCache.ts +145 -0
  314. package/src/inference/protocol.ts +119 -0
  315. package/src/inference/tokenStream.ts +92 -0
  316. package/src/inference/types.ts +115 -0
  317. package/src/inference/worker.ts +191 -0
  318. package/src/inference/workerEntry.ts +18 -0
  319. package/src/kvcache/advanced.ts +86 -0
  320. package/src/kvcache/compatibility.ts +51 -0
  321. package/src/kvcache/lru.ts +100 -0
  322. package/src/kvcache/paged.ts +270 -0
  323. package/src/kvcache/quant.ts +88 -0
  324. package/src/kvcache/radix.ts +68 -0
  325. package/src/kvcache/runtime.ts +239 -0
  326. package/src/kvcache/types.ts +167 -0
  327. package/src/learning/fingerprint.ts +50 -0
  328. package/src/learning/index.ts +15 -0
  329. package/src/learning/learner.ts +304 -0
  330. package/src/learning/ledger.ts +54 -0
  331. package/src/learning/selection.ts +137 -0
  332. package/src/learning/stats.ts +39 -0
  333. package/src/learning/types.ts +142 -0
  334. package/src/memory/envProbe.ts +30 -0
  335. package/src/memory/estimates.ts +98 -0
  336. package/src/memory/eviction.ts +145 -0
  337. package/src/memory/manager.ts +380 -0
  338. package/src/memory/managerTypes.ts +61 -0
  339. package/src/memory/pressure.ts +120 -0
  340. package/src/memory/spillTier.ts +159 -0
  341. package/src/memory/storageSink.ts +289 -0
  342. package/src/memory/types.ts +217 -0
  343. package/src/memory/victims.ts +80 -0
  344. package/src/memory/workspace.ts +74 -0
  345. package/src/memoryIntegration/index.ts +7 -0
  346. package/src/memoryIntegration/knowledgeMemory.ts +339 -0
  347. package/src/memoryIntegration/persistence.ts +54 -0
  348. package/src/memoryIntegration/text.ts +185 -0
  349. package/src/memoryIntegration/types.ts +145 -0
  350. package/src/ml.ts +180 -0
  351. package/src/node/childRelay.ts +16 -3
  352. package/src/node/portResponder.ts +5 -3
  353. package/src/node/processHost.ts +16 -15
  354. package/src/node/protocol.ts +1 -1
  355. package/src/node/workerProcess.ts +1 -1
  356. package/src/prefetch/admission.ts +74 -0
  357. package/src/prefetch/loadRegistry.ts +156 -0
  358. package/src/prefetch/predictor.ts +199 -0
  359. package/src/prefetch/prefetcher.ts +369 -0
  360. package/src/prefetch/residentStore.ts +121 -0
  361. package/src/prefetch/selfTuner.ts +110 -0
  362. package/src/prefetch/stats.ts +120 -0
  363. package/src/prefetch/streaming.ts +231 -0
  364. package/src/prefetch/targetIndex.ts +40 -0
  365. package/src/prefetch/types.ts +307 -0
  366. package/src/previewBridge.ts +25 -5
  367. package/src/previewMessages.ts +11 -6
  368. package/src/shared/clock.ts +9 -0
  369. package/src/shared/hardwareProbe.ts +68 -0
  370. package/src/shared/idb.ts +203 -0
  371. package/src/shared/tokens.ts +8 -0
  372. package/src/snapshotStore.ts +25 -10
@@ -0,0 +1,276 @@
1
+ import {
2
+ INFERENCE_CANCEL,
3
+ INFERENCE_DISPOSE_MODEL,
4
+ INFERENCE_DONE,
5
+ INFERENCE_ERROR,
6
+ INFERENCE_GENERATE,
7
+ INFERENCE_LOAD,
8
+ INFERENCE_LOG,
9
+ INFERENCE_PROBE,
10
+ INFERENCE_TOKEN,
11
+ type DoneReply,
12
+ type ErrorReply,
13
+ type InferenceReply,
14
+ type LogReply,
15
+ type ReadyReply,
16
+ type TokenReply,
17
+ type WireGenerateOptions,
18
+ } from './protocol.js';
19
+ import { createModelCache, type ModelCache } from './modelCache.js';
20
+ import { createTokenStream, type TokenStream } from './tokenStream.js';
21
+ import type {
22
+ ChatMessage,
23
+ GenerateOptions,
24
+ GenerationStats,
25
+ InferenceToken,
26
+ ModelRef,
27
+ RuntimeCapabilities,
28
+ } from './types.js';
29
+
30
+ /** Minimal Worker surface the client needs (real Worker, tests, or a port). */
31
+ export interface InferenceWorkerLike {
32
+ postMessage(message: unknown, transfer?: Transferable[]): void;
33
+ terminate(): void;
34
+ onmessage: ((event: { data: unknown }) => void) | null;
35
+ onerror: ((event: { message?: string }) => void) | null;
36
+ }
37
+
38
+ export interface InferenceClientOptions {
39
+ /**
40
+ * Spawn the inference worker. Default: a classic Worker on the bundled
41
+ * `worker.bundle.js` that sits beside this module in `dist/inference/`
42
+ * (also exported as `@seanhogg/builderforce-webcontainers/ml/worker.js`).
43
+ */
44
+ createWorker?: () => InferenceWorkerLike;
45
+ /** Model cache; default IndexedDB via `createModelCache()`. */
46
+ cache?: ModelCache;
47
+ /**
48
+ * Fetch a model's bytes from its source URL (only called on a cache miss).
49
+ * The returned buffer is TRANSFERRED to the worker, so do not keep using it.
50
+ */
51
+ fetchModel?: (ref: ModelRef) => Promise<ArrayBuffer>;
52
+ logger?: (message: string) => void;
53
+ }
54
+
55
+ interface Pending {
56
+ resolve: (reply: never) => void;
57
+ reject: (error: Error) => void;
58
+ }
59
+
60
+ export interface InferenceClient {
61
+ probe(): Promise<RuntimeCapabilities>;
62
+ /**
63
+ * Ensure the model is cached and loaded in the worker. After this resolves,
64
+ * `generate()` makes no network access — the bytes came from the cache.
65
+ */
66
+ load(ref: ModelRef): Promise<void>;
67
+ generate(messages: ChatMessage[], options?: GenerateOptions): AsyncIterable<InferenceToken> & {
68
+ stats: Promise<GenerationStats>;
69
+ };
70
+ cancel(): void;
71
+ dispose(): Promise<void>;
72
+ readonly cache: ModelCache;
73
+ }
74
+
75
+ const toError = (error: unknown): Error => (error instanceof Error ? error : new Error(String(error)));
76
+
77
+ function defaultFetchModel(ref: ModelRef): Promise<ArrayBuffer> {
78
+ if (!ref.source || ref.source.kind !== 'url') return Promise.reject(new Error(`model ${ref.id} has no url source`));
79
+ const url = ref.source.url;
80
+ return fetch(url).then((res) => {
81
+ if (!res.ok) throw new Error(`failed to fetch ${url}: ${res.status}`);
82
+ return res.arrayBuffer();
83
+ });
84
+ }
85
+
86
+ function defaultCreateWorker(): InferenceWorkerLike {
87
+ // The IIFE bundle emitted by scripts/build-sw.mjs beside dist/inference/client.js.
88
+ // ESM only (import.meta.url): a CommonJS consumer of ./ml must pass `createWorker`.
89
+ return new Worker(new URL('./worker.bundle.js', import.meta.url)) as unknown as InferenceWorkerLike;
90
+ }
91
+
92
+ export function createInferenceClient(options: InferenceClientOptions = {}): InferenceClient {
93
+ const cache = options.cache ?? createModelCache();
94
+ const fetchModel = options.fetchModel ?? defaultFetchModel;
95
+ const spawn = options.createWorker ?? defaultCreateWorker;
96
+
97
+ let activeWorker: InferenceWorkerLike | undefined;
98
+ let nextId = 1;
99
+ const pending = new Map<number, Pending>();
100
+ const streams = new Map<number, TokenStream>();
101
+ let workerFailed: Error | undefined;
102
+
103
+ const handleReply = (reply: InferenceReply) => {
104
+ switch (reply.type) {
105
+ case INFERENCE_LOG:
106
+ options.logger?.((reply as LogReply).message);
107
+ return;
108
+ case INFERENCE_TOKEN: {
109
+ const token = reply as TokenReply;
110
+ streams.get(token.requestId)?.push({ token: token.token, index: token.index });
111
+ return;
112
+ }
113
+ case INFERENCE_DONE: {
114
+ const done = reply as DoneReply;
115
+ streams.get(done.requestId)?.done({
116
+ promptTokens: done.promptTokens,
117
+ promptTokensEstimated: done.promptTokensEstimated,
118
+ generatedTokens: done.generatedTokens,
119
+ ttftMs: done.ttftMs,
120
+ decodeMs: done.decodeMs,
121
+ tokensPerSecond: done.tokensPerSecond,
122
+ });
123
+ return;
124
+ }
125
+ case INFERENCE_ERROR: {
126
+ const error = reply as ErrorReply;
127
+ const waiting = pending.get(error.id);
128
+ if (waiting) {
129
+ pending.delete(error.id);
130
+ waiting.reject(new Error(error.message));
131
+ } else {
132
+ streams.get(error.id)?.fail(new Error(error.message));
133
+ }
134
+ return;
135
+ }
136
+ }
137
+ const id = (reply as { id: number }).id;
138
+ const waiting = pending.get(id);
139
+ if (waiting) {
140
+ pending.delete(id);
141
+ (waiting.resolve as (value: never) => void)(reply as never);
142
+ }
143
+ };
144
+
145
+ /** Reject every in-flight request and stream; later calls reject immediately. */
146
+ const failAll = (error: Error) => {
147
+ workerFailed = error;
148
+ const waiting = [...pending.values()];
149
+ const open = [...streams.values()];
150
+ pending.clear();
151
+ streams.clear();
152
+ for (const p of waiting) p.reject(error);
153
+ for (const s of open) s.fail(error);
154
+ };
155
+
156
+ const getWorker = (): InferenceWorkerLike => {
157
+ if (activeWorker) return activeWorker;
158
+ const w = spawn();
159
+ w.onmessage = (event) => handleReply(event.data as InferenceReply);
160
+ w.onerror = (event) => failAll(new Error(event.message ?? 'inference worker failed'));
161
+ activeWorker = w;
162
+ return w;
163
+ };
164
+
165
+ const request = <T>(message: Record<string, unknown>, transfer?: Transferable[]): Promise<T> =>
166
+ new Promise<T>((resolve, reject) => {
167
+ if (workerFailed) return reject(workerFailed);
168
+ const id = nextId++;
169
+ pending.set(id, { resolve: resolve as (reply: never) => void, reject });
170
+ try {
171
+ getWorker().postMessage({ ...message, id }, transfer);
172
+ } catch (error) {
173
+ pending.delete(id);
174
+ reject(toError(error));
175
+ }
176
+ });
177
+
178
+ /** Ask the worker to stop one generate request (fire-and-forget). */
179
+ const sendCancel = (requestId: number) => {
180
+ try {
181
+ activeWorker?.postMessage({ type: INFERENCE_CANCEL, id: nextId++, requestId });
182
+ } catch {
183
+ /* worker gone: nothing to cancel */
184
+ }
185
+ };
186
+
187
+ const client: InferenceClient = {
188
+ cache,
189
+
190
+ async probe() {
191
+ const reply = await request<ReadyReply>({ type: INFERENCE_PROBE });
192
+ return reply.capabilities;
193
+ },
194
+
195
+ async load(ref) {
196
+ // Per the ModelCache ownership contract, `get` hands back a buffer we own
197
+ // and `put` never retains ours — so the bytes are transferred to the
198
+ // worker with no copy made here (IndexedDB's own clone is the only one).
199
+ const cached = await cache.get(ref.id);
200
+ let weights: ArrayBuffer;
201
+ if (cached) {
202
+ weights = cached.bytes;
203
+ } else {
204
+ weights = await fetchModel(ref);
205
+ // Awaited BEFORE the transfer below: `put` clones the bytes (IndexedDB's
206
+ // structured clone runs once its store has opened), and a transfer first
207
+ // would hand it a detached buffer. The write is best-effort — a quota abort
208
+ // (or any cache failure) must not stop the model loading; it just isn't cached.
209
+ try {
210
+ await cache.put(ref.id, {
211
+ quantization: ref.quantization,
212
+ paramsB: ref.paramsB,
213
+ bytes: weights,
214
+ size: weights.byteLength,
215
+ });
216
+ } catch (error) {
217
+ options.logger?.(`model cache write failed for ${ref.id}: ${toError(error).message}`);
218
+ }
219
+ }
220
+ await request({ type: INFERENCE_LOAD, ref, weights }, [weights]);
221
+ },
222
+
223
+ generate(messages, genOptions = {}) {
224
+ const id = nextId++;
225
+ // An AbortSignal is not structured-cloneable: it stays host-side and is
226
+ // translated into an INFERENCE_CANCEL for this request.
227
+ const { signal, ...wireOptions } = genOptions;
228
+ const onAbort = () => sendCancel(id);
229
+ const stream = createTokenStream(() => {
230
+ streams.delete(id);
231
+ signal?.removeEventListener('abort', onAbort);
232
+ });
233
+ const result = (s: TokenStream) =>
234
+ Object.assign(
235
+ s.iterate(() => sendCancel(id)),
236
+ { stats: s.stats },
237
+ );
238
+
239
+ if (workerFailed) {
240
+ stream.fail(workerFailed);
241
+ return result(stream);
242
+ }
243
+ try {
244
+ const w = getWorker();
245
+ streams.set(id, stream);
246
+ w.postMessage({ type: INFERENCE_GENERATE, id, messages, options: wireOptions satisfies WireGenerateOptions });
247
+ } catch (error) {
248
+ stream.fail(toError(error)); // also removes the map entry
249
+ return result(stream);
250
+ }
251
+ if (signal) {
252
+ if (signal.aborted) onAbort();
253
+ else signal.addEventListener('abort', onAbort, { once: true });
254
+ }
255
+ return result(stream);
256
+ },
257
+
258
+ cancel() {
259
+ try {
260
+ activeWorker?.postMessage({ type: INFERENCE_CANCEL, id: nextId++ });
261
+ } catch {
262
+ /* worker gone */
263
+ }
264
+ },
265
+
266
+ async dispose() {
267
+ try {
268
+ await request({ type: INFERENCE_DISPOSE_MODEL });
269
+ } finally {
270
+ activeWorker?.terminate();
271
+ activeWorker = undefined;
272
+ }
273
+ },
274
+ };
275
+ return client;
276
+ }
@@ -0,0 +1,145 @@
1
+ /**
2
+ * Phase 1 — Local CPU Inference Engine: model asset cache.
3
+ *
4
+ * Stores quantized model weights in IndexedDB so a model provisioned once is
5
+ * available offline for every later session. Keys are the model id; entries
6
+ * carry the bytes plus a small metadata record (or, with `valueFormat: 'raw'`,
7
+ * just the bytes — see `ModelCacheOptions.valueFormat`). Uses the same defensive
8
+ * posture as the rest of the runtime: if IndexedDB is unavailable the cache
9
+ * is a no-op and provisioning simply re-downloads.
10
+ *
11
+ * Buffer ownership contract (every `ModelCache` implementation): `put` must not
12
+ * retain the caller's `bytes` buffer (IndexedDB structured-clones it; the
13
+ * in-memory cache copies it), and `get` returns a buffer the caller owns and
14
+ * may transfer. That is what lets `InferenceClient.load()` transfer the bytes
15
+ * to the worker without making its own copy.
16
+ */
17
+
18
+ import { lazyIdbStore } from '../shared/idb.js';
19
+
20
+ export interface ModelCacheEntry {
21
+ /** Model id, e.g. `builderforce/tiny-instruct-q4`. */
22
+ id: string;
23
+ /** Absent on legacy entries stored as raw bytes. */
24
+ quantization?: string;
25
+ /** Absent on legacy entries stored as raw bytes. */
26
+ paramsB?: number;
27
+ bytes: ArrayBuffer;
28
+ /** Byte length of `bytes`, recorded at put time for bookkeeping. */
29
+ size: number;
30
+ /** When the entry was written, epoch ms (0 for legacy raw-bytes entries). */
31
+ storedAt: number;
32
+ }
33
+
34
+ export interface ModelCache {
35
+ get(id: string): Promise<ModelCacheEntry | undefined>;
36
+ put(id: string, entry: Omit<ModelCacheEntry, 'id' | 'storedAt'>): Promise<void>;
37
+ has(id: string): Promise<boolean>;
38
+ delete(id: string): Promise<void>;
39
+ /** Ids currently held, for capacity management and diagnostics. */
40
+ keys(): Promise<string[]>;
41
+ }
42
+
43
+ /**
44
+ * IndexedDB-backed model cache. Pass an explicit `indexedDB` in tests (the
45
+ * `fake-indexeddb` package is already a dev dependency of this package);
46
+ * in the browser it defaults to the global.
47
+ *
48
+ * `dbName` / `storeName` / `dbVersion` let a host adopt an existing weights
49
+ * database (e.g. Builderforce Studio's `builderforce-studio-weights` / `weights`
50
+ * / v1, whose values are raw ArrayBuffers keyed by path) without migrating it.
51
+ */
52
+ export interface ModelCacheOptions {
53
+ indexedDB?: IDBFactory;
54
+ /** Default `bfwc-inference-models`. */
55
+ dbName?: string;
56
+ /** Default `models`. */
57
+ storeName?: string;
58
+ /** Default 1. The upgrade only creates the store when it is missing. */
59
+ dbVersion?: number;
60
+ /**
61
+ * How `put` stores a value. Default `'entry'`.
62
+ * - `'entry'`: a `ModelCacheEntry` object (bytes + metadata).
63
+ * - `'raw'`: the `ArrayBuffer` itself, with no wrapper — byte-identical to
64
+ * Builderforce Studio's legacy weights format, so older builds on the same
65
+ * origin keep reading what this cache writes. `quantization` / `paramsB`
66
+ * are NOT persisted in raw mode; `get` returns `{ id, bytes, size, storedAt: 0 }`.
67
+ *
68
+ * `get` reads both shapes in either mode.
69
+ */
70
+ valueFormat?: ModelCacheValueFormat;
71
+ }
72
+
73
+ export type ModelCacheValueFormat = 'entry' | 'raw';
74
+
75
+ function isArrayBuffer(value: unknown): value is ArrayBuffer {
76
+ return value instanceof ArrayBuffer || Object.prototype.toString.call(value) === '[object ArrayBuffer]';
77
+ }
78
+
79
+ /** Normalise a stored value: a legacy raw ArrayBuffer is wrapped as an entry. */
80
+ function toEntry(id: string, value: unknown): ModelCacheEntry | undefined {
81
+ if (value === undefined || value === null) return undefined;
82
+ if (isArrayBuffer(value)) return { id, bytes: value, size: value.byteLength, storedAt: 0 };
83
+ return value as ModelCacheEntry;
84
+ }
85
+
86
+ export function createModelCache(options: ModelCacheOptions = {}): ModelCache {
87
+ const factory = options.indexedDB ?? (globalThis as { indexedDB?: IDBFactory }).indexedDB;
88
+ const config = {
89
+ dbName: options.dbName ?? 'bfwc-inference-models',
90
+ storeName: options.storeName ?? 'models',
91
+ version: options.dbVersion ?? 1,
92
+ };
93
+ const raw = options.valueFormat === 'raw';
94
+ const open = lazyIdbStore(factory, config);
95
+
96
+ return {
97
+ async get(id) {
98
+ const store = await open();
99
+ if (!store) return undefined;
100
+ return toEntry(id, await store.run('readonly', (s) => s.get(id) as IDBRequest<unknown>));
101
+ },
102
+ async put(id, entry) {
103
+ const store = await open();
104
+ if (!store) return;
105
+ const value: ArrayBuffer | ModelCacheEntry = raw ? entry.bytes : { ...entry, id, storedAt: Date.now() };
106
+ await store.run('readwrite', (s) => s.put(value, id));
107
+ },
108
+ async has(id) {
109
+ const store = await open();
110
+ if (!store) return false;
111
+ return (await store.run('readonly', (s) => s.count(id))) > 0;
112
+ },
113
+ async delete(id) {
114
+ await (await open())?.run('readwrite', (s) => s.delete(id));
115
+ },
116
+ async keys() {
117
+ const store = await open();
118
+ if (!store) return [];
119
+ return (await store.run('readonly', (s) => s.getAllKeys())).map(String);
120
+ },
121
+ };
122
+ }
123
+
124
+ /** In-memory cache for tests and non-persistent fallbacks. Copies on put and get (see the ownership contract). */
125
+ export function createMemoryModelCache(): ModelCache {
126
+ const map = new Map<string, ModelCacheEntry>();
127
+ return {
128
+ async get(id) {
129
+ const entry = map.get(id);
130
+ return entry ? { ...entry, bytes: entry.bytes.slice(0) } : undefined;
131
+ },
132
+ async put(id, entry) {
133
+ map.set(id, { ...entry, bytes: entry.bytes.slice(0), id, storedAt: Date.now() });
134
+ },
135
+ async has(id) {
136
+ return map.has(id);
137
+ },
138
+ async delete(id) {
139
+ map.delete(id);
140
+ },
141
+ async keys() {
142
+ return [...map.keys()];
143
+ },
144
+ };
145
+ }
@@ -0,0 +1,119 @@
1
+ /**
2
+ * Phase 1 — Local CPU Inference Engine: worker protocol.
3
+ *
4
+ * Messages between the host-side `InferenceClient` and the inference worker.
5
+ * Structured so a future Phase 4/5 can add streaming KV-cache and prefill
6
+ * messages without reworking the envelope.
7
+ */
8
+
9
+ import type { ChatMessage, GenerateOptions, ModelRef, RuntimeCapabilities } from './types.js';
10
+
11
+ /** Generate options as they cross the wire: an AbortSignal is not structured-cloneable, so it never does. */
12
+ export type WireGenerateOptions = Omit<GenerateOptions, 'signal'>;
13
+
14
+ export const INFERENCE_PROBE = 'inference:probe';
15
+ export const INFERENCE_LOAD = 'inference:load';
16
+ export const INFERENCE_GENERATE = 'inference:generate';
17
+ export const INFERENCE_CANCEL = 'inference:cancel';
18
+ export const INFERENCE_DISPOSE_MODEL = 'inference:dispose-model';
19
+ export const INFERENCE_CAPABILITIES = 'inference:capabilities';
20
+ export const INFERENCE_READY = 'inference:ready';
21
+ export const INFERENCE_ERROR = 'inference:error';
22
+ export const INFERENCE_LOG = 'inference:log';
23
+ export const INFERENCE_TOKEN = 'inference:token';
24
+ export const INFERENCE_DONE = 'inference:done';
25
+
26
+ export interface ProbeRequest {
27
+ type: typeof INFERENCE_PROBE;
28
+ id: number;
29
+ }
30
+
31
+ export interface LoadRequest {
32
+ type: typeof INFERENCE_LOAD;
33
+ id: number;
34
+ ref: ModelRef;
35
+ /** Model bytes, transferred by the host (ArrayBuffer transfer). */
36
+ weights: ArrayBuffer;
37
+ }
38
+
39
+ export interface GenerateRequest {
40
+ type: typeof INFERENCE_GENERATE;
41
+ id: number;
42
+ messages: ChatMessage[];
43
+ options?: WireGenerateOptions;
44
+ }
45
+
46
+ export interface CancelRequest {
47
+ type: typeof INFERENCE_CANCEL;
48
+ id: number;
49
+ /** The generate request to cancel; absent = whatever generation is running. */
50
+ requestId?: number;
51
+ }
52
+
53
+ export interface DisposeModelRequest {
54
+ type: typeof INFERENCE_DISPOSE_MODEL;
55
+ id: number;
56
+ }
57
+
58
+ export type InferenceRequest =
59
+ | ProbeRequest
60
+ | LoadRequest
61
+ | GenerateRequest
62
+ | CancelRequest
63
+ | DisposeModelRequest;
64
+
65
+ export interface ReadyReply {
66
+ type: typeof INFERENCE_READY;
67
+ id: number;
68
+ capabilities: RuntimeCapabilities;
69
+ }
70
+
71
+ export interface ErrorReply {
72
+ type: typeof INFERENCE_ERROR;
73
+ id: number;
74
+ message: string;
75
+ }
76
+
77
+ export interface LogReply {
78
+ type: typeof INFERENCE_LOG;
79
+ id: number;
80
+ message: string;
81
+ }
82
+
83
+ export interface TokenReply {
84
+ type: typeof INFERENCE_TOKEN;
85
+ id: number;
86
+ requestId: number;
87
+ token: string;
88
+ index: number;
89
+ }
90
+
91
+ export interface DoneReply {
92
+ type: typeof INFERENCE_DONE;
93
+ id: number;
94
+ requestId: number;
95
+ promptTokens: number;
96
+ /** True when `promptTokens` is the ~4 chars/token heuristic, not the backend's tokenizer count. */
97
+ promptTokensEstimated: boolean;
98
+ generatedTokens: number;
99
+ ttftMs: number;
100
+ decodeMs: number;
101
+ tokensPerSecond: number;
102
+ }
103
+
104
+ export type InferenceReply =
105
+ | ReadyReply
106
+ | ErrorReply
107
+ | LogReply
108
+ | TokenReply
109
+ | DoneReply;
110
+
111
+ /** Stats part of a DoneReply, shared with the public API. */
112
+ export interface GenerationStatsPayload {
113
+ promptTokens: number;
114
+ promptTokensEstimated: boolean;
115
+ generatedTokens: number;
116
+ ttftMs: number;
117
+ decodeMs: number;
118
+ tokensPerSecond: number;
119
+ }
@@ -0,0 +1,92 @@
1
+ /**
2
+ * Phase 1 — per-request token stream state for `InferenceClient.generate()`.
3
+ *
4
+ * ONE object holds everything a generate request owns on the host side — the
5
+ * token queue, the iterator's waiter, and the stats promise — so the client
6
+ * keeps a single `Map<id, TokenStream>` and cleanup can never leave a queue or
7
+ * waiter behind for a finished id.
8
+ */
9
+
10
+ import type { GenerationStats, InferenceToken } from './types.js';
11
+
12
+ export interface TokenStream {
13
+ /** A token from the worker. Dropped once the consumer has stopped iterating. */
14
+ push(token: InferenceToken): void;
15
+ /** The worker finished the request. */
16
+ done(stats: GenerationStats): void;
17
+ /** The request failed (worker error reply, worker crash, failed post). */
18
+ fail(error: Error): void;
19
+ /** Resolves with the final stats; rejects when the request fails. Never unhandled. */
20
+ readonly stats: Promise<GenerationStats>;
21
+ /**
22
+ * The consumer-facing iterator. `onEarlyClose` runs when the consumer stops
23
+ * iterating (break / return / throw) before the stream settled.
24
+ */
25
+ iterate(onEarlyClose: () => void): AsyncIterable<InferenceToken>;
26
+ }
27
+
28
+ export function createTokenStream(onSettled: () => void): TokenStream {
29
+ const queue: InferenceToken[] = [];
30
+ let notify: (() => void) | undefined;
31
+ let settled = false;
32
+ let detached = false;
33
+ let error: Error | undefined;
34
+ let resolveStats!: (stats: GenerationStats) => void;
35
+ let rejectStats!: (error: Error) => void;
36
+ const stats = new Promise<GenerationStats>((resolve, reject) => {
37
+ resolveStats = resolve;
38
+ rejectStats = reject;
39
+ });
40
+ stats.catch(() => {}); // The consumer may only observe the iterator; never leave this unhandled.
41
+
42
+ const wake = () => {
43
+ const n = notify;
44
+ notify = undefined;
45
+ n?.();
46
+ };
47
+ const settle = () => {
48
+ settled = true;
49
+ onSettled();
50
+ wake();
51
+ };
52
+
53
+ return {
54
+ push(token) {
55
+ if (settled || detached) return;
56
+ queue.push(token);
57
+ wake();
58
+ },
59
+ done(s) {
60
+ if (settled) return;
61
+ resolveStats(s);
62
+ settle();
63
+ },
64
+ fail(e) {
65
+ if (settled) return;
66
+ error = e;
67
+ rejectStats(e);
68
+ settle();
69
+ },
70
+ stats,
71
+ iterate(onEarlyClose) {
72
+ return (async function* (): AsyncIterable<InferenceToken> {
73
+ try {
74
+ for (;;) {
75
+ while (queue.length) yield queue.shift() as InferenceToken;
76
+ if (error) throw error;
77
+ if (settled) return;
78
+ await new Promise<void>((resolve) => {
79
+ notify = resolve;
80
+ });
81
+ }
82
+ } finally {
83
+ if (!settled && !detached) {
84
+ detached = true;
85
+ queue.length = 0;
86
+ onEarlyClose();
87
+ }
88
+ }
89
+ })();
90
+ },
91
+ };
92
+ }