@qvac/llm-llamacpp 0.61.0 → 0.61.2

package/README.md

@@ -180,6 +180,8 @@ const config = {
 | device            | `"gpu"` or `"cpu"`                          | — (required)                 | Device to run inference on                            |
 | gpu_layers        | integer                                     | 0                            | Number of model layers to offload to GPU              |
 | ctx_size          | 0 – model-dependent                         | 4096 (0 = loaded from model) | Context window size                                   |
+| poll              | integer 0 – 100                             | 0                            | Main GGML threadpool busy-wait level. `0` sleeps while idle; higher values can reduce wake latency but consume CPU during GPU work |
+| poll-batch        | integer 0 – 100                             | `poll`                       | Batch GGML threadpool busy-wait level                  |
 | lora              | string                                      | —                            | Path to LoRA adapter file                             |
 | temp              | 0.00 – 2.00                                 | 0.8                          | Sampling temperature                                  |
 | top_p             | 0 – 1                                       | 0.9                          | Top-p (nucleus) sampling                              |

package/index.d.ts

@@ -141,6 +141,17 @@ declare namespace LlmLlamacpp {
         backend?: string;
         gpu_layers?: NumericLike;
         ctx_size?: NumericLike;
+        /**
+         * Main GGML threadpool busy-wait level, from 0 (sleep while idle) to 100
+         * (maximum polling). Defaults to 0. Values above 0 can reduce worker wake
+         * latency for CPU-heavy workloads, but consume CPU while GPU work runs.
+         */
+        poll?: NumericLike;
+        /**
+         * Batch GGML threadpool busy-wait level. Defaults to `poll`, or 0 when
+         * neither option is set.
+         */
+        "poll-batch"?: NumericLike;
         system_prompt?: string;
         lora?: string;
         temp?: NumericLike;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@qvac/llm-llamacpp",
-  "version": "0.61.0",
+  "version": "0.61.2",
   "description": "llama addon for qvac",
   "addon": true,
   "scripts": {
@@ -18,7 +18,7 @@
     "quickstart": "node -e \"const fs=require('fs');const path=require('path');const {execSync}=require('child_process');const prebuilds=path.join(process.cwd(),'prebuilds');const source=path.join(process.cwd(),'node_modules','@qvac','llm-llamacpp','prebuilds');if(!fs.existsSync(prebuilds)){if(!fs.existsSync(source)){execSync('npm install @qvac/llm-llamacpp@latest',{stdio:'inherit'});}if(!fs.existsSync(source)){throw new Error('Prebuilds not found after install.');}fs.cpSync(source,prebuilds,{recursive:true});}execSync('bare examples/quickstart.js',{stdio:'inherit'});\"",
     "update:quickstart-section": "node ./scripts/quickstart-testing/update_readme.js",
     "test": "npm run lint && npm run test:unit && npm run test:integration",
-    "test:integration": "npm run build:ts && npm run test:integration:generate && bare test/integration/all.js --exit",
+    "test:integration": "npm run build:ts && npm run test:integration:generate && bare test/integration/fit.test.js && bare test/integration/all.js --exit",
     "test:mobile:generate": "npm run generate:benchmark-shards && bare ./scripts/generate-mobile-integration-tests.js",
     "test:mobile:validate": "npm run generate:benchmark-shards && node scripts/validate-mobile-tests.js && node scripts/validate-mobile-manifest.js",
     "typecheck": "tsc --noEmit -p tsconfig.json",

package/prebuilds/darwin-arm64/qvac__llm-llamacpp.bare.exports

@@ -7,7 +7,7 @@
     ],
     "current_versions": [
       {
-        "version": "0.61"
+        "version": "0.61.2"
       }
     ],
     "exported_symbols": [
@@ -84,7 +84,6 @@
             "__ZN24llama_finetuning_helpers23optEpochCallbackWrapperEbP16ggml_opt_contextP16ggml_opt_datasetP15ggml_opt_resultxxx",
             "__ZThn24_N10LlamaModelD0Ev",
             "__ZN14MtmdLlmContext8setNPastEi",
-            "__ZN14TextLlmContext26initializeOwnedThreadpoolsEv",
             "__ZN30qvac_lib_inference_addon_llama8batching19MultiRequestBatcher10addRequestEONSt3__16vectorIiNS2_9allocatorIiEEEERj",
             "__ZN30qvac_lib_inference_addon_llama8batching19MultiRequestBatcher19sampleAndAppendIdleERKNSt3__18functionIFijiEEE",
             "__ZN10LlamaModel30discardFinetuneCancelSaveModesERKNSt3__16vectorIyNS0_9allocatorIyEEEE",

package/prebuilds/darwin-x64/qvac__llm-llamacpp.bare.exports

@@ -7,7 +7,7 @@
     ],
     "current_versions": [
       {
-        "version": "0.61"
+        "version": "0.61.2"
       }
     ],
     "exported_symbols": [
@@ -798,7 +798,6 @@
             "__ZN24llama_finetuning_helpers23optEpochCallbackWrapperEbP16ggml_opt_contextP16ggml_opt_datasetP15ggml_opt_resultxxx",
             "__ZN14MtmdLlmContext8setNPastEi",
             "__ZThn24_N10LlamaModelD0Ev",
-            "__ZN14TextLlmContext26initializeOwnedThreadpoolsEv",
             "__ZN30qvac_lib_inference_addon_llama8batching7RequestC1EjO11PrefillPlanji",
             "__ZN30qvac_lib_inference_addon_llama5utils25partialSnapshotFromStreamERNSt3__113basic_istreamIcNS1_11char_traitsIcEEEEyiNS0_15SnapshotStorageERKNS1_12basic_stringIcS4_NS1_9allocatorIcEEEERKNS1_8functionIFvPKhmEEERNS0_21SequenceStateSnapshotE",
             "__ZN10LlamaModelD0Ev",

package/prebuilds/ios-arm64/qvac__llm-llamacpp.bare.exports

@@ -7,7 +7,7 @@
     ],
     "current_versions": [
       {
-        "version": "0.61"
+        "version": "0.61.2"
       }
     ],
     "exported_symbols": [
@@ -84,7 +84,6 @@
             "__ZN24llama_finetuning_helpers23optEpochCallbackWrapperEbP16ggml_opt_contextP16ggml_opt_datasetP15ggml_opt_resultxxx",
             "__ZThn24_N10LlamaModelD0Ev",
             "__ZN14MtmdLlmContext8setNPastEi",
-            "__ZN14TextLlmContext26initializeOwnedThreadpoolsEv",
             "__ZN30qvac_lib_inference_addon_llama8batching19MultiRequestBatcher10addRequestEONSt3__16vectorIiNS2_9allocatorIiEEEERj",
             "__ZN30qvac_lib_inference_addon_llama8batching19MultiRequestBatcher19sampleAndAppendIdleERKNSt3__18functionIFijiEEE",
             "__ZN10LlamaModel30discardFinetuneCancelSaveModesERKNSt3__16vectorIyNS0_9allocatorIyEEEE",

package/prebuilds/ios-arm64-simulator/qvac__llm-llamacpp.bare.exports

@@ -7,7 +7,7 @@
     ],
     "current_versions": [
       {
-        "version": "0.61"
+        "version": "0.61.2"
       }
     ],
     "exported_symbols": [
@@ -84,7 +84,6 @@
             "__ZN24llama_finetuning_helpers23optEpochCallbackWrapperEbP16ggml_opt_contextP16ggml_opt_datasetP15ggml_opt_resultxxx",
             "__ZThn24_N10LlamaModelD0Ev",
             "__ZN14MtmdLlmContext8setNPastEi",
-            "__ZN14TextLlmContext26initializeOwnedThreadpoolsEv",
             "__ZN30qvac_lib_inference_addon_llama8batching19MultiRequestBatcher10addRequestEONSt3__16vectorIiNS2_9allocatorIiEEEERj",
             "__ZN30qvac_lib_inference_addon_llama8batching19MultiRequestBatcher19sampleAndAppendIdleERKNSt3__18functionIFijiEEE",
             "__ZN10LlamaModel30discardFinetuneCancelSaveModesERKNSt3__16vectorIyNS0_9allocatorIyEEEE",

package/prebuilds/ios-x64-simulator/qvac__llm-llamacpp.bare.exports

@@ -7,7 +7,7 @@
     ],
     "current_versions": [
       {
-        "version": "0.61"
+        "version": "0.61.2"
       }
     ],
     "exported_symbols": [
@@ -811,7 +811,6 @@
             "__ZN24llama_finetuning_helpers23optEpochCallbackWrapperEbP16ggml_opt_contextP16ggml_opt_datasetP15ggml_opt_resultxxx",
             "__ZN14MtmdLlmContext8setNPastEi",
             "__ZThn24_N10LlamaModelD0Ev",
-            "__ZN14TextLlmContext26initializeOwnedThreadpoolsEv",
             "__ZN30qvac_lib_inference_addon_llama8batching7RequestC1EjO11PrefillPlanji",
             "__ZN30qvac_lib_inference_addon_llama5utils25partialSnapshotFromStreamERNSt3__113basic_istreamIcNS1_11char_traitsIcEEEEyiNS0_15SnapshotStorageERKNS1_12basic_stringIcS4_NS1_9allocatorIcEEEERKNS1_8functionIFvPKhmEEERNS0_21SequenceStateSnapshotE",
             "__ZN10LlamaModelD0Ev",