node-llama-cpp 2.7.4 → 2.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/README.md +3 -2
  2. package/dist/cli/commands/BuildCommand.d.ts +2 -2
  3. package/dist/cli/commands/BuildCommand.js +1 -1
  4. package/dist/cli/commands/BuildCommand.js.map +1 -1
  5. package/dist/cli/commands/ClearCommand.js +2 -1
  6. package/dist/cli/commands/ClearCommand.js.map +1 -1
  7. package/dist/cli/commands/DownloadCommand.d.ts +4 -5
  8. package/dist/cli/commands/DownloadCommand.js +3 -2
  9. package/dist/cli/commands/DownloadCommand.js.map +1 -1
  10. package/dist/commands.d.ts +2 -1
  11. package/dist/commands.js +2 -1
  12. package/dist/commands.js.map +1 -1
  13. package/dist/config.d.ts +1 -0
  14. package/dist/config.js +1 -0
  15. package/dist/config.js.map +1 -1
  16. package/dist/index.d.ts +2 -1
  17. package/dist/index.js +2 -1
  18. package/dist/index.js.map +1 -1
  19. package/dist/utils/cloneLlamaCppRepo.d.ts +1 -0
  20. package/dist/utils/cloneLlamaCppRepo.js +26 -1
  21. package/dist/utils/cloneLlamaCppRepo.js.map +1 -1
  22. package/dist/utils/getBuildDefaults.d.ts +6 -0
  23. package/dist/utils/getBuildDefaults.js +10 -0
  24. package/dist/utils/getBuildDefaults.js.map +1 -0
  25. package/dist/utils/getReleaseInfo.d.ts +7 -0
  26. package/dist/utils/getReleaseInfo.js +30 -0
  27. package/dist/utils/getReleaseInfo.js.map +1 -0
  28. package/llama/binariesGithubRelease.json +1 -1
  29. package/llama/gitRelease.bundle +0 -0
  30. package/llamaBins/linux-arm64/llama-addon.node +0 -0
  31. package/llamaBins/linux-armv7l/llama-addon.node +0 -0
  32. package/llamaBins/linux-x64/llama-addon.node +0 -0
  33. package/llamaBins/mac-arm64/ggml-metal.metal +162 -34
  34. package/llamaBins/mac-arm64/llama-addon.node +0 -0
  35. package/llamaBins/mac-x64/ggml-metal.metal +162 -34
  36. package/llamaBins/mac-x64/llama-addon.node +0 -0
  37. package/llamaBins/win-x64/llama-addon.exp +0 -0
  38. package/llamaBins/win-x64/llama-addon.lib +0 -0
  39. package/llamaBins/win-x64/llama-addon.node +0 -0
  40. package/package.json +2 -1
package/README.md CHANGED
@@ -1,7 +1,7 @@
1
1
  <div align="center">
2
2
  <img alt="node-llama-cpp Logo" src="https://media.githubusercontent.com/media/withcatai/node-llama-cpp/master/assets/logo.roundEdges.png" width="360px" />
3
- <h1>Node Llama.cpp</h1>
4
- <p>Node.js bindings for llama.cpp</p>
3
+ <h1>node-llama-cpp</h1>
4
+ <p>Run AI models locally on your machine</p>
5
5
  <sub>Pre-built bindings are provided with a fallback to building from source with cmake</sub>
6
6
  <p></p>
7
7
  </div>
@@ -29,6 +29,7 @@
29
29
  * [API reference](https://withcatai.github.io/node-llama-cpp/api/classes/LlamaModel)
30
30
  * [CLI help](https://withcatai.github.io/node-llama-cpp/guide/cli/)
31
31
  * [Changelog](https://github.com/withcatai/node-llama-cpp/releases)
32
+ * [Roadmap](https://github.com/orgs/withcatai/projects/1)
32
33
 
33
34
  ## Installation
34
35
  ```bash
package/dist/cli/commands/BuildCommand.d.ts CHANGED
@@ -2,8 +2,8 @@ import { CommandModule } from "yargs";
2
2
  type BuildCommand = {
3
3
  arch?: string;
4
4
  nodeTarget?: string;
5
- metal: boolean;
6
- cuda: boolean;
5
+ metal?: boolean;
6
+ cuda?: boolean;
7
7
  };
8
8
  export declare const BuildCommand: CommandModule<object, BuildCommand>;
9
9
  export declare function BuildLlamaCppCommand({ arch, nodeTarget, metal, cuda }: BuildCommand): Promise<void>;
package/dist/cli/commands/BuildCommand.js CHANGED
@@ -37,7 +37,7 @@ export const BuildCommand = {
37
37
  },
38
38
  handler: BuildLlamaCppCommand
39
39
  };
40
- export async function BuildLlamaCppCommand({ arch, nodeTarget, metal, cuda }) {
40
+ export async function BuildLlamaCppCommand({ arch = undefined, nodeTarget = undefined, metal = defaultLlamaCppMetalSupport, cuda = defaultLlamaCppCudaSupport }) {
41
41
  if (!(await fs.pathExists(llamaCppDirectory))) {
42
42
  console.log(chalk.red('llama.cpp is not downloaded. Please run "node-llama-cpp download" first'));
43
43
  process.exit(1);
package/dist/cli/commands/BuildCommand.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"BuildCommand.js","sourceRoot":"","sources":["../../../src/cli/commands/BuildCommand.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAE9B,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,MAAM,UAAU,CAAC;AAC1B,OAAO,EAAC,eAAe,EAAC,MAAM,gCAAgC,CAAC;AAC/D,OAAO,OAAO,MAAM,wBAAwB,CAAC;AAC7C,OAAO,EAAC,eAAe,EAAC,MAAM,gCAAgC,CAAC;AAC/D,OAAO,EAAC,0BAA0B,EAAE,2BAA2B,EAAE,iBAAiB,EAAC,MAAM,iBAAiB,CAAC;AAC3G,OAAO,EAAC,qBAAqB,EAAC,MAAM,sBAAsB,CAAC;AAC3D,OAAO,cAAc,MAAM,+BAA+B,CAAC;AAC3D,OAAO,EAAC,wBAAwB,EAAC,MAAM,gBAAgB,CAAC;AASxD,MAAM,CAAC,MAAM,YAAY,GAAwC;IAC7D,OAAO,EAAE,OAAO;IAChB,QAAQ,EAAE,4CAA4C;IACtD,OAAO,CAAC,KAAK;QACT,MAAM,qBAAqB,GAAG,wBAAwB,EAAE,CAAC;QAEzD,OAAO,KAAK;aACP,MAAM,CAAC,MAAM,EAAE;YACZ,KAAK,EAAE,GAAG;YACV,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,2CAA2C;SAC3D,CAAC;aACD,MAAM,CAAC,YAAY,EAAE;YAClB,KAAK,EAAE,GAAG;YACV,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,gEAAgE;SAChF,CAAC;aACD,MAAM,CAAC,OAAO,EAAE;YACb,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,2BAA2B,IAAI,qBAAqB;YAC7D,WAAW,EAAE,2KAA2K;SAC3L,CAAC;aACD,MAAM,CAAC,MAAM,EAAE;YACZ,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,0BAA0B;YACnC,WAAW,EAAE,uGAAuG;SACvH,CAAC,CAAC;IACX,CAAC;IACD,OAAO,EAAE,oBAAoB;CAChC,CAAC;AAEF,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,EAAC,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,IAAI,EAAe;IACpF,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,UAAU,CAAC,iBAAiB,CAAC,CAAC,EAAE;QAC3C,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,yEAAyE,CAAC,CAAC,CAAC;QAClG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACnB;IAED,IAAI,KAAK,IAAI,OAAO,CAAC,QAAQ,KAAK,QAAQ,EAAE;QACxC,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;KACpD;IAED,IAAI,IAAI,EAAE;QACN,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;KACnD;IAED,MAAM,qBAAqB,CAAC,IAAI,CAAC,CAAC;IAElC,MAAM,cAAc,CAAC;QACjB,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC;QAC1C,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC;QACzC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,6BAA6B,CAAC;KAClD,EAAE,KAAK,IAAI,EAAE;QACV,MAAM,eAAe,CAAC;YAClB,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;YAC7B,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;YAC/C,cAAc,EAAE,IAAI;YACpB,KAAK;YACL,IAAI;SACP,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;IAEH,MAAM,OAAO,CAAC;QACV,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,0BAA0B,CAAC;QAC/C,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,yBAAyB,CAAC;QAC9C,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,kCAAkC,CAAC;KACvD,EAAE,KAAK,IAAI,EAAE;QACV,MAAM,eAAe,EAAE,CAAC;IAC5B,CAAC,CAAC,CAAC;AACP,CAAC"}
1
+ {"version":3,"file":"BuildCommand.js","sourceRoot":"","sources":["../../../src/cli/commands/BuildCommand.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAE9B,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,MAAM,UAAU,CAAC;AAC1B,OAAO,EAAC,eAAe,EAAC,MAAM,gCAAgC,CAAC;AAC/D,OAAO,OAAO,MAAM,wBAAwB,CAAC;AAC7C,OAAO,EAAC,eAAe,EAAC,MAAM,gCAAgC,CAAC;AAC/D,OAAO,EAAC,0BAA0B,EAAE,2BAA2B,EAAE,iBAAiB,EAAC,MAAM,iBAAiB,CAAC;AAC3G,OAAO,EAAC,qBAAqB,EAAC,MAAM,sBAAsB,CAAC;AAC3D,OAAO,cAAc,MAAM,+BAA+B,CAAC;AAC3D,OAAO,EAAC,wBAAwB,EAAC,MAAM,gBAAgB,CAAC;AASxD,MAAM,CAAC,MAAM,YAAY,GAAwC;IAC7D,OAAO,EAAE,OAAO;IAChB,QAAQ,EAAE,4CAA4C;IACtD,OAAO,CAAC,KAAK;QACT,MAAM,qBAAqB,GAAG,wBAAwB,EAAE,CAAC;QAEzD,OAAO,KAAK;aACP,MAAM,CAAC,MAAM,EAAE;YACZ,KAAK,EAAE,GAAG;YACV,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,2CAA2C;SAC3D,CAAC;aACD,MAAM,CAAC,YAAY,EAAE;YAClB,KAAK,EAAE,GAAG;YACV,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,gEAAgE;SAChF,CAAC;aACD,MAAM,CAAC,OAAO,EAAE;YACb,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,2BAA2B,IAAI,qBAAqB;YAC7D,WAAW,EAAE,2KAA2K;SAC3L,CAAC;aACD,MAAM,CAAC,MAAM,EAAE;YACZ,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,0BAA0B;YACnC,WAAW,EAAE,uGAAuG;SACvH,CAAC,CAAC;IACX,CAAC;IACD,OAAO,EAAE,oBAAoB;CAChC,CAAC;AAEF,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,EACvC,IAAI,GAAG,SAAS,EAChB,UAAU,GAAG,SAAS,EACtB,KAAK,GAAG,2BAA2B,EACnC,IAAI,GAAG,0BAA0B,EACtB;IACX,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,UAAU,CAAC,iBAAiB,CAAC,CAAC,EAAE;QAC3C,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,yEAAyE,CAAC,CAAC,CAAC;QAClG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KACnB;IAED,IAAI,KAAK,IAAI,OAAO,CAAC,QAAQ,KAAK,QAAQ,EAAE;QACxC,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;KACpD;IAED,IAAI,IAAI,EAAE;QACN,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;KACnD;IAED,MAAM,qBAAqB,CAAC,IAAI,CAAC,CAAC;IAElC,MAAM,cAAc,CAAC;QACjB,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC;QAC1C,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC;QACzC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,6BAA6B,CAAC;KAClD,EAAE,KAAK,IAAI,EAAE;QACV,MAAM,eAAe,CAAC;YAClB,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;YAC7B,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;YAC/C,cAAc,EAAE,IAAI;YACpB,KAAK;YACL,IAAI;SACP,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;IAEH,MAAM,OAAO,CAAC;QACV,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,0BAA0B,CAAC;QAC/C,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,yBAAyB,CAAC;QAC9C,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,kCAAkC,CAAC;KACvD,EAAE,KAAK,IAAI,EAAE;QACV,MAAM,eAAe,EAAE,CAAC;IAC5B,CAAC,CAAC,CAAC;AACP,CAAC"}
package/dist/cli/commands/ClearCommand.js CHANGED
@@ -1,6 +1,6 @@
1
1
  import fs from "fs-extra";
2
2
  import chalk from "chalk";
3
- import { llamaCppDirectory } from "../../config.js";
3
+ import { llamaCppDirectory, llamaCppDirectoryTagFilePath } from "../../config.js";
4
4
  import withOra from "../../utils/withOra.js";
5
5
  import { clearLlamaBuild } from "../../utils/clearLlamaBuild.js";
6
6
  import { setUsedBinFlag } from "../../utils/usedBinFlag.js";
@@ -28,6 +28,7 @@ export async function ClearLlamaCppBuildCommand({ type }) {
28
28
  fail: chalk.blue("Failed to clear source")
29
29
  }, async () => {
30
30
  await fs.remove(llamaCppDirectory);
31
+ await fs.remove(llamaCppDirectoryTagFilePath);
31
32
  });
32
33
  }
33
34
  if (type === "build" || type === "all") {
package/dist/cli/commands/ClearCommand.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"ClearCommand.js","sourceRoot":"","sources":["../../../src/cli/commands/ClearCommand.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,UAAU,CAAC;AAC1B,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAC,iBAAiB,EAAC,MAAM,iBAAiB,CAAC;AAClD,OAAO,OAAO,MAAM,wBAAwB,CAAC;AAC7C,OAAO,EAAC,eAAe,EAAC,MAAM,gCAAgC,CAAC;AAC/D,OAAO,EAAC,cAAc,EAAC,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAC,eAAe,EAAE,mBAAmB,EAAC,MAAM,sBAAsB,CAAC;AAM1E,MAAM,CAAC,MAAM,YAAY,GAAwC;IAC7D,OAAO,EAAE,cAAc;IACvB,OAAO,EAAE,CAAC,OAAO,CAAC;IAClB,QAAQ,EAAE,uCAAuC;IACjD,OAAO,CAAC,KAAK;QACT,OAAO,KAAK;aACP,MAAM,CAAC,MAAM,EAAE;YACZ,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,CAAkC;YAC7E,OAAO,EAAE,KAA6B;YACtC,WAAW,EAAE,gBAAgB;SAChC,CAAC,CAAC;IACX,CAAC;IACD,OAAO,EAAE,yBAAyB;CACrC,CAAC;AAEF,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAAC,EAAC,IAAI,EAAe;IAChE,IAAI,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,KAAK,EAAE;QACrC,MAAM,OAAO,CAAC;YACV,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC;YACtC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC;YACrC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,wBAAwB,CAAC;SAC7C,EAAE,KAAK,IAAI,EAAE;YACV,MAAM,EAAE,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC;QACvC,CAAC,CAAC,CAAC;KACN;IAED,IAAI,IAAI,KAAK,OAAO,IAAI,IAAI,KAAK,KAAK,EAAE;QACpC,MAAM,OAAO,CAAC;YACV,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC;YACrC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC;YACpC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,uBAAuB,CAAC;SAC5C,EAAE,KAAK,IAAI,EAAE;YACV,MAAM,eAAe,EAAE,CAAC;QAC5B,CAAC,CAAC,CAAC;KACN;IAED,IAAI,IAAI,KAAK,OAAO,IAAI,IAAI,KAAK,KAAK,EAAE;QACpC,MAAM,OAAO,CAAC;YACV,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,yBAAyB,CAAC;YAC9C,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,wBAAwB,CAAC;YAC7C,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,gCAAgC,CAAC;SACrD,EAAE,KAAK,IAAI,EAAE;YACV,MAAM,mBAAmB,EAAE,CAAC;YAC5B,MAAM,eAAe,EAAE,CAAC;QAC5B,CAAC,CAAC,CAAC;KACN;IAED,MAAM,cAAc,CAAC,kBAAkB,CAAC,CAAC;AAC7C,CAAC"}
1
+ {"version":3,"file":"ClearCommand.js","sourceRoot":"","sources":["../../../src/cli/commands/ClearCommand.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,UAAU,CAAC;AAC1B,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAC,iBAAiB,EAAE,4BAA4B,EAAC,MAAM,iBAAiB,CAAC;AAChF,OAAO,OAAO,MAAM,wBAAwB,CAAC;AAC7C,OAAO,EAAC,eAAe,EAAC,MAAM,gCAAgC,CAAC;AAC/D,OAAO,EAAC,cAAc,EAAC,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAC,eAAe,EAAE,mBAAmB,EAAC,MAAM,sBAAsB,CAAC;AAM1E,MAAM,CAAC,MAAM,YAAY,GAAwC;IAC7D,OAAO,EAAE,cAAc;IACvB,OAAO,EAAE,CAAC,OAAO,CAAC;IAClB,QAAQ,EAAE,uCAAuC;IACjD,OAAO,CAAC,KAAK;QACT,OAAO,KAAK;aACP,MAAM,CAAC,MAAM,EAAE;YACZ,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,CAAkC;YAC7E,OAAO,EAAE,KAA6B;YACtC,WAAW,EAAE,gBAAgB;SAChC,CAAC,CAAC;IACX,CAAC;IACD,OAAO,EAAE,yBAAyB;CACrC,CAAC;AAEF,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAAC,EAAC,IAAI,EAAe;IAChE,IAAI,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,KAAK,EAAE;QACrC,MAAM,OAAO,CAAC;YACV,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC;YACtC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC;YACrC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,wBAAwB,CAAC;SAC7C,EAAE,KAAK,IAAI,EAAE;YACV,MAAM,EAAE,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC;YACnC,MAAM,EAAE,CAAC,MAAM,CAAC,4BAA4B,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;KACN;IAED,IAAI,IAAI,KAAK,OAAO,IAAI,IAAI,KAAK,KAAK,EAAE;QACpC,MAAM,OAAO,CAAC;YACV,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC;YACrC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC;YACpC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,uBAAuB,CAAC;SAC5C,EAAE,KAAK,IAAI,EAAE;YACV,MAAM,eAAe,EAAE,CAAC;QAC5B,CAAC,CAAC,CAAC;KACN;IAED,IAAI,IAAI,KAAK,OAAO,IAAI,IAAI,KAAK,KAAK,EAAE;QACpC,MAAM,OAAO,CAAC;YACV,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,yBAAyB,CAAC;YAC9C,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,wBAAwB,CAAC;YAC7C,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,gCAAgC,CAAC;SACrD,EAAE,KAAK,IAAI,EAAE;YACV,MAAM,mBAAmB,EAAE,CAAC;YAC5B,MAAM,eAAe,EAAE,CAAC;QAC5B,CAAC,CAAC,CAAC;KACN;IAED,MAAM,cAAc,CAAC,kBAAkB,CAAC,CAAC;AAC7C,CAAC"}
package/dist/cli/commands/DownloadCommand.d.ts CHANGED
@@ -1,14 +1,13 @@
1
1
  import { CommandModule } from "yargs";
2
2
  type DownloadCommandArgs = {
3
- repo: string;
4
- release: "latest" | string;
3
+ repo?: string;
4
+ release?: "latest" | string;
5
5
  arch?: string;
6
6
  nodeTarget?: string;
7
- metal: boolean;
8
- cuda: boolean;
7
+ metal?: boolean;
8
+ cuda?: boolean;
9
9
  skipBuild?: boolean;
10
10
  noBundle?: boolean;
11
- updateBinariesReleaseMetadataAndSaveGitBundle?: boolean;
12
11
  };
13
12
  export declare const DownloadCommand: CommandModule<object, DownloadCommandArgs>;
14
13
  export declare function DownloadLlamaCppCommand({ repo, release, arch, nodeTarget, metal, cuda, skipBuild, noBundle, updateBinariesReleaseMetadataAndSaveGitBundle }: DownloadCommandArgs): Promise<void>;
package/dist/cli/commands/DownloadCommand.js CHANGED
@@ -2,7 +2,7 @@ import process from "process";
2
2
  import { Octokit } from "octokit";
3
3
  import fs from "fs-extra";
4
4
  import chalk from "chalk";
5
- import { defaultLlamaCppCudaSupport, defaultLlamaCppGitHubRepo, defaultLlamaCppMetalSupport, defaultLlamaCppRelease, isCI, llamaCppDirectory } from "../../config.js";
5
+ import { defaultLlamaCppCudaSupport, defaultLlamaCppGitHubRepo, defaultLlamaCppMetalSupport, defaultLlamaCppRelease, isCI, llamaCppDirectory, llamaCppDirectoryTagFilePath } from "../../config.js";
6
6
  import { compileLlamaCpp } from "../../utils/compileLLamaCpp.js";
7
7
  import withOra from "../../utils/withOra.js";
8
8
  import { clearTempFolder } from "../../utils/clearTempFolder.js";
@@ -70,7 +70,7 @@ export const DownloadCommand = {
70
70
  },
71
71
  handler: DownloadLlamaCppCommand
72
72
  };
73
- export async function DownloadLlamaCppCommand({ repo, release, arch, nodeTarget, metal, cuda, skipBuild, noBundle, updateBinariesReleaseMetadataAndSaveGitBundle }) {
73
+ export async function DownloadLlamaCppCommand({ repo = defaultLlamaCppGitHubRepo, release = defaultLlamaCppRelease, arch = undefined, nodeTarget = undefined, metal = defaultLlamaCppMetalSupport, cuda = defaultLlamaCppCudaSupport, skipBuild = false, noBundle = false, updateBinariesReleaseMetadataAndSaveGitBundle = false }) {
74
74
  const useBundle = noBundle != true;
75
75
  const octokit = new Octokit();
76
76
  const [githubOwner, githubRepo] = repo.split("/");
@@ -128,6 +128,7 @@ export async function DownloadLlamaCppCommand({ repo, release, arch, nodeTarget,
128
128
  fail: chalk.blue("Failed to remove existing llama.cpp directory")
129
129
  }, async () => {
130
130
  await fs.remove(llamaCppDirectory);
131
+ await fs.remove(llamaCppDirectoryTagFilePath);
131
132
  });
132
133
  console.log(chalk.blue("Cloning llama.cpp"));
133
134
  await cloneLlamaCppRepo(githubOwner, githubRepo, githubReleaseTag, useBundle);
package/dist/cli/commands/DownloadCommand.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"DownloadCommand.js","sourceRoot":"","sources":["../../../src/cli/commands/DownloadCommand.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAE9B,OAAO,EAAC,OAAO,EAAC,MAAM,SAAS,CAAC;AAChC,OAAO,EAAE,MAAM,UAAU,CAAC;AAC1B,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EACH,0BAA0B,EAAE,yBAAyB,EAAE,2BAA2B,EAAE,sBAAsB,EAAE,IAAI,EAAE,iBAAiB,EACtI,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAC,eAAe,EAAC,MAAM,gCAAgC,CAAC;AAC/D,OAAO,OAAO,MAAM,wBAAwB,CAAC;AAC7C,OAAO,EAAC,eAAe,EAAC,MAAM,gCAAgC,CAAC;AAC/D,OAAO,EAAC,wBAAwB,EAAC,MAAM,sCAAsC,CAAC;AAC9E,OAAO,EAAC,qBAAqB,EAAC,MAAM,sBAAsB,CAAC;AAC3D,OAAO,cAAc,MAAM,+BAA+B,CAAC;AAC3D,OAAO,EAAC,wBAAwB,EAAC,MAAM,gBAAgB,CAAC;AACxD,OAAO,EACH,0BAA0B,EAC1B,qDAAqD,EACxD,MAAM,kCAAkC,CAAC;AAC1C,OAAO,EAAC,iBAAiB,EAAC,MAAM,kCAAkC,CAAC;AAcnE,MAAM,CAAC,MAAM,eAAe,GAA+C;IACvE,OAAO,EAAE,UAAU;IACnB,QAAQ,EAAE,gDAAgD;IAC1D,OAAO,CAAC,KAAK;QACT,MAAM,qBAAqB,GAAG,wBAAwB,EAAE,CAAC;QAEzD,OAAO,KAAK;aACP,MAAM,CAAC,MAAM,EAAE;YACZ,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,yBAAyB;YAClC,WAAW,EAAE,iIAAiI;SACjJ,CAAC;aACD,MAAM,CAAC,SAAS,EAAE;YACf,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,qBAAqB,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,sBAAsB;YAC3E,WAAW,EAAE,0KAA0K;SAC1L,CAAC;aACD,MAAM,CAAC,MAAM,EAAE;YACZ,KAAK,EAAE,GAAG;YACV,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,2CAA2C;SAC3D,CAAC;aACD,MAAM,CAAC,YAAY,EAAE;YAClB,KAAK,EAAE,GAAG;YACV,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,gEAAgE;SAChF,CAAC;aACD,MAAM,CAAC,OAAO,EAAE;YACb,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,2BAA2B,IAAI,qBAAqB;YAC7D,MAAM,EAAE,OAAO,CAAC,QAAQ,KAAK,QAAQ,IAAI,CAAC,qBAAqB;YAC/D,WAAW,EAAE,2KAA2K;SAC3L,CAAC;aACD,MAAM,CAAC,MAAM,EAAE;YACZ,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,0BAA0B;YACnC,WAAW,EAAE,uGAAuG;SACvH,CAAC;aACD,MAAM,CAAC,WAAW,EAAE;YACjB,KAAK,EAAE,IAAI;YACX,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,KAAK;YACd,WAAW,EAAE,8CAA8C;SAC9D,CAAC;aACD,MAAM,CAAC,UAAU,EAAE;YAChB,KAAK,EAAE,IAAI;YACX,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,KAAK;YACd,WAAW,EAAE,kGAAkG;SAClH,CAAC;aACD,MAAM,CAAC,+CAA+C,EAAE;YACrD,IAAI,EAAE,SAAS;YACf,MAAM,EAAE,IAAI;YACZ,OAAO,EAAE,KAAK;YACd,WAAW,EAAE,8FAA8F;SAC9G,CAAC,CAAC;IACX,CAAC;IACD,OAAO,EAAE,uBAAuB;CACnC,CAAC;AAEF,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,EAC1C,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,6CAA6C,EAC9F;IAClB,MAAM,SAAS,GAAG,QAAQ,IAAI,IAAI,CAAC;IACnC,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;IAC9B,MAAM,CAAC,WAAW,EAAE,UAAU,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAElD,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;IAChD,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,OAAO,EAAE,CAAC,CAAC;IACtD,IAAI,CAAC,SAAS,EAAE;QACZ,IAAI,KAAK,IAAI,OAAO,CAAC,QAAQ,KAAK,QAAQ,EAAE;YACxC,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;SACpD;QAED,IAAI,IAAI,EAAE;YACN,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;SACnD;KACJ;IACD,OAAO,CAAC,GAAG,EAAE,CAAC;IAKd,IAAI,gBAAgB,GAAkB,CAAC,SAAS,IAAI,CAAC,MAAM,0BAA0B,CAAC,WAAW,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC,IAAI,IAAI,CAAC;QAC7H,CAAC,CAAC,OAAO;QACT,CAAC,CAAC,IAAI,CAAC;IAEX,IAAI,gBAAgB,IAAI,IAAI;QACxB,MAAM,OAAO,CAAC;YACV,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,yBAAyB,CAAC;YAC9C,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,wBAAwB,CAAC;YAC7C,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,gCAAgC,CAAC;SACrD,EAAE,KAAK,IAAI,EAAE;YACV,IAAI,aAAa,GAA6B,IAAI,CAAC;YAEnD,IAAI;gBACA,IAAI,OAAO,KAAK,QAAQ,EAAE;oBACtB,aAAa,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC;wBACtD,KAAK,EAAE,WAAW;wBAClB,IAAI,EAAE,UAAU;qBACnB,CAAC,CAAC;iBACN;qBAAM;oBACH,aAAa,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC;wBACrD,KAAK,EAAE,WAAW;wBAClB,IAAI,EAAE,UAAU;wBAChB,GAAG,EAAE,OAAO;qBACf,CAAC,CAAC;iBACN;aACJ;YAAC,OAAO,GAAG,EAAE;gBACV,OAAO,CAAC,KAAK,CAAC,wCAAwC,EAAE,GAAG,CAAC,CAAC;aAChE;YAED,IAAI,aAAa,IAAI,IAAI,EAAE;gBACvB,MAAM,IAAI,KAAK,CAAC,2BAA2B,OAAO,SAAS,IAAI,GAAG,CAAC,CAAC;aACvE;YAED,IAAI,aAAa,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,EAAE;gBACrC,MAAM,IAAI,KAAK,CAAC,kCAAkC,OAAO,SAAS,IAAI,GAAG,CAAC,CAAC;aAC9E;YAED,gBAAgB,GAAG,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC;QACnD,CAAC,CAAC,CAAC;IAEP,MAAM,eAAe,EAAE,CAAC;IAExB,MAAM,OAAO,CAAC;QACV,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,uCAAuC,CAAC;QAC5D,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,sCAAsC,CAAC;QAC3D,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,+CAA+C,CAAC;KACpE,EAAE,KAAK,IAAI,EAAE;QACV,MAAM,EAAE,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC;IACvC,CAAC,CAAC,CAAC;IAEH,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC,CAAC;IAC7C,MAAM,iBAAiB,CAAC,WAAW,EAAE,UAAU,EAAE,gBAAiB,EAAE,SAAS,CAAC,CAAC;IAE/E,IAAI,CAAC,SAAS,EAAE;QACZ,MAAM,qBAAqB,CAAC,IAAI,CAAC,CAAC;QAElC,MAAM,cAAc,CAAC;YACjB,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC;YAC1C,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC;YACzC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,6BAA6B,CAAC;SAClD,EAAE,KAAK,IAAI,EAAE;YACV,MAAM,eAAe,CAAC;gBAClB,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;gBAC7B,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;gBAC/C,cAAc,EAAE,IAAI;gBACpB,KAAK;gBACL,IAAI;aACP,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;KACN;IAED,IAAI,IAAI,IAAI,6CAA6C,EAAE;QACvD,MAAM,wBAAwB,CAAC,gBAAiB,CAAC,CAAC;QAClD,MAAM,qDAAqD,EAAE,CAAC;KACjE;IAED,OAAO,CAAC,GAAG,EAAE,CAAC;IACd,OAAO,CAAC,GAAG,EAAE,CAAC;IACd,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;IAChD,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,OAAO,EAAE,CAAC,CAAC;IACtD,OAAO,CAAC,GAAG,EAAE,CAAC;IACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;AACrC,CAAC"}
1
+ {"version":3,"file":"DownloadCommand.js","sourceRoot":"","sources":["../../../src/cli/commands/DownloadCommand.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAE9B,OAAO,EAAC,OAAO,EAAC,MAAM,SAAS,CAAC;AAChC,OAAO,EAAE,MAAM,UAAU,CAAC;AAC1B,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EACH,0BAA0B,EAAE,yBAAyB,EAAE,2BAA2B,EAAE,sBAAsB,EAAE,IAAI,EAChH,iBAAiB,EAAE,4BAA4B,EAClD,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAC,eAAe,EAAC,MAAM,gCAAgC,CAAC;AAC/D,OAAO,OAAO,MAAM,wBAAwB,CAAC;AAC7C,OAAO,EAAC,eAAe,EAAC,MAAM,gCAAgC,CAAC;AAC/D,OAAO,EAAC,wBAAwB,EAAC,MAAM,sCAAsC,CAAC;AAC9E,OAAO,EAAC,qBAAqB,EAAC,MAAM,sBAAsB,CAAC;AAC3D,OAAO,cAAc,MAAM,+BAA+B,CAAC;AAC3D,OAAO,EAAC,wBAAwB,EAAC,MAAM,gBAAgB,CAAC;AACxD,OAAO,EACH,0BAA0B,EAC1B,qDAAqD,EACxD,MAAM,kCAAkC,CAAC;AAC1C,OAAO,EAAC,iBAAiB,EAAC,MAAM,kCAAkC,CAAC;AAgBnE,MAAM,CAAC,MAAM,eAAe,GAA+C;IACvE,OAAO,EAAE,UAAU;IACnB,QAAQ,EAAE,gDAAgD;IAC1D,OAAO,CAAC,KAAK;QACT,MAAM,qBAAqB,GAAG,wBAAwB,EAAE,CAAC;QAEzD,OAAO,KAAK;aACP,MAAM,CAAC,MAAM,EAAE;YACZ,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,yBAAyB;YAClC,WAAW,EAAE,iIAAiI;SACjJ,CAAC;aACD,MAAM,CAAC,SAAS,EAAE;YACf,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,qBAAqB,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,sBAAsB;YAC3E,WAAW,EAAE,0KAA0K;SAC1L,CAAC;aACD,MAAM,CAAC,MAAM,EAAE;YACZ,KAAK,EAAE,GAAG;YACV,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,2CAA2C;SAC3D,CAAC;aACD,MAAM,CAAC,YAAY,EAAE;YAClB,KAAK,EAAE,GAAG;YACV,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,gEAAgE;SAChF,CAAC;aACD,MAAM,CAAC,OAAO,EAAE;YACb,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,2BAA2B,IAAI,qBAAqB;YAC7D,MAAM,EAAE,OAAO,CAAC,QAAQ,KAAK,QAAQ,IAAI,CAAC,qBAAqB;YAC/D,WAAW,EAAE,2KAA2K;SAC3L,CAAC;aACD,MAAM,CAAC,MAAM,EAAE;YACZ,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,0BAA0B;YACnC,WAAW,EAAE,uGAAuG;SACvH,CAAC;aACD,MAAM,CAAC,WAAW,EAAE;YACjB,KAAK,EAAE,IAAI;YACX,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,KAAK;YACd,WAAW,EAAE,8CAA8C;SAC9D,CAAC;aACD,MAAM,CAAC,UAAU,EAAE;YAChB,KAAK,EAAE,IAAI;YACX,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,KAAK;YACd,WAAW,EAAE,kGAAkG;SAClH,CAAC;aACD,MAAM,CAAC,+CAA+C,EAAE;YACrD,IAAI,EAAE,SAAS;YACf,MAAM,EAAE,IAAI;YACZ,OAAO,EAAE,KAAK;YACd,WAAW,EAAE,8FAA8F;SAC9G,CAAC,CAAC;IACX,CAAC;IACD,OAAO,EAAE,uBAAuB;CACnC,CAAC;AAEF,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,EAC1C,IAAI,GAAG,yBAAyB,EAChC,OAAO,GAAG,sBAAsB,EAChC,IAAI,GAAG,SAAS,EAChB,UAAU,GAAG,SAAS,EACtB,KAAK,GAAG,2BAA2B,EACnC,IAAI,GAAG,0BAA0B,EACjC,SAAS,GAAG,KAAK,EACjB,QAAQ,GAAG,KAAK,EAChB,6CAA6C,GAAG,KAAK,EACnC;IAClB,MAAM,SAAS,GAAG,QAAQ,IAAI,IAAI,CAAC;IACnC,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;IAC9B,MAAM,CAAC,WAAW,EAAE,UAAU,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAElD,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;IAChD,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,OAAO,EAAE,CAAC,CAAC;IACtD,IAAI,CAAC,SAAS,EAAE;QACZ,IAAI,KAAK,IAAI,OAAO,CAAC,QAAQ,KAAK,QAAQ,EAAE;YACxC,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;SACpD;QAED,IAAI,IAAI,EAAE;YACN,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;SACnD;KACJ;IACD,OAAO,CAAC,GAAG,EAAE,CAAC;IAKd,IAAI,gBAAgB,GAAkB,CAAC,SAAS,IAAI,CAAC,MAAM,0BAA0B,CAAC,WAAW,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC,IAAI,IAAI,CAAC;QAC7H,CAAC,CAAC,OAAO;QACT,CAAC,CAAC,IAAI,CAAC;IAEX,IAAI,gBAAgB,IAAI,IAAI;QACxB,MAAM,OAAO,CAAC;YACV,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,yBAAyB,CAAC;YAC9C,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,wBAAwB,CAAC;YAC7C,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,gCAAgC,CAAC;SACrD,EAAE,KAAK,IAAI,EAAE;YACV,IAAI,aAAa,GAA6B,IAAI,CAAC;YAEnD,IAAI;gBACA,IAAI,OAAO,KAAK,QAAQ,EAAE;oBACtB,aAAa,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC;wBACtD,KAAK,EAAE,WAAW;wBAClB,IAAI,EAAE,UAAU;qBACnB,CAAC,CAAC;iBACN;qBAAM;oBACH,aAAa,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC;wBACrD,KAAK,EAAE,WAAW;wBAClB,IAAI,EAAE,UAAU;wBAChB,GAAG,EAAE,OAAO;qBACf,CAAC,CAAC;iBACN;aACJ;YAAC,OAAO,GAAG,EAAE;gBACV,OAAO,CAAC,KAAK,CAAC,wCAAwC,EAAE,GAAG,CAAC,CAAC;aAChE;YAED,IAAI,aAAa,IAAI,IAAI,EAAE;gBACvB,MAAM,IAAI,KAAK,CAAC,2BAA2B,OAAO,SAAS,IAAI,GAAG,CAAC,CAAC;aACvE;YAED,IAAI,aAAa,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,EAAE;gBACrC,MAAM,IAAI,KAAK,CAAC,kCAAkC,OAAO,SAAS,IAAI,GAAG,CAAC,CAAC;aAC9E;YAED,gBAAgB,GAAG,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC;QACnD,CAAC,CAAC,CAAC;IAEP,MAAM,eAAe,EAAE,CAAC;IAExB,MAAM,OAAO,CAAC;QACV,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,uCAAuC,CAAC;QAC5D,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,sCAAsC,CAAC;QAC3D,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,+CAA+C,CAAC;KACpE,EAAE,KAAK,IAAI,EAAE;QACV,MAAM,EAAE,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC;QACnC,MAAM,EAAE,CAAC,MAAM,CAAC,4BAA4B,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;IAEH,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC,CAAC;IAC7C,MAAM,iBAAiB,CAAC,WAAW,EAAE,UAAU,EAAE,gBAAiB,EAAE,SAAS,CAAC,CAAC;IAE/E,IAAI,CAAC,SAAS,EAAE;QACZ,MAAM,qBAAqB,CAAC,IAAI,CAAC,CAAC;QAElC,MAAM,cAAc,CAAC;YACjB,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC;YAC1C,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC;YACzC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,6BAA6B,CAAC;SAClD,EAAE,KAAK,IAAI,EAAE;YACV,MAAM,eAAe,CAAC;gBAClB,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;gBAC7B,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;gBAC/C,cAAc,EAAE,IAAI;gBACpB,KAAK;gBACL,IAAI;aACP,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;KACN;IAED,IAAI,IAAI,IAAI,6CAA6C,EAAE;QACvD,MAAM,wBAAwB,CAAC,gBAAiB,CAAC,CAAC;QAClD,MAAM,qDAAqD,EAAE,CAAC;KACjE;IAED,OAAO,CAAC,GAAG,EAAE,CAAC;IACd,OAAO,CAAC,GAAG,EAAE,CAAC;IACd,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;IAChD,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,OAAO,EAAE,CAAC,CAAC;IACtD,OAAO,CAAC,GAAG,EAAE,CAAC;IACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;AACrC,CAAC"}
package/dist/commands.d.ts CHANGED
@@ -1,4 +1,5 @@
1
1
  import { BuildLlamaCppCommand } from "./cli/commands/BuildCommand.js";
2
2
  import { DownloadLlamaCppCommand } from "./cli/commands/DownloadCommand.js";
3
3
  import { ClearLlamaCppBuildCommand } from "./cli/commands/ClearCommand.js";
4
- export { BuildLlamaCppCommand, DownloadLlamaCppCommand, ClearLlamaCppBuildCommand };
4
+ import { getBuildDefaults } from "./utils/getBuildDefaults.js";
5
+ export { BuildLlamaCppCommand, DownloadLlamaCppCommand, ClearLlamaCppBuildCommand, getBuildDefaults };
package/dist/commands.js CHANGED
@@ -1,5 +1,6 @@
1
1
  import { BuildLlamaCppCommand } from "./cli/commands/BuildCommand.js";
2
2
  import { DownloadLlamaCppCommand } from "./cli/commands/DownloadCommand.js";
3
3
  import { ClearLlamaCppBuildCommand } from "./cli/commands/ClearCommand.js";
4
- export { BuildLlamaCppCommand, DownloadLlamaCppCommand, ClearLlamaCppBuildCommand };
4
+ import { getBuildDefaults } from "./utils/getBuildDefaults.js";
5
+ export { BuildLlamaCppCommand, DownloadLlamaCppCommand, ClearLlamaCppBuildCommand, getBuildDefaults };
5
6
  //# sourceMappingURL=commands.js.map
package/dist/commands.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"commands.js","sourceRoot":"","sources":["../src/commands.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,oBAAoB,EAAC,MAAM,gCAAgC,CAAC;AACpE,OAAO,EAAC,uBAAuB,EAAC,MAAM,mCAAmC,CAAC;AAC1E,OAAO,EAAC,yBAAyB,EAAC,MAAM,gCAAgC,CAAC;AAEzE,OAAO,EAAC,oBAAoB,EAAE,uBAAuB,EAAE,yBAAyB,EAAC,CAAC"}
1
+ {"version":3,"file":"commands.js","sourceRoot":"","sources":["../src/commands.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,oBAAoB,EAAC,MAAM,gCAAgC,CAAC;AACpE,OAAO,EAAC,uBAAuB,EAAC,MAAM,mCAAmC,CAAC;AAC1E,OAAO,EAAC,yBAAyB,EAAC,MAAM,gCAAgC,CAAC;AACzE,OAAO,EAAC,gBAAgB,EAAC,MAAM,6BAA6B,CAAC;AAE7D,OAAO,EAAC,oBAAoB,EAAE,uBAAuB,EAAE,yBAAyB,EAAE,gBAAgB,EAAC,CAAC"}
package/dist/config.d.ts CHANGED
@@ -8,6 +8,7 @@ export declare const tempDownloadDirectory: string;
8
8
  export declare const chatCommandHistoryFilePath: string;
9
9
  export declare const usedBinFlagJsonPath: string;
10
10
  export declare const binariesGithubReleasePath: string;
11
+ export declare const llamaCppDirectoryTagFilePath: string;
11
12
  export declare const currentReleaseGitBundlePath: string;
12
13
  export declare const xpackDirectory: string;
13
14
  export declare const localXpacksStoreDirectory: string;
package/dist/config.js CHANGED
@@ -17,6 +17,7 @@ export const tempDownloadDirectory = path.join(os.tmpdir(), "node-llama-cpp", uu
17
17
  export const chatCommandHistoryFilePath = path.join(os.homedir(), ".node-llama-cpp.chat_repl_history");
18
18
  export const usedBinFlagJsonPath = path.join(llamaDirectory, "usedBin.json");
19
19
  export const binariesGithubReleasePath = path.join(llamaDirectory, "binariesGithubRelease.json");
20
+ export const llamaCppDirectoryTagFilePath = path.join(llamaDirectory, "llama.cpp.tag.json");
20
21
  export const currentReleaseGitBundlePath = path.join(llamaDirectory, "gitRelease.bundle");
21
22
  export const xpackDirectory = path.join(llamaDirectory, "xpack");
22
23
  export const localXpacksStoreDirectory = path.join(xpackDirectory, "store");
package/dist/config.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,aAAa,EAAC,MAAM,KAAK,CAAC;AAClC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,MAAM,MAAM,SAAS,CAAC;AAC7B,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAC,wBAAwB,EAAC,MAAM,kCAAkC,CAAC;AAE1E,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAE/D,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;AAGrC,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;AAClE,MAAM,CAAC,MAAM,wBAAwB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,YAAY,CAAC,CAAC;AAChF,MAAM,CAAC,MAAM,kBAAkB,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,WAAW,CAAC,CAAC;AAC1E,MAAM,CAAC,MAAM,0BAA0B,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;AAC1F,MAAM,CAAC,MAAM,iBAAiB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC;AACxE,MAAM,CAAC,MAAM,yBAAyB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC;AAC5F,MAAM,CAAC,MAAM,qBAAqB,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,gBAAgB,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;AACzF,MAAM,CAAC,MAAM,0BAA0B,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,mCAAmC,CAAC,CAAC;AACvG,MAAM,CAAC,MAAM,mBAAmB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,cAAc,CAAC,CAAC;AAC7E,MAAM,CAAC,MAAM,yBAAyB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,4BAA4B,CAAC,CAAC;AACjG,MAAM,CAAC,MAAM,2BAA2B,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,mBAAmB,CAAC,CAAC;AAC1F,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC;AACjE,MAAM,CAAC,MAAM,yBAAyB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC;AAC5E,MAAM,CAAC,MAAM,yBAAyB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC;AAC5E,MAAM,CAAC,MAAM,UAAU,GAAG,SAAS,CAAC;AAEpC,MAAM,CAAC,MAAM,IAAI,GAAG,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC;KAC5B,OAAO,CAAC,OAAO,CAAC;KAChB,MAAM,EAAE,CAAC;AACd,MAAM,CAAC,MAAM,yBAAyB,GAAG,GAAG,CAAC,GAAG,CAAC,qBAAqB,CAAC;KAClE,OAAO,CAAC,qBAAqB,CAAC;KAC9B,QAAQ,EAAE,CAAC;AAChB,MAAM,CAAC,MAAM,sBAAsB,GAAG,GAAG,CAAC,GAAG,CAAC,6BAA6B,CAAC;KACvE,OAAO,CAAC,MAAM,wBAAwB,EAAE,CAAC;KACzC,QAAQ,EAAE,CAAC;AAChB,MAAM,CAAC,MAAM,2BAA2B,GAAG,GAAG,CAAC,GAAG,CAAC,sBAAsB,CAAC;KACrE,OAAO,CAAC,OAAO,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC;KACzD,MAAM,EAAE,CAAC;AACd,MAAM,CAAC,MAAM,0BAA0B,GAAG,GAAG,CAAC,GAAG,CAAC,qBAAqB,CAAC;KACnE,OAAO,CAAC,OAAO,CAAC;KAChB,MAAM,EAAE,CAAC;AACd,MAAM,CAAC,MAAM,mBAAmB,GAAG,GAAG,CAAC,GAAG,CAAC,8BAA8B,CAAC;KACrE,OAAO,CAAC,OAAO,CAAC;KAChB,MAAM,EAAE,CAAC;AACd,MAAM,CAAC,MAAM,2BAA2B,GAAG,GAAG,CAAC,GAAG,CAAC,oCAAoC,CAAC;KACnF,OAAO,CAAC,yBAAyB,CAAC;KAClC,QAAQ,EAAE,CAAC;AAChB,MAAM,CAAC,MAAM,2BAA2B,GAAG,GAAG,CAAC,GAAG,CAAC,oCAAoC,CAAC;KACnF,OAAO,CAAC,yBAAyB,CAAC;KAClC,QAAQ,EAAE,CAAC;AAChB,MAAM,CAAC,MAAM,8BAA8B,GAAG,8BAA8B,CAAC;AAC7E,MAAM,CAAC,MAAM,uBAAuB,GAAG,+FAA+F;IAClI,+HAA+H;IAC/H,mFAAmF,CAAC;AACxF,MAAM,CAAC,MAAM,UAAU,GAAG,gBAAgB,CAAC;AAC3C,MAAM,CAAC,MAAM,YAAY,GAAG,WAAW,CAAC;AAExC,MAAM,gBAAgB,GAAG,4CAA4C,CAAC;AACtE,MAAM,CAAC,MAAM,qBAAqB,GAAG;IACjC,IAAI,EAAE,gBAAgB,GAAG,aAAa;CAChC,CAAC"}
1
+ {"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,aAAa,EAAC,MAAM,KAAK,CAAC;AAClC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,MAAM,MAAM,SAAS,CAAC;AAC7B,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAC,wBAAwB,EAAC,MAAM,kCAAkC,CAAC;AAE1E,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAE/D,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;AAGrC,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;AAClE,MAAM,CAAC,MAAM,wBAAwB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,YAAY,CAAC,CAAC;AAChF,MAAM,CAAC,MAAM,kBAAkB,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,WAAW,CAAC,CAAC;AAC1E,MAAM,CAAC,MAAM,0BAA0B,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;AAC1F,MAAM,CAAC,MAAM,iBAAiB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC;AACxE,MAAM,CAAC,MAAM,yBAAyB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC;AAC5F,MAAM,CAAC,MAAM,qBAAqB,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,gBAAgB,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;AACzF,MAAM,CAAC,MAAM,0BAA0B,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,mCAAmC,CAAC,CAAC;AACvG,MAAM,CAAC,MAAM,mBAAmB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,cAAc,CAAC,CAAC;AAC7E,MAAM,CAAC,MAAM,yBAAyB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,4BAA4B,CAAC,CAAC;AACjG,MAAM,CAAC,MAAM,4BAA4B,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,oBAAoB,CAAC,CAAC;AAC5F,MAAM,CAAC,MAAM,2BAA2B,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,mBAAmB,CAAC,CAAC;AAC1F,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC;AACjE,MAAM,CAAC,MAAM,yBAAyB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC;AAC5E,MAAM,CAAC,MAAM,yBAAyB,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC;AAC5E,MAAM,CAAC,MAAM,UAAU,GAAG,SAAS,CAAC;AAEpC,MAAM,CAAC,MAAM,IAAI,GAAG,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC;KAC5B,OAAO,CAAC,OAAO,CAAC;KAChB,MAAM,EAAE,CAAC;AACd,MAAM,CAAC,MAAM,yBAAyB,GAAG,GAAG,CAAC,GAAG,CAAC,qBAAqB,CAAC;KAClE,OAAO,CAAC,qBAAqB,CAAC;KAC9B,QAAQ,EAAE,CAAC;AAChB,MAAM,CAAC,MAAM,sBAAsB,GAAG,GAAG,CAAC,GAAG,CAAC,6BAA6B,CAAC;KACvE,OAAO,CAAC,MAAM,wBAAwB,EAAE,CAAC;KACzC,QAAQ,EAAE,CAAC;AAChB,MAAM,CAAC,MAAM,2BAA2B,GAAG,GAAG,CAAC,GAAG,CAAC,sBAAsB,CAAC;KACrE,OAAO,CAAC,OAAO,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC;KACzD,MAAM,EAAE,CAAC;AACd,MAAM,CAAC,MAAM,0BAA0B,GAAG,GAAG,CAAC,GAAG,CAAC,qBAAqB,CAAC;KACnE,OAAO,CAAC,OAAO,CAAC;KAChB,MAAM,EAAE,CAAC;AACd,MAAM,CAAC,MAAM,mBAAmB,GAAG,GAAG,CAAC,GAAG,CAAC,8BAA8B,CAAC;KACrE,OAAO,CAAC,OAAO,CAAC;KAChB,MAAM,EAAE,CAAC;AACd,MAAM,CAAC,MAAM,2BAA2B,GAAG,GAAG,CAAC,GAAG,CAAC,oCAAoC,CAAC;KACnF,OAAO,CAAC,yBAAyB,CAAC;KAClC,QAAQ,EAAE,CAAC;AAChB,MAAM,CAAC,MAAM,2BAA2B,GAAG,GAAG,CAAC,GAAG,CAAC,oCAAoC,CAAC;KACnF,OAAO,CAAC,yBAAyB,CAAC;KAClC,QAAQ,EAAE,CAAC;AAChB,MAAM,CAAC,MAAM,8BAA8B,GAAG,8BAA8B,CAAC;AAC7E,MAAM,CAAC,MAAM,uBAAuB,GAAG,+FAA+F;IAClI,+HAA+H;IAC/H,mFAAmF,CAAC;AACxF,MAAM,CAAC,MAAM,UAAU,GAAG,gBAAgB,CAAC;AAC3C,MAAM,CAAC,MAAM,YAAY,GAAG,WAAW,CAAC;AAExC,MAAM,gBAAgB,GAAG,4CAA4C,CAAC;AACtE,MAAM,CAAC,MAAM,qBAAqB,GAAG;IACjC,IAAI,EAAE,gBAAgB,GAAG,aAAa;CAChC,CAAC"}
package/dist/index.d.ts CHANGED
@@ -13,6 +13,7 @@ import { GeneralChatPromptWrapper } from "./chatWrappers/GeneralChatPromptWrappe
13
13
  import { ChatMLChatPromptWrapper } from "./chatWrappers/ChatMLChatPromptWrapper.js";
14
14
  import { FalconChatPromptWrapper } from "./chatWrappers/FalconChatPromptWrapper.js";
15
15
  import { getChatWrapperByBos } from "./chatWrappers/createChatWrapperByBos.js";
16
+ import { getReleaseInfo } from "./utils/getReleaseInfo.js";
16
17
  import { type ConversationInteraction, type Token } from "./types.js";
17
18
  import { type GbnfJsonArraySchema, type GbnfJsonBasicSchema, type GbnfJsonConstSchema, type GbnfJsonEnumSchema, type GbnfJsonObjectSchema, type GbnfJsonOneOfSchema, type GbnfJsonSchema, type GbnfJsonSchemaImmutableType, type GbnfJsonSchemaToType } from "./utils/gbnfJson/types.js";
18
- export { LlamaModel, type LlamaModelOptions, LlamaGrammar, type LlamaGrammarOptions, LlamaJsonSchemaGrammar, LlamaJsonSchemaValidationError, LlamaGrammarEvaluationState, type LlamaGrammarEvaluationStateOptions, LlamaContext, type LlamaContextOptions, type LlamaContextRepeatPenalty, LlamaChatSession, type LlamaChatSessionOptions, type LLamaChatPromptOptions, type LlamaChatSessionRepeatPenalty, type ConversationInteraction, AbortError, ChatPromptWrapper, EmptyChatPromptWrapper, LlamaChatPromptWrapper, GeneralChatPromptWrapper, ChatMLChatPromptWrapper, FalconChatPromptWrapper, getChatWrapperByBos, type Token, type GbnfJsonSchema, type GbnfJsonSchemaToType, type GbnfJsonSchemaImmutableType, type GbnfJsonBasicSchema, type GbnfJsonConstSchema, type GbnfJsonEnumSchema, type GbnfJsonOneOfSchema, type GbnfJsonObjectSchema, type GbnfJsonArraySchema };
19
+ export { LlamaModel, type LlamaModelOptions, LlamaGrammar, type LlamaGrammarOptions, LlamaJsonSchemaGrammar, LlamaJsonSchemaValidationError, LlamaGrammarEvaluationState, type LlamaGrammarEvaluationStateOptions, LlamaContext, type LlamaContextOptions, type LlamaContextRepeatPenalty, LlamaChatSession, type LlamaChatSessionOptions, type LLamaChatPromptOptions, type LlamaChatSessionRepeatPenalty, type ConversationInteraction, AbortError, ChatPromptWrapper, EmptyChatPromptWrapper, LlamaChatPromptWrapper, GeneralChatPromptWrapper, ChatMLChatPromptWrapper, FalconChatPromptWrapper, getChatWrapperByBos, getReleaseInfo, type Token, type GbnfJsonSchema, type GbnfJsonSchemaToType, type GbnfJsonSchemaImmutableType, type GbnfJsonBasicSchema, type GbnfJsonConstSchema, type GbnfJsonEnumSchema, type GbnfJsonOneOfSchema, type GbnfJsonObjectSchema, type GbnfJsonArraySchema };
package/dist/index.js CHANGED
@@ -13,5 +13,6 @@ import { GeneralChatPromptWrapper } from "./chatWrappers/GeneralChatPromptWrappe
13
13
  import { ChatMLChatPromptWrapper } from "./chatWrappers/ChatMLChatPromptWrapper.js";
14
14
  import { FalconChatPromptWrapper } from "./chatWrappers/FalconChatPromptWrapper.js";
15
15
  import { getChatWrapperByBos } from "./chatWrappers/createChatWrapperByBos.js";
16
- export { LlamaModel, LlamaGrammar, LlamaJsonSchemaGrammar, LlamaJsonSchemaValidationError, LlamaGrammarEvaluationState, LlamaContext, LlamaChatSession, AbortError, ChatPromptWrapper, EmptyChatPromptWrapper, LlamaChatPromptWrapper, GeneralChatPromptWrapper, ChatMLChatPromptWrapper, FalconChatPromptWrapper, getChatWrapperByBos };
16
+ import { getReleaseInfo } from "./utils/getReleaseInfo.js";
17
+ export { LlamaModel, LlamaGrammar, LlamaJsonSchemaGrammar, LlamaJsonSchemaValidationError, LlamaGrammarEvaluationState, LlamaContext, LlamaChatSession, AbortError, ChatPromptWrapper, EmptyChatPromptWrapper, LlamaChatPromptWrapper, GeneralChatPromptWrapper, ChatMLChatPromptWrapper, FalconChatPromptWrapper, getChatWrapperByBos, getReleaseInfo };
17
18
  //# sourceMappingURL=index.js.map
package/dist/index.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,UAAU,EAAyB,MAAM,gCAAgC,CAAC;AAClF,OAAO,EAAC,YAAY,EAA2B,MAAM,kCAAkC,CAAC;AACxF,OAAO,EAAC,sBAAsB,EAAC,MAAM,4CAA4C,CAAC;AAClF,OAAO,EAAC,8BAA8B,EAAC,MAAM,2DAA2D,CAAC;AACzG,OAAO,EAAC,2BAA2B,EAAqC,MAAM,iDAAiD,CAAC;AAChI,OAAO,EAAC,YAAY,EAA2D,MAAM,kCAAkC,CAAC;AACxH,OAAO,EACH,gBAAgB,EACnB,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAC3C,OAAO,EAAC,iBAAiB,EAAC,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAC,sBAAsB,EAAC,MAAM,0CAA0C,CAAC;AAChF,OAAO,EAAC,sBAAsB,EAAC,MAAM,0CAA0C,CAAC;AAChF,OAAO,EAAC,wBAAwB,EAAC,MAAM,4CAA4C,CAAC;AACpF,OAAO,EAAC,uBAAuB,EAAC,MAAM,2CAA2C,CAAC;AAClF,OAAO,EAAC,uBAAuB,EAAC,MAAM,2CAA2C,CAAC;AAClF,OAAO,EAAC,mBAAmB,EAAC,MAAM,0CAA0C,CAAC;AAS7E,OAAO,EACH,UAAU,EAEV,YAAY,EAEZ,sBAAsB,EACtB,8BAA8B,EAC9B,2BAA2B,EAE3B,YAAY,EAGZ,gBAAgB,EAKhB,UAAU,EACV,iBAAiB,EACjB,sBAAsB,EACtB,sBAAsB,EACtB,wBAAwB,EACxB,uBAAuB,EACvB,uBAAuB,EACvB,mBAAmB,EAWtB,CAAC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,UAAU,EAAyB,MAAM,gCAAgC,CAAC;AAClF,OAAO,EAAC,YAAY,EAA2B,MAAM,kCAAkC,CAAC;AACxF,OAAO,EAAC,sBAAsB,EAAC,MAAM,4CAA4C,CAAC;AAClF,OAAO,EAAC,8BAA8B,EAAC,MAAM,2DAA2D,CAAC;AACzG,OAAO,EAAC,2BAA2B,EAAqC,MAAM,iDAAiD,CAAC;AAChI,OAAO,EAAC,YAAY,EAA2D,MAAM,kCAAkC,CAAC;AACxH,OAAO,EACH,gBAAgB,EACnB,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAC3C,OAAO,EAAC,iBAAiB,EAAC,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAC,sBAAsB,EAAC,MAAM,0CAA0C,CAAC;AAChF,OAAO,EAAC,sBAAsB,EAAC,MAAM,0CAA0C,CAAC;AAChF,OAAO,EAAC,wBAAwB,EAAC,MAAM,4CAA4C,CAAC;AACpF,OAAO,EAAC,uBAAuB,EAAC,MAAM,2CAA2C,CAAC;AAClF,OAAO,EAAC,uBAAuB,EAAC,MAAM,2CAA2C,CAAC;AAClF,OAAO,EAAC,mBAAmB,EAAC,MAAM,0CAA0C,CAAC;AAC7E,OAAO,EAAC,cAAc,EAAC,MAAM,2BAA2B,CAAC;AASzD,OAAO,EACH,UAAU,EAEV,YAAY,EAEZ,sBAAsB,EACtB,8BAA8B,EAC9B,2BAA2B,EAE3B,YAAY,EAGZ,gBAAgB,EAKhB,UAAU,EACV,iBAAiB,EACjB,sBAAsB,EACtB,sBAAsB,EACtB,wBAAwB,EACxB,uBAAuB,EACvB,uBAAuB,EACvB,mBAAmB,EACnB,cAAc,EAWjB,CAAC"}
package/dist/utils/cloneLlamaCppRepo.d.ts CHANGED
@@ -1 +1,2 @@
1
1
  export declare function cloneLlamaCppRepo(githubOwner: string, githubRepo: string, tag: string, useBundles?: boolean): Promise<void>;
2
+ export declare function getClonedLlamaCppRepoReleaseTag(): Promise<string | null>;
package/dist/utils/cloneLlamaCppRepo.js CHANGED
@@ -2,7 +2,7 @@ import simpleGit from "simple-git";
2
2
  import cliProgress from "cli-progress";
3
3
  import chalk from "chalk";
4
4
  import fs from "fs-extra";
5
- import { llamaCppDirectory } from "../config.js";
5
+ import { llamaCppDirectory, llamaCppDirectoryTagFilePath } from "../config.js";
6
6
  import { getGitBundlePathForRelease } from "./gitReleaseBundles.js";
7
7
  export async function cloneLlamaCppRepo(githubOwner, githubRepo, tag, useBundles = true) {
8
8
  const gitBundleForTag = !useBundles ? null : await getGitBundlePathForRelease(githubOwner, githubRepo, tag);
@@ -45,6 +45,7 @@ export async function cloneLlamaCppRepo(githubOwner, githubRepo, tag, useBundles
45
45
  }
46
46
  catch (err) {
47
47
  await fs.remove(llamaCppDirectory);
48
+ await fs.remove(llamaCppDirectoryTagFilePath);
48
49
  console.error("Failed to clone git bundle, cloning from GitHub instead", err);
49
50
  printCloneErrorHelp(String(err));
50
51
  }
@@ -62,6 +63,18 @@ export async function cloneLlamaCppRepo(githubOwner, githubRepo, tag, useBundles
62
63
  printCloneErrorHelp(String(err));
63
64
  throw err;
64
65
  }
66
+ try {
67
+ const clonedLlamaCppRepoTagJson = {
68
+ tag
69
+ };
70
+ await fs.writeJson(llamaCppDirectoryTagFilePath, clonedLlamaCppRepoTagJson, {
71
+ spaces: 4
72
+ });
73
+ }
74
+ catch (err) {
75
+ console.error("Failed to write llama.cpp tag file", err);
76
+ throw err;
77
+ }
65
78
  }
66
79
  function printCloneErrorHelp(error) {
67
80
  // This error happens with some docker images where the current user is different
@@ -74,4 +87,16 @@ function printCloneErrorHelp(error) {
74
87
  chalk.yellow("Or run this command to fix it everywhere:") + "\n" +
75
88
  'git config --global --add safe.directory "*"');
76
89
  }
90
+ export async function getClonedLlamaCppRepoReleaseTag() {
91
+ if (!(await fs.pathExists(llamaCppDirectoryTagFilePath)))
92
+ return null;
93
+ try {
94
+ const clonedLlamaCppRepoTagJson = await fs.readJson(llamaCppDirectoryTagFilePath);
95
+ return clonedLlamaCppRepoTagJson.tag;
96
+ }
97
+ catch (err) {
98
+ console.error("Failed to read llama.cpp tag file", err);
99
+ return null;
100
+ }
101
+ }
77
102
  //# sourceMappingURL=cloneLlamaCppRepo.js.map
package/dist/utils/cloneLlamaCppRepo.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"cloneLlamaCppRepo.js","sourceRoot":"","sources":["../../src/utils/cloneLlamaCppRepo.ts"],"names":[],"mappings":"AAAA,OAAO,SAAsB,MAAM,YAAY,CAAC;AAChD,OAAO,WAAW,MAAM,cAAc,CAAC;AACvC,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,MAAM,UAAU,CAAC;AAC1B,OAAO,EAAC,iBAAiB,EAAC,MAAM,cAAc,CAAC;AAC/C,OAAO,EAAC,0BAA0B,EAAC,MAAM,wBAAwB,CAAC;AAGlE,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,WAAmB,EAAE,UAAkB,EAAE,GAAW,EAAE,aAAsB,IAAI;IACpH,MAAM,eAAe,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,0BAA0B,CAAC,WAAW,EAAE,UAAU,EAAE,GAAG,CAAC,CAAC;IAC5G,MAAM,YAAY,GAAG,sBAAsB,WAAW,IAAI,UAAU,MAAM,CAAC;IAE3E,KAAK,UAAU,oBAAoB,CAAI,SAAiB,EAAE,QAAyD;QAC/G,MAAM,WAAW,GAAG,IAAI,WAAW,CAAC,GAAG,CAAC;YACpC,eAAe,EAAE,KAAK;YACtB,UAAU,EAAE,IAAI;YAChB,WAAW,EAAE,IAAI;YACjB,MAAM,EAAE,GAAG,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,KAAK,KAAK,CAAC,MAAM,CAAC,eAAe,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,EAAE;SACpI,EAAE,WAAW,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;QAEvC,WAAW,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE;YACtB,KAAK,EAAE,EAAE;YACT,IAAI,EAAE,GAAG,WAAW,IAAI,UAAU,KAAK,SAAS,GAAG;SACtD,CAAC,CAAC;QAEH,MAAM,oBAAoB,GAAG,SAAS,CAAC;YACnC,QAAQ,CAAC,EAAC,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAC;gBACjC,MAAM,aAAa,GAAG,CAAC,SAAS,GAAG,GAAG,CAAC,GAAG,CAAC,QAAQ,GAAG,KAAK,CAAC,CAAC;gBAE7D,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC;YAChE,CAAC;SACJ,CAAC,CAAC;QAEH,IAAI;YACA,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,oBAAoB,CAAC,CAAC;YAEjD,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAExB,OAAO,GAAG,CAAC;SACd;gBAAS;YACN,WAAW,CAAC,IAAI,EAAE,CAAC;SACtB;IACL,CAAC;IAED,IAAI,eAAe,IAAI,IAAI,EAAE;QACzB,IAAI;YACA,MAAM,oBAAoB,CAAC,cAAc,EAAE,KAAK,EAAE,oBAAoB,EAAE,EAAE;gBACtE,MAAM,oBAAoB,CAAC,KAAK,CAAC,eAAe,EAAE,iBAAiB,EAAE;oBACjE,SAAS,EAAE,IAAI;iBAClB,CAAC,CAAC;gBAEH,MAAM,SAAS,CAAC,iBAAiB,CAAC,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;YAC9D,CAAC,CAAC,CAAC;YACH,OAAO;SACV;QAAC,OAAO,GAAG,EAAE;YACV,MAAM,EAAE,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC;YACnC,OAAO,CAAC,KAAK,CAAC,yDAAyD,EAAE,GAAG,CAAC,CAAC;YAE9E,mBAAmB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;SACpC;KACJ;IAED,IAAI;QACA,MAAM,oBAAoB,CAAC,QAAQ,EAAE,KAAK,EAAE,oBAAoB,EAAE,EAAE;YAChE,MAAM,oBAAoB,CAAC,KAAK,CAAC,YAAY,EAAE,iBAAiB,EAAE;gBAC9D,SAAS,EAAE,CAAC;gBACZ,UAAU,EAAE,GAAG;gBACf,SAAS,EAAE,IAAI;aAClB,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;KACN;IAAC,OAAO,GAAG,EAAE;QACV,mBAAmB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;QAEjC,MAAM,GAAG,CAAC;KACb;AACL,CAAC;AAED,SAAS,mBAAmB,CAAC,KAAa;IACtC,iFAAiF;IACjF,wDAAwD;IACxD,6EAA6E;IAC7E,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,0CAA0C,CAAC;QACxE,OAAO,CAAC,IAAI,CAAC,IAAI;YACb,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,0FAA0F,CAAC,GAAG,IAAI;YAChJ,4CAA4C,GAAG,iBAAiB,GAAG,OAAO;YAC1E,KAAK,CAAC,MAAM,CAAC,2CAA2C,CAAC,GAAG,IAAI;YAChE,8CAA8C,CACjD,CAAC;AACV,CAAC"}
1
+ {"version":3,"file":"cloneLlamaCppRepo.js","sourceRoot":"","sources":["../../src/utils/cloneLlamaCppRepo.ts"],"names":[],"mappings":"AAAA,OAAO,SAAsB,MAAM,YAAY,CAAC;AAChD,OAAO,WAAW,MAAM,cAAc,CAAC;AACvC,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,MAAM,UAAU,CAAC;AAC1B,OAAO,EAAC,iBAAiB,EAAE,4BAA4B,EAAC,MAAM,cAAc,CAAC;AAC7E,OAAO,EAAC,0BAA0B,EAAC,MAAM,wBAAwB,CAAC;AAOlE,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,WAAmB,EAAE,UAAkB,EAAE,GAAW,EAAE,aAAsB,IAAI;IACpH,MAAM,eAAe,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,0BAA0B,CAAC,WAAW,EAAE,UAAU,EAAE,GAAG,CAAC,CAAC;IAC5G,MAAM,YAAY,GAAG,sBAAsB,WAAW,IAAI,UAAU,MAAM,CAAC;IAE3E,KAAK,UAAU,oBAAoB,CAAI,SAAiB,EAAE,QAAyD;QAC/G,MAAM,WAAW,GAAG,IAAI,WAAW,CAAC,GAAG,CAAC;YACpC,eAAe,EAAE,KAAK;YACtB,UAAU,EAAE,IAAI;YAChB,WAAW,EAAE,IAAI;YACjB,MAAM,EAAE,GAAG,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,KAAK,KAAK,CAAC,MAAM,CAAC,eAAe,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,EAAE;SACpI,EAAE,WAAW,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;QAEvC,WAAW,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE;YACtB,KAAK,EAAE,EAAE;YACT,IAAI,EAAE,GAAG,WAAW,IAAI,UAAU,KAAK,SAAS,GAAG;SACtD,CAAC,CAAC;QAEH,MAAM,oBAAoB,GAAG,SAAS,CAAC;YACnC,QAAQ,CAAC,EAAC,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAC;gBACjC,MAAM,aAAa,GAAG,CAAC,SAAS,GAAG,GAAG,CAAC,GAAG,CAAC,QAAQ,GAAG,KAAK,CAAC,CAAC;gBAE7D,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC;YAChE,CAAC;SACJ,CAAC,CAAC;QAEH,IAAI;YACA,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,oBAAoB,CAAC,CAAC;YAEjD,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAExB,OAAO,GAAG,CAAC;SACd;gBAAS;YACN,WAAW,CAAC,IAAI,EAAE,CAAC;SACtB;IACL,CAAC;IAED,IAAI,eAAe,IAAI,IAAI,EAAE;QACzB,IAAI;YACA,MAAM,oBAAoB,CAAC,cAAc,EAAE,KAAK,EAAE,oBAAoB,EAAE,EAAE;gBACtE,MAAM,oBAAoB,CAAC,KAAK,CAAC,eAAe,EAAE,iBAAiB,EAAE;oBACjE,SAAS,EAAE,IAAI;iBAClB,CAAC,CAAC;gBAEH,MAAM,SAAS,CAAC,iBAAiB,CAAC,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;YAC9D,CAAC,CAAC,CAAC;YACH,OAAO;SACV;QAAC,OAAO,GAAG,EAAE;YACV,MAAM,EAAE,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC;YACnC,MAAM,EAAE,CAAC,MAAM,CAAC,4BAA4B,CAAC,CAAC;YAC9C,OAAO,CAAC,KAAK,CAAC,yDAAyD,EAAE,GAAG,CAAC,CAAC;YAE9E,mBAAmB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;SACpC;KACJ;IAED,IAAI;QACA,MAAM,oBAAoB,CAAC,QAAQ,EAAE,KAAK,EAAE,oBAAoB,EAAE,EAAE;YAChE,MAAM,oBAAoB,CAAC,KAAK,CAAC,YAAY,EAAE,iBAAiB,EAAE;gBAC9D,SAAS,EAAE,CAAC;gBACZ,UAAU,EAAE,GAAG;gBACf,SAAS,EAAE,IAAI;aAClB,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;KACN;IAAC,OAAO,GAAG,EAAE;QACV,mBAAmB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;QAEjC,MAAM,GAAG,CAAC;KACb;IAED,IAAI;QACA,MAAM,yBAAyB,GAA8B;YACzD,GAAG;SACN,CAAC;QAEF,MAAM,EAAE,CAAC,SAAS,CAAC,4BAA4B,EAAE,yBAAyB,EAAE;YACxE,MAAM,EAAE,CAAC;SACZ,CAAC,CAAC;KACN;IAAC,OAAO,GAAG,EAAE;QACV,OAAO,CAAC,KAAK,CAAC,oCAAoC,EAAE,GAAG,CAAC,CAAC;QAEzD,MAAM,GAAG,CAAC;KACb;AACL,CAAC;AAED,SAAS,mBAAmB,CAAC,KAAa;IACtC,iFAAiF;IACjF,wDAAwD;IACxD,6EAA6E;IAC7E,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,0CAA0C,CAAC;QACxE,OAAO,CAAC,IAAI,CAAC,IAAI;YACb,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,0FAA0F,CAAC,GAAG,IAAI;YAChJ,4CAA4C,GAAG,iBAAiB,GAAG,OAAO;YAC1E,KAAK,CAAC,MAAM,CAAC,2CAA2C,CAAC,GAAG,IAAI;YAChE,8CAA8C,CACjD,CAAC;AACV,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,+BAA+B;IACjD,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,UAAU,CAAC,4BAA4B,CAAC,CAAC;QACpD,OAAO,IAAI,CAAC;IAEhB,IAAI;QACA,MAAM,yBAAyB,GAA8B,MAAM,EAAE,CAAC,QAAQ,CAAC,4BAA4B,CAAC,CAAC;QAE7G,OAAO,yBAAyB,CAAC,GAAG,CAAC;KACxC;IAAC,OAAO,GAAG,EAAE;QACV,OAAO,CAAC,KAAK,CAAC,mCAAmC,EAAE,GAAG,CAAC,CAAC;QACxD,OAAO,IAAI,CAAC;KACf;AACL,CAAC"}
package/dist/utils/getBuildDefaults.d.ts ADDED
@@ -0,0 +1,6 @@
1
+ export declare function getBuildDefaults(): Promise<{
2
+ repo: string;
3
+ release: string;
4
+ metalSupport: boolean;
5
+ cudaSupport: boolean;
6
+ }>;
package/dist/utils/getBuildDefaults.js ADDED
@@ -0,0 +1,10 @@
1
+ import { defaultLlamaCppCudaSupport, defaultLlamaCppGitHubRepo, defaultLlamaCppMetalSupport, defaultLlamaCppRelease } from "../config.js";
2
+ export async function getBuildDefaults() {
3
+ return {
4
+ repo: defaultLlamaCppGitHubRepo,
5
+ release: defaultLlamaCppRelease,
6
+ metalSupport: defaultLlamaCppMetalSupport,
7
+ cudaSupport: defaultLlamaCppCudaSupport
8
+ };
9
+ }
10
+ //# sourceMappingURL=getBuildDefaults.js.map
package/dist/utils/getBuildDefaults.js.map ADDED
@@ -0,0 +1 @@
1
+ {"version":3,"file":"getBuildDefaults.js","sourceRoot":"","sources":["../../src/utils/getBuildDefaults.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,0BAA0B,EAAE,yBAAyB,EAAE,2BAA2B,EAAE,sBAAsB,EAC7G,MAAM,cAAc,CAAC;AAEtB,MAAM,CAAC,KAAK,UAAU,gBAAgB;IAClC,OAAO;QACH,IAAI,EAAE,yBAAyB;QAC/B,OAAO,EAAE,sBAAsB;QAC/B,YAAY,EAAE,2BAA2B;QACzC,WAAW,EAAE,0BAA0B;KAC1C,CAAC;AACN,CAAC"}
package/dist/utils/getReleaseInfo.d.ts ADDED
@@ -0,0 +1,7 @@
1
+ export declare function getReleaseInfo(): Promise<{
2
+ llamaCpp: {
3
+ binarySource: string;
4
+ release: string;
5
+ };
6
+ moduleVersion: string;
7
+ }>;
package/dist/utils/getReleaseInfo.js ADDED
@@ -0,0 +1,30 @@
1
+ import path from "path";
2
+ import { fileURLToPath } from "url";
3
+ import fs from "fs-extra";
4
+ import { getUsedBinFlag } from "./usedBinFlag.js";
5
+ import { getClonedLlamaCppRepoReleaseTag } from "./cloneLlamaCppRepo.js";
6
+ import { getBinariesGithubRelease } from "./binariesGithubRelease.js";
7
+ const __dirname = path.dirname(fileURLToPath(import.meta.url));
8
+ export async function getReleaseInfo() {
9
+ const [usedBinFlag, moduleVersion] = await Promise.all([
10
+ getUsedBinFlag(),
11
+ getModuleVersion()
12
+ ]);
13
+ const release = usedBinFlag === "prebuiltBinaries"
14
+ ? await getBinariesGithubRelease()
15
+ : (await getClonedLlamaCppRepoReleaseTag() ?? await getBinariesGithubRelease());
16
+ return {
17
+ llamaCpp: {
18
+ binarySource: usedBinFlag === "prebuiltBinaries"
19
+ ? "included"
20
+ : "builtLocally",
21
+ release
22
+ },
23
+ moduleVersion
24
+ };
25
+ }
26
+ async function getModuleVersion() {
27
+ const packageJson = await fs.readJson(path.join(__dirname, "..", "..", "package.json"));
28
+ return packageJson.version;
29
+ }
30
+ //# sourceMappingURL=getReleaseInfo.js.map
package/dist/utils/getReleaseInfo.js.map ADDED
@@ -0,0 +1 @@
1
+ {"version":3,"file":"getReleaseInfo.js","sourceRoot":"","sources":["../../src/utils/getReleaseInfo.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAC,aAAa,EAAC,MAAM,KAAK,CAAC;AAClC,OAAO,EAAE,MAAM,UAAU,CAAC;AAC1B,OAAO,EAAC,cAAc,EAAC,MAAM,kBAAkB,CAAC;AAChD,OAAO,EAAC,+BAA+B,EAAC,MAAM,wBAAwB,CAAC;AACvE,OAAO,EAAC,wBAAwB,EAAC,MAAM,4BAA4B,CAAC;AAEpE,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAE/D,MAAM,CAAC,KAAK,UAAU,cAAc;IAChC,MAAM,CAAC,WAAW,EAAE,aAAa,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QACnD,cAAc,EAAE;QAChB,gBAAgB,EAAE;KACrB,CAAC,CAAC;IAEH,MAAM,OAAO,GAAG,WAAW,KAAK,kBAAkB;QAC9C,CAAC,CAAC,MAAM,wBAAwB,EAAE;QAClC,CAAC,CAAC,CAAC,MAAM,+BAA+B,EAAE,IAAI,MAAM,wBAAwB,EAAE,CAAC,CAAC;IAEpF,OAAO;QACH,QAAQ,EAAE;YACN,YAAY,EAAE,WAAW,KAAK,kBAAkB;gBAC5C,CAAC,CAAC,UAAU;gBACZ,CAAC,CAAC,cAAc;YACpB,OAAO;SACV;QACD,aAAa;KAChB,CAAC;AACN,CAAC;AAED,KAAK,UAAU,gBAAgB;IAC3B,MAAM,WAAW,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC;IAExF,OAAO,WAAW,CAAC,OAAO,CAAC;AAC/B,CAAC"}
package/llama/binariesGithubRelease.json CHANGED
@@ -1,3 +1,3 @@
1
1
  {
2
- "release": "b1428"
2
+ "release": "b1492"
3
3
  }
package/llama/gitRelease.bundle CHANGED
Binary file
package/llamaBins/mac-arm64/ggml-metal.metal CHANGED
@@ -184,36 +184,73 @@ kernel void kernel_soft_max(
184
184
  constant int64_t & ne00,
185
185
  constant int64_t & ne01,
186
186
  constant int64_t & ne02,
187
- uint3 tgpig[[threadgroup_position_in_grid]],
188
- uint3 tpitg[[thread_position_in_threadgroup]],
189
- uint3 ntg[[threads_per_threadgroup]]) {
190
- const int64_t i03 = tgpig[2];
191
- const int64_t i02 = tgpig[1];
192
- const int64_t i01 = tgpig[0];
187
+ threadgroup float * buf [[threadgroup(0)]],
188
+ uint tgpig[[threadgroup_position_in_grid]],
189
+ uint tpitg[[thread_position_in_threadgroup]],
190
+ uint sgitg[[simdgroup_index_in_threadgroup]],
191
+ uint tiisg[[thread_index_in_simdgroup]],
192
+ uint ntg[[threads_per_threadgroup]]) {
193
+ const int64_t i03 = (tgpig) / (ne02*ne01);
194
+ const int64_t i02 = (tgpig - i03*ne02*ne01) / ne01;
195
+ const int64_t i01 = (tgpig - i03*ne02*ne01 - i02*ne01);
193
196
 
194
197
  device const float * psrc0 = src0 + i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;
195
198
  device float * pdst = dst + i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;
196
199
 
197
200
  // parallel max
198
- float lmax = tpitg[0] < ne00 ? psrc0[tpitg[0]] : -INFINITY;
199
- for (int i00 = tpitg[0] + ntg[0]; i00 < ne00; i00 += ntg[0]) {
201
+ float lmax = tpitg < ne00 ? psrc0[tpitg] : -INFINITY;
202
+
203
+ for (int i00 = tpitg + ntg; i00 < ne00; i00 += ntg) {
200
204
  lmax = MAX(lmax, psrc0[i00]);
201
205
  }
202
- const float max = simd_max(lmax);
206
+
207
+ float max = simd_max(lmax);
208
+ if (tiisg == 0) {
209
+ buf[sgitg] = max;
210
+ }
211
+
212
+ threadgroup_barrier(mem_flags::mem_threadgroup);
213
+
214
+ // broadcast, simd group number is ntg / 32
215
+ for (uint i = ntg / 32 / 2; i > 0; i /= 2) {
216
+ if (tpitg < i) {
217
+ buf[tpitg] = MAX(buf[tpitg], buf[tpitg + i]);
218
+ }
219
+ }
220
+
221
+ threadgroup_barrier(mem_flags::mem_threadgroup);
222
+
223
+ max = buf[0];
203
224
 
204
225
  // parallel sum
205
226
  float lsum = 0.0f;
206
- for (int i00 = tpitg[0]; i00 < ne00; i00 += ntg[0]) {
227
+ for (int i00 = tpitg; i00 < ne00; i00 += ntg) {
207
228
  const float exp_psrc0 = exp(psrc0[i00] - max);
208
229
  lsum += exp_psrc0;
209
230
  // Remember the result of exp here. exp is expensive, so we really do not
210
- // whish to compute it twice.
231
+ // wish to compute it twice.
211
232
  pdst[i00] = exp_psrc0;
212
233
  }
213
234
 
214
- const float sum = simd_sum(lsum);
235
+ float sum = simd_sum(lsum);
236
+ if (tiisg == 0) {
237
+ buf[sgitg] = sum;
238
+ }
239
+
240
+ threadgroup_barrier(mem_flags::mem_threadgroup);
215
241
 
216
- for (int i00 = tpitg[0]; i00 < ne00; i00 += ntg[0]) {
242
+ // broadcast, simd group number is ntg / 32
243
+ for (uint i = ntg / 32 / 2; i > 0; i /= 2) {
244
+ if (tpitg < i) {
245
+ buf[tpitg] += buf[tpitg + i];
246
+ }
247
+ }
248
+
249
+ threadgroup_barrier(mem_flags::mem_threadgroup);
250
+
251
+ sum = buf[0];
252
+
253
+ for (int i00 = tpitg; i00 < ne00; i00 += ntg) {
217
254
  pdst[i00] /= sum;
218
255
  }
219
256
  }
@@ -224,37 +261,73 @@ kernel void kernel_soft_max_4(
224
261
  constant int64_t & ne00,
225
262
  constant int64_t & ne01,
226
263
  constant int64_t & ne02,
227
- uint3 tgpig[[threadgroup_position_in_grid]],
228
- uint3 tpitg[[thread_position_in_threadgroup]],
229
- uint3 ntg[[threads_per_threadgroup]]) {
230
- const int64_t i03 = tgpig[2];
231
- const int64_t i02 = tgpig[1];
232
- const int64_t i01 = tgpig[0];
264
+ threadgroup float * buf [[threadgroup(0)]],
265
+ uint tgpig[[threadgroup_position_in_grid]],
266
+ uint tpitg[[thread_position_in_threadgroup]],
267
+ uint sgitg[[simdgroup_index_in_threadgroup]],
268
+ uint tiisg[[thread_index_in_simdgroup]],
269
+ uint ntg[[threads_per_threadgroup]]) {
270
+ const int64_t i03 = (tgpig) / (ne02*ne01);
271
+ const int64_t i02 = (tgpig - i03*ne02*ne01) / ne01;
272
+ const int64_t i01 = (tgpig - i03*ne02*ne01 - i02*ne01);
233
273
 
234
274
  device const float4 * psrc4 = (device const float4 *)(src0 + i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00);
235
275
  device float4 * pdst4 = (device float4 *)(dst + i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00);
236
276
 
237
277
  // parallel max
238
- float4 lmax4 = tpitg[0] < ne00/4 ? psrc4[tpitg[0]] : -INFINITY;
239
- for (int i00 = tpitg[0] + ntg[0]; i00 < ne00/4; i00 += ntg[0]) {
278
+ float4 lmax4 = tpitg < ne00/4 ? psrc4[tpitg] : -INFINITY;
279
+
280
+ for (int i00 = tpitg + ntg; i00 < ne00/4; i00 += ntg) {
240
281
  lmax4 = fmax(lmax4, psrc4[i00]);
241
282
  }
242
- float lmax = MAX(MAX(lmax4[0], lmax4[1]), MAX(lmax4[2], lmax4[3]));
243
283
 
244
- const float max = simd_max(lmax);
284
+ const float lmax = MAX(MAX(lmax4[0], lmax4[1]), MAX(lmax4[2], lmax4[3]));
285
+ float max = simd_max(lmax);
286
+ if (tiisg == 0) {
287
+ buf[sgitg] = max;
288
+ }
289
+
290
+ threadgroup_barrier(mem_flags::mem_threadgroup);
291
+
292
+ // broadcast, simd group number is ntg / 32
293
+ for (uint i = ntg / 32 / 2; i > 0; i /= 2) {
294
+ if (tpitg < i) {
295
+ buf[tpitg] = MAX(buf[tpitg], buf[tpitg + i]);
296
+ }
297
+ }
298
+
299
+ threadgroup_barrier(mem_flags::mem_threadgroup);
300
+
301
+ max = buf[0];
245
302
 
246
303
  // parallel sum
247
304
  float4 lsum4 = 0.0f;
248
- for (int i00 = tpitg[0]; i00 < ne00/4; i00 += ntg[0]) {
305
+ for (int i00 = tpitg; i00 < ne00/4; i00 += ntg) {
249
306
  const float4 exp_psrc4 = exp(psrc4[i00] - max);
250
307
  lsum4 += exp_psrc4;
251
308
  pdst4[i00] = exp_psrc4;
252
309
  }
253
- float lsum = lsum4[0] + lsum4[1] + lsum4[2] + lsum4[3];
254
310
 
255
- const float sum = simd_sum(lsum);
311
+ const float lsum = lsum4[0] + lsum4[1] + lsum4[2] + lsum4[3];
312
+ float sum = simd_sum(lsum);
313
+ if (tiisg == 0) {
314
+ buf[sgitg] = sum;
315
+ }
316
+
317
+ threadgroup_barrier(mem_flags::mem_threadgroup);
318
+
319
+ // broadcast, simd group number is ntg / 32
320
+ for (uint i = ntg / 32 / 2; i > 0; i /= 2) {
321
+ if (tpitg < i) {
322
+ buf[tpitg] += buf[tpitg + i];
323
+ }
324
+ }
325
+
326
+ threadgroup_barrier(mem_flags::mem_threadgroup);
327
+
328
+ sum = buf[0];
256
329
 
257
- for (int i00 = tpitg[0]; i00 < ne00/4; i00 += ntg[0]) {
330
+ for (int i00 = tpitg; i00 < ne00/4; i00 += ntg) {
258
331
  pdst4[i00] /= sum;
259
332
  }
260
333
  }
@@ -274,7 +347,7 @@ kernel void kernel_diag_mask_inf(
274
347
  dst[i02*ne01*ne00 + i01*ne00 + i00] = -INFINITY;
275
348
  } else {
276
349
  dst[i02*ne01*ne00 + i01*ne00 + i00] = src0[i02*ne01*ne00 + i01*ne00 + i00];
277
- }
350
+ }
278
351
  }
279
352
 
280
353
  kernel void kernel_diag_mask_inf_8(
@@ -988,6 +1061,45 @@ kernel void kernel_alibi_f32(
988
1061
  }
989
1062
  }
990
1063
 
1064
+ static float rope_yarn_ramp(const float low, const float high, const int i0) {
1065
+ const float y = (i0 / 2 - low) / max(0.001f, high - low);
1066
+ return 1.0f - min(1.0f, max(0.0f, y));
1067
+ }
1068
+
1069
+ // YaRN algorithm based on LlamaYaRNScaledRotaryEmbedding.py from https://github.com/jquesnelle/yarn
1070
+ // MIT licensed. Copyright (c) 2023 Jeffrey Quesnelle and Bowen Peng.
1071
+ static void rope_yarn(
1072
+ float theta_extrap, float freq_scale, float corr_dims[2], int64_t i0, float ext_factor, float mscale,
1073
+ thread float * cos_theta, thread float * sin_theta
1074
+ ) {
1075
+ // Get n-d rotational scaling corrected for extrapolation
1076
+ float theta_interp = freq_scale * theta_extrap;
1077
+ float theta = theta_interp;
1078
+ if (ext_factor != 0.0f) {
1079
+ float ramp_mix = rope_yarn_ramp(corr_dims[0], corr_dims[1], i0) * ext_factor;
1080
+ theta = theta_interp * (1 - ramp_mix) + theta_extrap * ramp_mix;
1081
+
1082
+ // Get n-d magnitude scaling corrected for interpolation
1083
+ mscale *= 1.0f + 0.1f * log(1.0f / freq_scale);
1084
+ }
1085
+ *cos_theta = cos(theta) * mscale;
1086
+ *sin_theta = sin(theta) * mscale;
1087
+ }
1088
+
1089
+ // Apparently solving `n_rot = 2pi * x * base^((2 * max_pos_emb) / n_dims)` for x, we get
1090
+ // `corr_fac(n_rot) = n_dims * log(max_pos_emb / (n_rot * 2pi)) / (2 * log(base))`
1091
+ static float rope_yarn_corr_factor(int n_dims, int n_orig_ctx, float n_rot, float base) {
1092
+ return n_dims * log(n_orig_ctx / (n_rot * 2 * M_PI_F)) / (2 * log(base));
1093
+ }
1094
+
1095
+ static void rope_yarn_corr_dims(
1096
+ int n_dims, int n_orig_ctx, float freq_base, float beta_fast, float beta_slow, float dims[2]
1097
+ ) {
1098
+ // start and end correction dims
1099
+ dims[0] = max(0.0f, floor(rope_yarn_corr_factor(n_dims, n_orig_ctx, beta_fast, freq_base)));
1100
+ dims[1] = min(n_dims - 1.0f, ceil(rope_yarn_corr_factor(n_dims, n_orig_ctx, beta_slow, freq_base)));
1101
+ }
1102
+
991
1103
  typedef void (rope_t)(
992
1104
  device const void * src0,
993
1105
  device const int32_t * src1,
@@ -1011,8 +1123,13 @@ typedef void (rope_t)(
1011
1123
  constant int & n_past,
1012
1124
  constant int & n_dims,
1013
1125
  constant int & mode,
1126
+ constant int & n_orig_ctx,
1014
1127
  constant float & freq_base,
1015
1128
  constant float & freq_scale,
1129
+ constant float & ext_factor,
1130
+ constant float & attn_factor,
1131
+ constant float & beta_fast,
1132
+ constant float & beta_slow,
1016
1133
  uint tiitg[[thread_index_in_threadgroup]],
1017
1134
  uint3 tptg[[threads_per_threadgroup]],
1018
1135
  uint3 tgpig[[threadgroup_position_in_grid]]);
@@ -1041,8 +1158,13 @@ kernel void kernel_rope(
1041
1158
  constant int & n_past,
1042
1159
  constant int & n_dims,
1043
1160
  constant int & mode,
1161
+ constant int & n_orig_ctx,
1044
1162
  constant float & freq_base,
1045
1163
  constant float & freq_scale,
1164
+ constant float & ext_factor,
1165
+ constant float & attn_factor,
1166
+ constant float & beta_fast,
1167
+ constant float & beta_slow,
1046
1168
  uint tiitg[[thread_index_in_threadgroup]],
1047
1169
  uint3 tptg[[threads_per_threadgroup]],
1048
1170
  uint3 tgpig[[threadgroup_position_in_grid]]) {
@@ -1052,19 +1174,22 @@ kernel void kernel_rope(
1052
1174
 
1053
1175
  const bool is_neox = mode & 2;
1054
1176
 
1177
+ float corr_dims[2];
1178
+ rope_yarn_corr_dims(n_dims, n_orig_ctx, freq_base, beta_fast, beta_slow, corr_dims);
1179
+
1055
1180
  device const int32_t * pos = src1;
1056
1181
 
1057
1182
  const int64_t p = pos[i2];
1058
1183
 
1059
- const float theta_0 = freq_scale * (float)p;
1184
+ const float theta_0 = (float)p;
1060
1185
  const float inv_ndims = -1.f/n_dims;
1061
1186
 
1062
1187
  if (!is_neox) {
1063
1188
  for (int64_t i0 = 2*tiitg; i0 < ne0; i0 += 2*tptg.x) {
1064
1189
 
1065
1190
  const float theta = theta_0 * pow(freq_base, inv_ndims*i0);
1066
- const float cos_theta = cos(theta);
1067
- const float sin_theta = sin(theta);
1191
+ float cos_theta, sin_theta;
1192
+ rope_yarn(theta, freq_scale, corr_dims, i0, ext_factor, attn_factor, &cos_theta, &sin_theta);
1068
1193
 
1069
1194
  device const T * const src = (device T *)((device char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);
1070
1195
  device T * dst_data = (device T *)((device char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
@@ -1079,9 +1204,12 @@ kernel void kernel_rope(
1079
1204
  for (int64_t ib = 0; ib < ne0/n_dims; ++ib) {
1080
1205
  for (int64_t ic = 2*tiitg; ic < n_dims; ic += 2*tptg.x) {
1081
1206
 
1082
- const float theta = theta_0 * pow(freq_base, inv_ndims*ic - ib);
1083
- const float cos_theta = cos(theta);
1084
- const float sin_theta = sin(theta);
1207
+ // simplified from `(ib * n_dims + ic) * inv_ndims`
1208
+ const float cur_rot = inv_ndims*ic - ib;
1209
+
1210
+ const float theta = theta_0 * pow(freq_base, cur_rot);
1211
+ float cos_theta, sin_theta;
1212
+ rope_yarn(theta, freq_scale, corr_dims, cur_rot, ext_factor, attn_factor, &cos_theta, &sin_theta);
1085
1213
 
1086
1214
  const int64_t i0 = ib*n_dims + ic/2;
1087
1215
 
package/llamaBins/mac-x64/ggml-metal.metal CHANGED
@@ -184,36 +184,73 @@ kernel void kernel_soft_max(
184
184
  constant int64_t & ne00,
185
185
  constant int64_t & ne01,
186
186
  constant int64_t & ne02,
187
- uint3 tgpig[[threadgroup_position_in_grid]],
188
- uint3 tpitg[[thread_position_in_threadgroup]],
189
- uint3 ntg[[threads_per_threadgroup]]) {
190
- const int64_t i03 = tgpig[2];
191
- const int64_t i02 = tgpig[1];
192
- const int64_t i01 = tgpig[0];
187
+ threadgroup float * buf [[threadgroup(0)]],
188
+ uint tgpig[[threadgroup_position_in_grid]],
189
+ uint tpitg[[thread_position_in_threadgroup]],
190
+ uint sgitg[[simdgroup_index_in_threadgroup]],
191
+ uint tiisg[[thread_index_in_simdgroup]],
192
+ uint ntg[[threads_per_threadgroup]]) {
193
+ const int64_t i03 = (tgpig) / (ne02*ne01);
194
+ const int64_t i02 = (tgpig - i03*ne02*ne01) / ne01;
195
+ const int64_t i01 = (tgpig - i03*ne02*ne01 - i02*ne01);
193
196
 
194
197
  device const float * psrc0 = src0 + i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;
195
198
  device float * pdst = dst + i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00;
196
199
 
197
200
  // parallel max
198
- float lmax = tpitg[0] < ne00 ? psrc0[tpitg[0]] : -INFINITY;
199
- for (int i00 = tpitg[0] + ntg[0]; i00 < ne00; i00 += ntg[0]) {
201
+ float lmax = tpitg < ne00 ? psrc0[tpitg] : -INFINITY;
202
+
203
+ for (int i00 = tpitg + ntg; i00 < ne00; i00 += ntg) {
200
204
  lmax = MAX(lmax, psrc0[i00]);
201
205
  }
202
- const float max = simd_max(lmax);
206
+
207
+ float max = simd_max(lmax);
208
+ if (tiisg == 0) {
209
+ buf[sgitg] = max;
210
+ }
211
+
212
+ threadgroup_barrier(mem_flags::mem_threadgroup);
213
+
214
+ // broadcast, simd group number is ntg / 32
215
+ for (uint i = ntg / 32 / 2; i > 0; i /= 2) {
216
+ if (tpitg < i) {
217
+ buf[tpitg] = MAX(buf[tpitg], buf[tpitg + i]);
218
+ }
219
+ }
220
+
221
+ threadgroup_barrier(mem_flags::mem_threadgroup);
222
+
223
+ max = buf[0];
203
224
 
204
225
  // parallel sum
205
226
  float lsum = 0.0f;
206
- for (int i00 = tpitg[0]; i00 < ne00; i00 += ntg[0]) {
227
+ for (int i00 = tpitg; i00 < ne00; i00 += ntg) {
207
228
  const float exp_psrc0 = exp(psrc0[i00] - max);
208
229
  lsum += exp_psrc0;
209
230
  // Remember the result of exp here. exp is expensive, so we really do not
210
- // whish to compute it twice.
231
+ // wish to compute it twice.
211
232
  pdst[i00] = exp_psrc0;
212
233
  }
213
234
 
214
- const float sum = simd_sum(lsum);
235
+ float sum = simd_sum(lsum);
236
+ if (tiisg == 0) {
237
+ buf[sgitg] = sum;
238
+ }
239
+
240
+ threadgroup_barrier(mem_flags::mem_threadgroup);
215
241
 
216
- for (int i00 = tpitg[0]; i00 < ne00; i00 += ntg[0]) {
242
+ // broadcast, simd group number is ntg / 32
243
+ for (uint i = ntg / 32 / 2; i > 0; i /= 2) {
244
+ if (tpitg < i) {
245
+ buf[tpitg] += buf[tpitg + i];
246
+ }
247
+ }
248
+
249
+ threadgroup_barrier(mem_flags::mem_threadgroup);
250
+
251
+ sum = buf[0];
252
+
253
+ for (int i00 = tpitg; i00 < ne00; i00 += ntg) {
217
254
  pdst[i00] /= sum;
218
255
  }
219
256
  }
@@ -224,37 +261,73 @@ kernel void kernel_soft_max_4(
224
261
  constant int64_t & ne00,
225
262
  constant int64_t & ne01,
226
263
  constant int64_t & ne02,
227
- uint3 tgpig[[threadgroup_position_in_grid]],
228
- uint3 tpitg[[thread_position_in_threadgroup]],
229
- uint3 ntg[[threads_per_threadgroup]]) {
230
- const int64_t i03 = tgpig[2];
231
- const int64_t i02 = tgpig[1];
232
- const int64_t i01 = tgpig[0];
264
+ threadgroup float * buf [[threadgroup(0)]],
265
+ uint tgpig[[threadgroup_position_in_grid]],
266
+ uint tpitg[[thread_position_in_threadgroup]],
267
+ uint sgitg[[simdgroup_index_in_threadgroup]],
268
+ uint tiisg[[thread_index_in_simdgroup]],
269
+ uint ntg[[threads_per_threadgroup]]) {
270
+ const int64_t i03 = (tgpig) / (ne02*ne01);
271
+ const int64_t i02 = (tgpig - i03*ne02*ne01) / ne01;
272
+ const int64_t i01 = (tgpig - i03*ne02*ne01 - i02*ne01);
233
273
 
234
274
  device const float4 * psrc4 = (device const float4 *)(src0 + i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00);
235
275
  device float4 * pdst4 = (device float4 *)(dst + i03*ne02*ne01*ne00 + i02*ne01*ne00 + i01*ne00);
236
276
 
237
277
  // parallel max
238
- float4 lmax4 = tpitg[0] < ne00/4 ? psrc4[tpitg[0]] : -INFINITY;
239
- for (int i00 = tpitg[0] + ntg[0]; i00 < ne00/4; i00 += ntg[0]) {
278
+ float4 lmax4 = tpitg < ne00/4 ? psrc4[tpitg] : -INFINITY;
279
+
280
+ for (int i00 = tpitg + ntg; i00 < ne00/4; i00 += ntg) {
240
281
  lmax4 = fmax(lmax4, psrc4[i00]);
241
282
  }
242
- float lmax = MAX(MAX(lmax4[0], lmax4[1]), MAX(lmax4[2], lmax4[3]));
243
283
 
244
- const float max = simd_max(lmax);
284
+ const float lmax = MAX(MAX(lmax4[0], lmax4[1]), MAX(lmax4[2], lmax4[3]));
285
+ float max = simd_max(lmax);
286
+ if (tiisg == 0) {
287
+ buf[sgitg] = max;
288
+ }
289
+
290
+ threadgroup_barrier(mem_flags::mem_threadgroup);
291
+
292
+ // broadcast, simd group number is ntg / 32
293
+ for (uint i = ntg / 32 / 2; i > 0; i /= 2) {
294
+ if (tpitg < i) {
295
+ buf[tpitg] = MAX(buf[tpitg], buf[tpitg + i]);
296
+ }
297
+ }
298
+
299
+ threadgroup_barrier(mem_flags::mem_threadgroup);
300
+
301
+ max = buf[0];
245
302
 
246
303
  // parallel sum
247
304
  float4 lsum4 = 0.0f;
248
- for (int i00 = tpitg[0]; i00 < ne00/4; i00 += ntg[0]) {
305
+ for (int i00 = tpitg; i00 < ne00/4; i00 += ntg) {
249
306
  const float4 exp_psrc4 = exp(psrc4[i00] - max);
250
307
  lsum4 += exp_psrc4;
251
308
  pdst4[i00] = exp_psrc4;
252
309
  }
253
- float lsum = lsum4[0] + lsum4[1] + lsum4[2] + lsum4[3];
254
310
 
255
- const float sum = simd_sum(lsum);
311
+ const float lsum = lsum4[0] + lsum4[1] + lsum4[2] + lsum4[3];
312
+ float sum = simd_sum(lsum);
313
+ if (tiisg == 0) {
314
+ buf[sgitg] = sum;
315
+ }
316
+
317
+ threadgroup_barrier(mem_flags::mem_threadgroup);
318
+
319
+ // broadcast, simd group number is ntg / 32
320
+ for (uint i = ntg / 32 / 2; i > 0; i /= 2) {
321
+ if (tpitg < i) {
322
+ buf[tpitg] += buf[tpitg + i];
323
+ }
324
+ }
325
+
326
+ threadgroup_barrier(mem_flags::mem_threadgroup);
327
+
328
+ sum = buf[0];
256
329
 
257
- for (int i00 = tpitg[0]; i00 < ne00/4; i00 += ntg[0]) {
330
+ for (int i00 = tpitg; i00 < ne00/4; i00 += ntg) {
258
331
  pdst4[i00] /= sum;
259
332
  }
260
333
  }
@@ -274,7 +347,7 @@ kernel void kernel_diag_mask_inf(
274
347
  dst[i02*ne01*ne00 + i01*ne00 + i00] = -INFINITY;
275
348
  } else {
276
349
  dst[i02*ne01*ne00 + i01*ne00 + i00] = src0[i02*ne01*ne00 + i01*ne00 + i00];
277
- }
350
+ }
278
351
  }
279
352
 
280
353
  kernel void kernel_diag_mask_inf_8(
@@ -988,6 +1061,45 @@ kernel void kernel_alibi_f32(
988
1061
  }
989
1062
  }
990
1063
 
1064
+ static float rope_yarn_ramp(const float low, const float high, const int i0) {
1065
+ const float y = (i0 / 2 - low) / max(0.001f, high - low);
1066
+ return 1.0f - min(1.0f, max(0.0f, y));
1067
+ }
1068
+
1069
+ // YaRN algorithm based on LlamaYaRNScaledRotaryEmbedding.py from https://github.com/jquesnelle/yarn
1070
+ // MIT licensed. Copyright (c) 2023 Jeffrey Quesnelle and Bowen Peng.
1071
+ static void rope_yarn(
1072
+ float theta_extrap, float freq_scale, float corr_dims[2], int64_t i0, float ext_factor, float mscale,
1073
+ thread float * cos_theta, thread float * sin_theta
1074
+ ) {
1075
+ // Get n-d rotational scaling corrected for extrapolation
1076
+ float theta_interp = freq_scale * theta_extrap;
1077
+ float theta = theta_interp;
1078
+ if (ext_factor != 0.0f) {
1079
+ float ramp_mix = rope_yarn_ramp(corr_dims[0], corr_dims[1], i0) * ext_factor;
1080
+ theta = theta_interp * (1 - ramp_mix) + theta_extrap * ramp_mix;
1081
+
1082
+ // Get n-d magnitude scaling corrected for interpolation
1083
+ mscale *= 1.0f + 0.1f * log(1.0f / freq_scale);
1084
+ }
1085
+ *cos_theta = cos(theta) * mscale;
1086
+ *sin_theta = sin(theta) * mscale;
1087
+ }
1088
+
1089
+ // Apparently solving `n_rot = 2pi * x * base^((2 * max_pos_emb) / n_dims)` for x, we get
1090
+ // `corr_fac(n_rot) = n_dims * log(max_pos_emb / (n_rot * 2pi)) / (2 * log(base))`
1091
+ static float rope_yarn_corr_factor(int n_dims, int n_orig_ctx, float n_rot, float base) {
1092
+ return n_dims * log(n_orig_ctx / (n_rot * 2 * M_PI_F)) / (2 * log(base));
1093
+ }
1094
+
1095
+ static void rope_yarn_corr_dims(
1096
+ int n_dims, int n_orig_ctx, float freq_base, float beta_fast, float beta_slow, float dims[2]
1097
+ ) {
1098
+ // start and end correction dims
1099
+ dims[0] = max(0.0f, floor(rope_yarn_corr_factor(n_dims, n_orig_ctx, beta_fast, freq_base)));
1100
+ dims[1] = min(n_dims - 1.0f, ceil(rope_yarn_corr_factor(n_dims, n_orig_ctx, beta_slow, freq_base)));
1101
+ }
1102
+
991
1103
  typedef void (rope_t)(
992
1104
  device const void * src0,
993
1105
  device const int32_t * src1,
@@ -1011,8 +1123,13 @@ typedef void (rope_t)(
1011
1123
  constant int & n_past,
1012
1124
  constant int & n_dims,
1013
1125
  constant int & mode,
1126
+ constant int & n_orig_ctx,
1014
1127
  constant float & freq_base,
1015
1128
  constant float & freq_scale,
1129
+ constant float & ext_factor,
1130
+ constant float & attn_factor,
1131
+ constant float & beta_fast,
1132
+ constant float & beta_slow,
1016
1133
  uint tiitg[[thread_index_in_threadgroup]],
1017
1134
  uint3 tptg[[threads_per_threadgroup]],
1018
1135
  uint3 tgpig[[threadgroup_position_in_grid]]);
@@ -1041,8 +1158,13 @@ kernel void kernel_rope(
1041
1158
  constant int & n_past,
1042
1159
  constant int & n_dims,
1043
1160
  constant int & mode,
1161
+ constant int & n_orig_ctx,
1044
1162
  constant float & freq_base,
1045
1163
  constant float & freq_scale,
1164
+ constant float & ext_factor,
1165
+ constant float & attn_factor,
1166
+ constant float & beta_fast,
1167
+ constant float & beta_slow,
1046
1168
  uint tiitg[[thread_index_in_threadgroup]],
1047
1169
  uint3 tptg[[threads_per_threadgroup]],
1048
1170
  uint3 tgpig[[threadgroup_position_in_grid]]) {
@@ -1052,19 +1174,22 @@ kernel void kernel_rope(
1052
1174
 
1053
1175
  const bool is_neox = mode & 2;
1054
1176
 
1177
+ float corr_dims[2];
1178
+ rope_yarn_corr_dims(n_dims, n_orig_ctx, freq_base, beta_fast, beta_slow, corr_dims);
1179
+
1055
1180
  device const int32_t * pos = src1;
1056
1181
 
1057
1182
  const int64_t p = pos[i2];
1058
1183
 
1059
- const float theta_0 = freq_scale * (float)p;
1184
+ const float theta_0 = (float)p;
1060
1185
  const float inv_ndims = -1.f/n_dims;
1061
1186
 
1062
1187
  if (!is_neox) {
1063
1188
  for (int64_t i0 = 2*tiitg; i0 < ne0; i0 += 2*tptg.x) {
1064
1189
 
1065
1190
  const float theta = theta_0 * pow(freq_base, inv_ndims*i0);
1066
- const float cos_theta = cos(theta);
1067
- const float sin_theta = sin(theta);
1191
+ float cos_theta, sin_theta;
1192
+ rope_yarn(theta, freq_scale, corr_dims, i0, ext_factor, attn_factor, &cos_theta, &sin_theta);
1068
1193
 
1069
1194
  device const T * const src = (device T *)((device char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);
1070
1195
  device T * dst_data = (device T *)((device char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
@@ -1079,9 +1204,12 @@ kernel void kernel_rope(
1079
1204
  for (int64_t ib = 0; ib < ne0/n_dims; ++ib) {
1080
1205
  for (int64_t ic = 2*tiitg; ic < n_dims; ic += 2*tptg.x) {
1081
1206
 
1082
- const float theta = theta_0 * pow(freq_base, inv_ndims*ic - ib);
1083
- const float cos_theta = cos(theta);
1084
- const float sin_theta = sin(theta);
1207
+ // simplified from `(ib * n_dims + ic) * inv_ndims`
1208
+ const float cur_rot = inv_ndims*ic - ib;
1209
+
1210
+ const float theta = theta_0 * pow(freq_base, cur_rot);
1211
+ float cos_theta, sin_theta;
1212
+ rope_yarn(theta, freq_scale, corr_dims, cur_rot, ext_factor, attn_factor, &cos_theta, &sin_theta);
1085
1213
 
1086
1214
  const int64_t i0 = ib*n_dims + ic/2;
1087
1215
 
package/llamaBins/mac-x64/llama-addon.node CHANGED
Binary file
package/llamaBins/win-x64/llama-addon.exp CHANGED
Binary file
package/llamaBins/win-x64/llama-addon.lib CHANGED
Binary file
package/llamaBins/win-x64/llama-addon.node CHANGED
Binary file
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "node-llama-cpp",
3
- "version": "2.7.4",
3
+ "version": "2.8.0",
4
4
  "description": "Run AI models locally on your machine with node.js bindings for llama.cpp. Force a JSON schema on the model output on the generation level",
5
5
  "main": "dist/index.js",
6
6
  "type": "module",
@@ -70,6 +70,7 @@
70
70
  "keywords": [
71
71
  "llama",
72
72
  "llama-cpp",
73
+ "llama.cpp",
73
74
  "bindings",
74
75
  "ai",
75
76
  "cmake",