@huggingface/tasks 0.21.58 → 0.21.59
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commonjs/local-apps.d.ts.map +1 -1
- package/dist/commonjs/local-apps.js +31 -12
- package/dist/commonjs/local-apps.spec.js +24 -0
- package/dist/esm/local-apps.d.ts.map +1 -1
- package/dist/esm/local-apps.js +31 -12
- package/dist/esm/local-apps.spec.js +24 -0
- package/package.json +1 -1
- package/src/local-apps.spec.ts +28 -0
- package/src/local-apps.ts +35 -12
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"local-apps.d.ts","sourceRoot":"","sources":["../../src/local-apps.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAKnD,MAAM,WAAW,eAAe;IAC/B;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;CAC3B;AAED;;GAEG;AACH,MAAM,MAAM,QAAQ,GAAG;IACtB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,KAAK,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,EAAE,GAAG,CAAC,CAAC,KAAK,EAAE,SAAS,KAAK;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC,CAAC;IACpG;;OAEG;IACH,QAAQ,EAAE,YAAY,CAAC;IACvB;;OAEG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB;;OAEG;IACH,kBAAkB,EAAE,CAAC,KAAK,EAAE,SAAS,KAAK,OAAO,CAAC;CAClD,GAAG,CACD;IACA;;OAEG;IACH,QAAQ,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,CAAC,EAAE,MAAM,KAAK,GAAG,CAAC;CACtD,GACD;IACA;;;;OAIG;IACH,OAAO,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,CAAC,EAAE,MAAM,KAAK,MAAM,GAAG,MAAM,EAAE,GAAG,eAAe,GAAG,eAAe,EAAE,CAAC;CACzG,CACH,CAAC;AAsBF,iBAAS,UAAU,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAE7C;AAED,iBAAS,mBAAmB,CAAC,KAAK,EAAE,SAAS,WAE5C;AAED,iBAAS,WAAW,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAU9C;AAED,iBAAS,wBAAwB,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAE3D;AA0BD,iBAAS,cAAc,CAAC,KAAK,EAAE,SAAS,WAEvC;
|
|
1
|
+
{"version":3,"file":"local-apps.d.ts","sourceRoot":"","sources":["../../src/local-apps.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAKnD,MAAM,WAAW,eAAe;IAC/B;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;CAC3B;AAED;;GAEG;AACH,MAAM,MAAM,QAAQ,GAAG;IACtB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,KAAK,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,EAAE,GAAG,CAAC,CAAC,KAAK,EAAE,SAAS,KAAK;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC,CAAC;IACpG;;OAEG;IACH,QAAQ,EAAE,YAAY,CAAC;IACvB;;OAEG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB;;OAEG;IACH,kBAAkB,EAAE,CAAC,KAAK,EAAE,SAAS,KAAK,OAAO,CAAC;CAClD,GAAG,CACD;IACA;;OAEG;IACH,QAAQ,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,CAAC,EAAE,MAAM,KAAK,GAAG,CAAC;CACtD,GACD;IACA;;;;OAIG;IACH,OAAO,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,CAAC,EAAE,MAAM,KAAK,MAAM,GAAG,MAAM,EAAE,GAAG,eAAe,GAAG,eAAe,EAAE,CAAC;CACzG,CACH,CAAC;AAsBF,iBAAS,UAAU,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAE7C;AAED,iBAAS,mBAAmB,CAAC,KAAK,EAAE,SAAS,WAE5C;AAED,iBAAS,WAAW,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAU9C;AAED,iBAAS,wBAAwB,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAE3D;AA0BD,iBAAS,cAAc,CAAC,KAAK,EAAE,SAAS,WAEvC;AAWD,iBAAS,4BAA4B,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAM/D;AAwbD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,UAAU;;;;;;yBAtbS,SAAS,aAAa,MAAM,KAAG,eAAe,EAAE;;;;;;;yBA+DzC,SAAS,aAAa,MAAM,KAAG,eAAe,EAAE;;;;;;;yBAyC3D,SAAS,KAAG,eAAe,EAAE;;;;;;oCAwW3B,SAAS;yBAlTT,SAAS,KAAG,eAAe,EAAE;;;;;;;yBAoF9B,SAAS,KAAG,eAAe,EAAE;;;;;;;yBA7B/B,SAAS,KAAG,eAAe,EAAE;;;;;;;;;;;;;;yBApIzB,SAAS,aAAa,MAAM,KAAG,eAAe,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;yBAJjD,SAAS,aAAa,MAAM,KAAG,MAAM;;;;;;;;;;;;;;yBAkT1B,SAAS,aAAa,MAAM,KAAG,MAAM;;;;;;;yBAM9C,SAAS,aAAa,MAAM,KAAG,eAAe,EAAE;;;;;;;yBAlGtD,SAAS,aAAa,MAAM,KAAG,eAAe,EAAE;;;;;;;yBAkCvC,SAAS,aAAa,MAAM,KAAG,eAAe,EAAE;;;;;;;yBA2BnD,SAAS,aAAa,MAAM,KAAG,eAAe,EAAE;;CAqS5C,CAAC;AAErC,MAAM,MAAM,WAAW,GAAG,MAAM,OAAO,UAAU,CAAC"}
|
|
@@ -60,6 +60,12 @@ function getChatTemplate(model) {
|
|
|
60
60
|
function isUnslothModel(model) {
|
|
61
61
|
return model.tags.includes("unsloth") || isLlamaCppGgufModel(model);
|
|
62
62
|
}
|
|
63
|
+
function isEmbeddingModel(model) {
|
|
64
|
+
return (!model.tags.includes("decision-model") &&
|
|
65
|
+
(model.tags.includes("feature-extraction") ||
|
|
66
|
+
model.pipeline_tag === "feature-extraction" ||
|
|
67
|
+
model.pipeline_tag === "sentence-similarity"));
|
|
68
|
+
}
|
|
63
69
|
function isToolCallingLocalAgentModel(model) {
|
|
64
70
|
return ((isLlamaCppGgufModel(model) || isMlxModel(model)) &&
|
|
65
71
|
model.tags.includes("conversational") &&
|
|
@@ -74,27 +80,35 @@ function getQuantTag(filepath) {
|
|
|
74
80
|
return quantLabel ? `:${quantLabel}` : defaultTag;
|
|
75
81
|
}
|
|
76
82
|
const snippetLlamacpp = (model, filepath) => {
|
|
83
|
+
const isEmbedding = isEmbeddingModel(model);
|
|
77
84
|
const serverCommand = (binary) => {
|
|
78
|
-
const snippet =
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
85
|
+
const snippet = isEmbedding
|
|
86
|
+
? [
|
|
87
|
+
"# Start a local OpenAI-compatible embeddings server:",
|
|
88
|
+
`${binary} -hf ${model.id}${getQuantTag(filepath)} --embeddings`,
|
|
89
|
+
]
|
|
90
|
+
: [
|
|
91
|
+
"# Start a local OpenAI-compatible server with a web UI:",
|
|
92
|
+
`${binary} -hf ${model.id}${getQuantTag(filepath)}`,
|
|
93
|
+
];
|
|
82
94
|
return snippet.join("\n");
|
|
83
95
|
};
|
|
84
96
|
const cliCommand = (binary) => {
|
|
85
97
|
const snippet = ["# Run inference directly in the terminal:", `${binary} -hf ${model.id}${getQuantTag(filepath)}`];
|
|
86
98
|
return snippet.join("\n");
|
|
87
99
|
};
|
|
100
|
+
// Embedding models don't generate text, so there is no CLI command for them
|
|
101
|
+
const commands = (serverBinary, cliBinary) => isEmbedding ? [serverCommand(serverBinary)] : [serverCommand(serverBinary), cliCommand(cliBinary)];
|
|
88
102
|
return [
|
|
89
103
|
{
|
|
90
104
|
title: "Install (macOS, Linux)",
|
|
91
105
|
setup: "curl -LsSf https://llama.app/install.sh | sh",
|
|
92
|
-
content:
|
|
106
|
+
content: commands("llama serve", "llama cli"),
|
|
93
107
|
},
|
|
94
108
|
{
|
|
95
109
|
title: "Install from WinGet (Windows)",
|
|
96
110
|
setup: "winget install llama.cpp",
|
|
97
|
-
content:
|
|
111
|
+
content: commands("llama serve", "llama cli"),
|
|
98
112
|
},
|
|
99
113
|
{
|
|
100
114
|
title: "Use pre-built binary",
|
|
@@ -103,7 +117,7 @@ const snippetLlamacpp = (model, filepath) => {
|
|
|
103
117
|
"# Download pre-built binary from:",
|
|
104
118
|
"# https://github.com/ggerganov/llama.cpp/releases",
|
|
105
119
|
].join("\n"),
|
|
106
|
-
content:
|
|
120
|
+
content: commands("./llama-server", "./llama-cli"),
|
|
107
121
|
},
|
|
108
122
|
{
|
|
109
123
|
title: "Build from source code",
|
|
@@ -113,12 +127,17 @@ const snippetLlamacpp = (model, filepath) => {
|
|
|
113
127
|
"cmake -B build",
|
|
114
128
|
"cmake --build build -j --target llama-server llama-cli",
|
|
115
129
|
].join("\n"),
|
|
116
|
-
content:
|
|
117
|
-
},
|
|
118
|
-
{
|
|
119
|
-
title: "Use Docker",
|
|
120
|
-
content: snippetDockerModelRunner(model, filepath),
|
|
130
|
+
content: commands("./build/bin/llama-server", "./build/bin/llama-cli"),
|
|
121
131
|
},
|
|
132
|
+
// `docker model run` starts a chat session, which embedding models don't support
|
|
133
|
+
...(isEmbedding
|
|
134
|
+
? []
|
|
135
|
+
: [
|
|
136
|
+
{
|
|
137
|
+
title: "Use Docker",
|
|
138
|
+
content: snippetDockerModelRunner(model, filepath),
|
|
139
|
+
},
|
|
140
|
+
]),
|
|
122
141
|
];
|
|
123
142
|
};
|
|
124
143
|
const snippetNodeLlamaCppCli = (model, filepath) => {
|
|
@@ -33,6 +33,30 @@ llama serve -hf mlabonne/gemma-2b-GGUF:{{QUANT_TAG}}`,
|
|
|
33
33
|
llama cli -hf mlabonne/gemma-2b-GGUF:{{QUANT_TAG}}`,
|
|
34
34
|
]);
|
|
35
35
|
});
|
|
36
|
+
(0, vitest_1.it)("llama.cpp embedding", async () => {
|
|
37
|
+
const { snippet: snippetFunc } = local_apps_js_1.LOCAL_APPS["llama.cpp"];
|
|
38
|
+
const model = {
|
|
39
|
+
id: "ggml-org/embeddinggemma-300M-GGUF",
|
|
40
|
+
tags: ["feature-extraction"],
|
|
41
|
+
inference: "",
|
|
42
|
+
};
|
|
43
|
+
const snippet = snippetFunc(model);
|
|
44
|
+
(0, vitest_1.expect)(snippet[0].content).toEqual([
|
|
45
|
+
`# Start a local OpenAI-compatible embeddings server:
|
|
46
|
+
llama serve -hf ggml-org/embeddinggemma-300M-GGUF:{{QUANT_TAG}} --embeddings`,
|
|
47
|
+
]);
|
|
48
|
+
(0, vitest_1.expect)(snippet.map((s) => s.title)).not.toContain("Use Docker");
|
|
49
|
+
});
|
|
50
|
+
(0, vitest_1.it)("llama.cpp decision model is not treated as an embedding model", async () => {
|
|
51
|
+
const { snippet: snippetFunc } = local_apps_js_1.LOCAL_APPS["llama.cpp"];
|
|
52
|
+
const model = {
|
|
53
|
+
id: "ggml-org/Julia-1-GGUF",
|
|
54
|
+
tags: ["feature-extraction", "decision-model"],
|
|
55
|
+
inference: "",
|
|
56
|
+
};
|
|
57
|
+
const snippet = snippetFunc(model);
|
|
58
|
+
(0, vitest_1.expect)(snippet[0].content[0]).not.toContain("--embeddings");
|
|
59
|
+
});
|
|
36
60
|
(0, vitest_1.it)("vLLM conversational llm", async () => {
|
|
37
61
|
const { snippet: snippetFunc } = local_apps_js_1.LOCAL_APPS["vllm"];
|
|
38
62
|
const model = {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"local-apps.d.ts","sourceRoot":"","sources":["../../src/local-apps.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAKnD,MAAM,WAAW,eAAe;IAC/B;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;CAC3B;AAED;;GAEG;AACH,MAAM,MAAM,QAAQ,GAAG;IACtB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,KAAK,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,EAAE,GAAG,CAAC,CAAC,KAAK,EAAE,SAAS,KAAK;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC,CAAC;IACpG;;OAEG;IACH,QAAQ,EAAE,YAAY,CAAC;IACvB;;OAEG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB;;OAEG;IACH,kBAAkB,EAAE,CAAC,KAAK,EAAE,SAAS,KAAK,OAAO,CAAC;CAClD,GAAG,CACD;IACA;;OAEG;IACH,QAAQ,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,CAAC,EAAE,MAAM,KAAK,GAAG,CAAC;CACtD,GACD;IACA;;;;OAIG;IACH,OAAO,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,CAAC,EAAE,MAAM,KAAK,MAAM,GAAG,MAAM,EAAE,GAAG,eAAe,GAAG,eAAe,EAAE,CAAC;CACzG,CACH,CAAC;AAsBF,iBAAS,UAAU,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAE7C;AAED,iBAAS,mBAAmB,CAAC,KAAK,EAAE,SAAS,WAE5C;AAED,iBAAS,WAAW,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAU9C;AAED,iBAAS,wBAAwB,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAE3D;AA0BD,iBAAS,cAAc,CAAC,KAAK,EAAE,SAAS,WAEvC;
|
|
1
|
+
{"version":3,"file":"local-apps.d.ts","sourceRoot":"","sources":["../../src/local-apps.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAKnD,MAAM,WAAW,eAAe;IAC/B;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;CAC3B;AAED;;GAEG;AACH,MAAM,MAAM,QAAQ,GAAG;IACtB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,KAAK,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,EAAE,GAAG,CAAC,CAAC,KAAK,EAAE,SAAS,KAAK;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC,CAAC;IACpG;;OAEG;IACH,QAAQ,EAAE,YAAY,CAAC;IACvB;;OAEG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB;;OAEG;IACH,kBAAkB,EAAE,CAAC,KAAK,EAAE,SAAS,KAAK,OAAO,CAAC;CAClD,GAAG,CACD;IACA;;OAEG;IACH,QAAQ,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,CAAC,EAAE,MAAM,KAAK,GAAG,CAAC;CACtD,GACD;IACA;;;;OAIG;IACH,OAAO,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,CAAC,EAAE,MAAM,KAAK,MAAM,GAAG,MAAM,EAAE,GAAG,eAAe,GAAG,eAAe,EAAE,CAAC;CACzG,CACH,CAAC;AAsBF,iBAAS,UAAU,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAE7C;AAED,iBAAS,mBAAmB,CAAC,KAAK,EAAE,SAAS,WAE5C;AAED,iBAAS,WAAW,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAU9C;AAED,iBAAS,wBAAwB,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAE3D;AA0BD,iBAAS,cAAc,CAAC,KAAK,EAAE,SAAS,WAEvC;AAWD,iBAAS,4BAA4B,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAM/D;AAwbD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,UAAU;;;;;;yBAtbS,SAAS,aAAa,MAAM,KAAG,eAAe,EAAE;;;;;;;yBA+DzC,SAAS,aAAa,MAAM,KAAG,eAAe,EAAE;;;;;;;yBAyC3D,SAAS,KAAG,eAAe,EAAE;;;;;;oCAwW3B,SAAS;yBAlTT,SAAS,KAAG,eAAe,EAAE;;;;;;;yBAoF9B,SAAS,KAAG,eAAe,EAAE;;;;;;;yBA7B/B,SAAS,KAAG,eAAe,EAAE;;;;;;;;;;;;;;yBApIzB,SAAS,aAAa,MAAM,KAAG,eAAe,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;yBAJjD,SAAS,aAAa,MAAM,KAAG,MAAM;;;;;;;;;;;;;;yBAkT1B,SAAS,aAAa,MAAM,KAAG,MAAM;;;;;;;yBAM9C,SAAS,aAAa,MAAM,KAAG,eAAe,EAAE;;;;;;;yBAlGtD,SAAS,aAAa,MAAM,KAAG,eAAe,EAAE;;;;;;;yBAkCvC,SAAS,aAAa,MAAM,KAAG,eAAe,EAAE;;;;;;;yBA2BnD,SAAS,aAAa,MAAM,KAAG,eAAe,EAAE;;CAqS5C,CAAC;AAErC,MAAM,MAAM,WAAW,GAAG,MAAM,OAAO,UAAU,CAAC"}
|
package/dist/esm/local-apps.js
CHANGED
|
@@ -57,6 +57,12 @@ function getChatTemplate(model) {
|
|
|
57
57
|
function isUnslothModel(model) {
|
|
58
58
|
return model.tags.includes("unsloth") || isLlamaCppGgufModel(model);
|
|
59
59
|
}
|
|
60
|
+
function isEmbeddingModel(model) {
|
|
61
|
+
return (!model.tags.includes("decision-model") &&
|
|
62
|
+
(model.tags.includes("feature-extraction") ||
|
|
63
|
+
model.pipeline_tag === "feature-extraction" ||
|
|
64
|
+
model.pipeline_tag === "sentence-similarity"));
|
|
65
|
+
}
|
|
60
66
|
function isToolCallingLocalAgentModel(model) {
|
|
61
67
|
return ((isLlamaCppGgufModel(model) || isMlxModel(model)) &&
|
|
62
68
|
model.tags.includes("conversational") &&
|
|
@@ -71,27 +77,35 @@ function getQuantTag(filepath) {
|
|
|
71
77
|
return quantLabel ? `:${quantLabel}` : defaultTag;
|
|
72
78
|
}
|
|
73
79
|
const snippetLlamacpp = (model, filepath) => {
|
|
80
|
+
const isEmbedding = isEmbeddingModel(model);
|
|
74
81
|
const serverCommand = (binary) => {
|
|
75
|
-
const snippet =
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
82
|
+
const snippet = isEmbedding
|
|
83
|
+
? [
|
|
84
|
+
"# Start a local OpenAI-compatible embeddings server:",
|
|
85
|
+
`${binary} -hf ${model.id}${getQuantTag(filepath)} --embeddings`,
|
|
86
|
+
]
|
|
87
|
+
: [
|
|
88
|
+
"# Start a local OpenAI-compatible server with a web UI:",
|
|
89
|
+
`${binary} -hf ${model.id}${getQuantTag(filepath)}`,
|
|
90
|
+
];
|
|
79
91
|
return snippet.join("\n");
|
|
80
92
|
};
|
|
81
93
|
const cliCommand = (binary) => {
|
|
82
94
|
const snippet = ["# Run inference directly in the terminal:", `${binary} -hf ${model.id}${getQuantTag(filepath)}`];
|
|
83
95
|
return snippet.join("\n");
|
|
84
96
|
};
|
|
97
|
+
// Embedding models don't generate text, so there is no CLI command for them
|
|
98
|
+
const commands = (serverBinary, cliBinary) => isEmbedding ? [serverCommand(serverBinary)] : [serverCommand(serverBinary), cliCommand(cliBinary)];
|
|
85
99
|
return [
|
|
86
100
|
{
|
|
87
101
|
title: "Install (macOS, Linux)",
|
|
88
102
|
setup: "curl -LsSf https://llama.app/install.sh | sh",
|
|
89
|
-
content:
|
|
103
|
+
content: commands("llama serve", "llama cli"),
|
|
90
104
|
},
|
|
91
105
|
{
|
|
92
106
|
title: "Install from WinGet (Windows)",
|
|
93
107
|
setup: "winget install llama.cpp",
|
|
94
|
-
content:
|
|
108
|
+
content: commands("llama serve", "llama cli"),
|
|
95
109
|
},
|
|
96
110
|
{
|
|
97
111
|
title: "Use pre-built binary",
|
|
@@ -100,7 +114,7 @@ const snippetLlamacpp = (model, filepath) => {
|
|
|
100
114
|
"# Download pre-built binary from:",
|
|
101
115
|
"# https://github.com/ggerganov/llama.cpp/releases",
|
|
102
116
|
].join("\n"),
|
|
103
|
-
content:
|
|
117
|
+
content: commands("./llama-server", "./llama-cli"),
|
|
104
118
|
},
|
|
105
119
|
{
|
|
106
120
|
title: "Build from source code",
|
|
@@ -110,12 +124,17 @@ const snippetLlamacpp = (model, filepath) => {
|
|
|
110
124
|
"cmake -B build",
|
|
111
125
|
"cmake --build build -j --target llama-server llama-cli",
|
|
112
126
|
].join("\n"),
|
|
113
|
-
content:
|
|
114
|
-
},
|
|
115
|
-
{
|
|
116
|
-
title: "Use Docker",
|
|
117
|
-
content: snippetDockerModelRunner(model, filepath),
|
|
127
|
+
content: commands("./build/bin/llama-server", "./build/bin/llama-cli"),
|
|
118
128
|
},
|
|
129
|
+
// `docker model run` starts a chat session, which embedding models don't support
|
|
130
|
+
...(isEmbedding
|
|
131
|
+
? []
|
|
132
|
+
: [
|
|
133
|
+
{
|
|
134
|
+
title: "Use Docker",
|
|
135
|
+
content: snippetDockerModelRunner(model, filepath),
|
|
136
|
+
},
|
|
137
|
+
]),
|
|
119
138
|
];
|
|
120
139
|
};
|
|
121
140
|
const snippetNodeLlamaCppCli = (model, filepath) => {
|
|
@@ -31,6 +31,30 @@ llama serve -hf mlabonne/gemma-2b-GGUF:{{QUANT_TAG}}`,
|
|
|
31
31
|
llama cli -hf mlabonne/gemma-2b-GGUF:{{QUANT_TAG}}`,
|
|
32
32
|
]);
|
|
33
33
|
});
|
|
34
|
+
it("llama.cpp embedding", async () => {
|
|
35
|
+
const { snippet: snippetFunc } = LOCAL_APPS["llama.cpp"];
|
|
36
|
+
const model = {
|
|
37
|
+
id: "ggml-org/embeddinggemma-300M-GGUF",
|
|
38
|
+
tags: ["feature-extraction"],
|
|
39
|
+
inference: "",
|
|
40
|
+
};
|
|
41
|
+
const snippet = snippetFunc(model);
|
|
42
|
+
expect(snippet[0].content).toEqual([
|
|
43
|
+
`# Start a local OpenAI-compatible embeddings server:
|
|
44
|
+
llama serve -hf ggml-org/embeddinggemma-300M-GGUF:{{QUANT_TAG}} --embeddings`,
|
|
45
|
+
]);
|
|
46
|
+
expect(snippet.map((s) => s.title)).not.toContain("Use Docker");
|
|
47
|
+
});
|
|
48
|
+
it("llama.cpp decision model is not treated as an embedding model", async () => {
|
|
49
|
+
const { snippet: snippetFunc } = LOCAL_APPS["llama.cpp"];
|
|
50
|
+
const model = {
|
|
51
|
+
id: "ggml-org/Julia-1-GGUF",
|
|
52
|
+
tags: ["feature-extraction", "decision-model"],
|
|
53
|
+
inference: "",
|
|
54
|
+
};
|
|
55
|
+
const snippet = snippetFunc(model);
|
|
56
|
+
expect(snippet[0].content[0]).not.toContain("--embeddings");
|
|
57
|
+
});
|
|
34
58
|
it("vLLM conversational llm", async () => {
|
|
35
59
|
const { snippet: snippetFunc } = LOCAL_APPS["vllm"];
|
|
36
60
|
const model = {
|
package/package.json
CHANGED
package/src/local-apps.spec.ts
CHANGED
|
@@ -37,6 +37,34 @@ llama cli -hf mlabonne/gemma-2b-GGUF:{{QUANT_TAG}}`,
|
|
|
37
37
|
]);
|
|
38
38
|
});
|
|
39
39
|
|
|
40
|
+
it("llama.cpp embedding", async () => {
|
|
41
|
+
const { snippet: snippetFunc } = LOCAL_APPS["llama.cpp"];
|
|
42
|
+
const model: ModelData = {
|
|
43
|
+
id: "ggml-org/embeddinggemma-300M-GGUF",
|
|
44
|
+
tags: ["feature-extraction"],
|
|
45
|
+
inference: "",
|
|
46
|
+
};
|
|
47
|
+
const snippet = snippetFunc(model);
|
|
48
|
+
|
|
49
|
+
expect(snippet[0].content).toEqual([
|
|
50
|
+
`# Start a local OpenAI-compatible embeddings server:
|
|
51
|
+
llama serve -hf ggml-org/embeddinggemma-300M-GGUF:{{QUANT_TAG}} --embeddings`,
|
|
52
|
+
]);
|
|
53
|
+
expect(snippet.map((s) => s.title)).not.toContain("Use Docker");
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
it("llama.cpp decision model is not treated as an embedding model", async () => {
|
|
57
|
+
const { snippet: snippetFunc } = LOCAL_APPS["llama.cpp"];
|
|
58
|
+
const model: ModelData = {
|
|
59
|
+
id: "ggml-org/Julia-1-GGUF",
|
|
60
|
+
tags: ["feature-extraction", "decision-model"],
|
|
61
|
+
inference: "",
|
|
62
|
+
};
|
|
63
|
+
const snippet = snippetFunc(model);
|
|
64
|
+
|
|
65
|
+
expect(snippet[0].content[0]).not.toContain("--embeddings");
|
|
66
|
+
});
|
|
67
|
+
|
|
40
68
|
it("vLLM conversational llm", async () => {
|
|
41
69
|
const { snippet: snippetFunc } = LOCAL_APPS["vllm"];
|
|
42
70
|
const model: ModelData = {
|
package/src/local-apps.ts
CHANGED
|
@@ -139,6 +139,15 @@ function isUnslothModel(model: ModelData) {
|
|
|
139
139
|
return model.tags.includes("unsloth") || isLlamaCppGgufModel(model);
|
|
140
140
|
}
|
|
141
141
|
|
|
142
|
+
function isEmbeddingModel(model: ModelData): boolean {
|
|
143
|
+
return (
|
|
144
|
+
!model.tags.includes("decision-model") &&
|
|
145
|
+
(model.tags.includes("feature-extraction") ||
|
|
146
|
+
model.pipeline_tag === "feature-extraction" ||
|
|
147
|
+
model.pipeline_tag === "sentence-similarity")
|
|
148
|
+
);
|
|
149
|
+
}
|
|
150
|
+
|
|
142
151
|
function isToolCallingLocalAgentModel(model: ModelData): boolean {
|
|
143
152
|
return (
|
|
144
153
|
(isLlamaCppGgufModel(model) || isMlxModel(model)) &&
|
|
@@ -159,27 +168,36 @@ function getQuantTag(filepath?: string): string {
|
|
|
159
168
|
}
|
|
160
169
|
|
|
161
170
|
const snippetLlamacpp = (model: ModelData, filepath?: string): LocalAppSnippet[] => {
|
|
171
|
+
const isEmbedding = isEmbeddingModel(model);
|
|
162
172
|
const serverCommand = (binary: string) => {
|
|
163
|
-
const snippet =
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
173
|
+
const snippet = isEmbedding
|
|
174
|
+
? [
|
|
175
|
+
"# Start a local OpenAI-compatible embeddings server:",
|
|
176
|
+
`${binary} -hf ${model.id}${getQuantTag(filepath)} --embeddings`,
|
|
177
|
+
]
|
|
178
|
+
: [
|
|
179
|
+
"# Start a local OpenAI-compatible server with a web UI:",
|
|
180
|
+
`${binary} -hf ${model.id}${getQuantTag(filepath)}`,
|
|
181
|
+
];
|
|
167
182
|
return snippet.join("\n");
|
|
168
183
|
};
|
|
169
184
|
const cliCommand = (binary: string) => {
|
|
170
185
|
const snippet = ["# Run inference directly in the terminal:", `${binary} -hf ${model.id}${getQuantTag(filepath)}`];
|
|
171
186
|
return snippet.join("\n");
|
|
172
187
|
};
|
|
188
|
+
// Embedding models don't generate text, so there is no CLI command for them
|
|
189
|
+
const commands = (serverBinary: string, cliBinary: string) =>
|
|
190
|
+
isEmbedding ? [serverCommand(serverBinary)] : [serverCommand(serverBinary), cliCommand(cliBinary)];
|
|
173
191
|
return [
|
|
174
192
|
{
|
|
175
193
|
title: "Install (macOS, Linux)",
|
|
176
194
|
setup: "curl -LsSf https://llama.app/install.sh | sh",
|
|
177
|
-
content:
|
|
195
|
+
content: commands("llama serve", "llama cli"),
|
|
178
196
|
},
|
|
179
197
|
{
|
|
180
198
|
title: "Install from WinGet (Windows)",
|
|
181
199
|
setup: "winget install llama.cpp",
|
|
182
|
-
content:
|
|
200
|
+
content: commands("llama serve", "llama cli"),
|
|
183
201
|
},
|
|
184
202
|
{
|
|
185
203
|
title: "Use pre-built binary",
|
|
@@ -188,7 +206,7 @@ const snippetLlamacpp = (model: ModelData, filepath?: string): LocalAppSnippet[]
|
|
|
188
206
|
"# Download pre-built binary from:",
|
|
189
207
|
"# https://github.com/ggerganov/llama.cpp/releases",
|
|
190
208
|
].join("\n"),
|
|
191
|
-
content:
|
|
209
|
+
content: commands("./llama-server", "./llama-cli"),
|
|
192
210
|
},
|
|
193
211
|
{
|
|
194
212
|
title: "Build from source code",
|
|
@@ -198,12 +216,17 @@ const snippetLlamacpp = (model: ModelData, filepath?: string): LocalAppSnippet[]
|
|
|
198
216
|
"cmake -B build",
|
|
199
217
|
"cmake --build build -j --target llama-server llama-cli",
|
|
200
218
|
].join("\n"),
|
|
201
|
-
content:
|
|
202
|
-
},
|
|
203
|
-
{
|
|
204
|
-
title: "Use Docker",
|
|
205
|
-
content: snippetDockerModelRunner(model, filepath),
|
|
219
|
+
content: commands("./build/bin/llama-server", "./build/bin/llama-cli"),
|
|
206
220
|
},
|
|
221
|
+
// `docker model run` starts a chat session, which embedding models don't support
|
|
222
|
+
...(isEmbedding
|
|
223
|
+
? []
|
|
224
|
+
: [
|
|
225
|
+
{
|
|
226
|
+
title: "Use Docker",
|
|
227
|
+
content: snippetDockerModelRunner(model, filepath),
|
|
228
|
+
},
|
|
229
|
+
]),
|
|
207
230
|
];
|
|
208
231
|
};
|
|
209
232
|
|