@huggingface/tasks 0.12.0 → 0.12.1

package/dist/index.cjs

@@ -6768,6 +6768,24 @@ var SKUS = {
 };
 
 // src/local-apps.ts
+function isGgufModel(model) {
+  return model.tags.includes("gguf");
+}
+function isAwqModel(model) {
+  return model.config?.quantization_config?.quant_method === "awq";
+}
+function isGptqModel(model) {
+  return model.config?.quantization_config?.quant_method === "gptq";
+}
+function isAqlmModel(model) {
+  return model.config?.quantization_config?.quant_method === "aqlm";
+}
+function isMarlinModel(model) {
+  return model.config?.quantization_config?.quant_method === "marlin";
+}
+function isTransformersModel(model) {
+  return model.tags.includes("transformers");
+}
 function isLlamaCppGgufModel(model) {
   return !!model.gguf?.context_length;
 }
@@ -6827,6 +6845,46 @@ var snippetLocalAI = (model, filepath) => {
     }
   ];
 };
+var snippetVllm = (model) => {
+  const runCommand = [
+    "",
+    "# Call the server using curl:",
+    `curl -X POST "http://localhost:8000/v1/chat/completions" \\ `,
+    `	-H "Content-Type: application/json" \\ `,
+    `	--data '{`,
+    `		"model": "${model.id}"`,
+    `		"messages": [`,
+    `			{"role": "user", "content": "Hello!"}`,
+    `		]`,
+    `	}'`
+  ];
+  return [
+    {
+      title: "Install from pip",
+      setup: ["# Install vLLM from pip:", "pip install vllm"].join("\n"),
+      content: ["# Load and run the model:", `vllm serve "${model.id}"`, ...runCommand].join("\n")
+    },
+    {
+      title: "Use Docker images",
+      setup: [
+        "# Deploy with docker on Linux:",
+        `docker run --runtime nvidia --gpus all \\`,
+        `	--name my_vllm_container \\`,
+        `	-v ~/.cache/huggingface:/root/.cache/huggingface \\`,
+        ` 	--env "HUGGING_FACE_HUB_TOKEN=<secret>" \\`,
+        `	-p 8000:8000 \\`,
+        `	--ipc=host \\`,
+        `	vllm/vllm-openai:latest \\`,
+        `	--model ${model.id}`
+      ].join("\n"),
+      content: [
+        "# Load and run the model:",
+        `docker exec -it my_vllm_container bash -c "vllm serve ${model.id}"`,
+        ...runCommand
+      ].join("\n")
+    }
+  ];
+};
 var LOCAL_APPS = {
   "llama.cpp": {
     prettyLabel: "llama.cpp",
@@ -6835,6 +6893,13 @@ var LOCAL_APPS = {
     displayOnModelPage: isLlamaCppGgufModel,
     snippet: snippetLlamacpp
   },
+  vllm: {
+    prettyLabel: "vLLM",
+    docsUrl: "https://docs.vllm.ai",
+    mainTask: "text-generation",
+    displayOnModelPage: (model) => isAwqModel(model) || isGptqModel(model) || isAqlmModel(model) || isMarlinModel(model) || isGgufModel(model) || isTransformersModel(model),
+    snippet: snippetVllm
+  },
   lmstudio: {
     prettyLabel: "LM Studio",
     docsUrl: "https://lmstudio.ai",

package/dist/index.js

@@ -6730,6 +6730,24 @@ var SKUS = {
 };
 
 // src/local-apps.ts
+function isGgufModel(model) {
+  return model.tags.includes("gguf");
+}
+function isAwqModel(model) {
+  return model.config?.quantization_config?.quant_method === "awq";
+}
+function isGptqModel(model) {
+  return model.config?.quantization_config?.quant_method === "gptq";
+}
+function isAqlmModel(model) {
+  return model.config?.quantization_config?.quant_method === "aqlm";
+}
+function isMarlinModel(model) {
+  return model.config?.quantization_config?.quant_method === "marlin";
+}
+function isTransformersModel(model) {
+  return model.tags.includes("transformers");
+}
 function isLlamaCppGgufModel(model) {
   return !!model.gguf?.context_length;
 }
@@ -6789,6 +6807,46 @@ var snippetLocalAI = (model, filepath) => {
     }
   ];
 };
+var snippetVllm = (model) => {
+  const runCommand = [
+    "",
+    "# Call the server using curl:",
+    `curl -X POST "http://localhost:8000/v1/chat/completions" \\ `,
+    `	-H "Content-Type: application/json" \\ `,
+    `	--data '{`,
+    `		"model": "${model.id}"`,
+    `		"messages": [`,
+    `			{"role": "user", "content": "Hello!"}`,
+    `		]`,
+    `	}'`
+  ];
+  return [
+    {
+      title: "Install from pip",
+      setup: ["# Install vLLM from pip:", "pip install vllm"].join("\n"),
+      content: ["# Load and run the model:", `vllm serve "${model.id}"`, ...runCommand].join("\n")
+    },
+    {
+      title: "Use Docker images",
+      setup: [
+        "# Deploy with docker on Linux:",
+        `docker run --runtime nvidia --gpus all \\`,
+        `	--name my_vllm_container \\`,
+        `	-v ~/.cache/huggingface:/root/.cache/huggingface \\`,
+        ` 	--env "HUGGING_FACE_HUB_TOKEN=<secret>" \\`,
+        `	-p 8000:8000 \\`,
+        `	--ipc=host \\`,
+        `	vllm/vllm-openai:latest \\`,
+        `	--model ${model.id}`
+      ].join("\n"),
+      content: [
+        "# Load and run the model:",
+        `docker exec -it my_vllm_container bash -c "vllm serve ${model.id}"`,
+        ...runCommand
+      ].join("\n")
+    }
+  ];
+};
 var LOCAL_APPS = {
   "llama.cpp": {
     prettyLabel: "llama.cpp",
@@ -6797,6 +6855,13 @@ var LOCAL_APPS = {
     displayOnModelPage: isLlamaCppGgufModel,
     snippet: snippetLlamacpp
   },
+  vllm: {
+    prettyLabel: "vLLM",
+    docsUrl: "https://docs.vllm.ai",
+    mainTask: "text-generation",
+    displayOnModelPage: (model) => isAwqModel(model) || isGptqModel(model) || isAqlmModel(model) || isMarlinModel(model) || isGgufModel(model) || isTransformersModel(model),
+    snippet: snippetVllm
+  },
   lmstudio: {
     prettyLabel: "LM Studio",
     docsUrl: "https://lmstudio.ai",

package/dist/src/local-apps.d.ts

@@ -71,6 +71,13 @@ export declare const LOCAL_APPS: {
         displayOnModelPage: typeof isLlamaCppGgufModel;
         snippet: (model: ModelData, filepath?: string) => LocalAppSnippet[];
     };
+    vllm: {
+        prettyLabel: string;
+        docsUrl: string;
+        mainTask: "text-generation";
+        displayOnModelPage: (model: ModelData) => boolean;
+        snippet: (model: ModelData) => LocalAppSnippet[];
+    };
     lmstudio: {
         prettyLabel: string;
         docsUrl: string;

package/dist/src/local-apps.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"local-apps.d.ts","sourceRoot":"","sources":["../../src/local-apps.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAEhD,MAAM,WAAW,eAAe;IAC/B;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,MAAM,QAAQ,GAAG;IACtB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,QAAQ,EAAE,YAAY,CAAC;IACvB;;OAEG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB;;OAEG;IACH,kBAAkB,EAAE,CAAC,KAAK,EAAE,SAAS,KAAK,OAAO,CAAC;CAClD,GAAG,CACD;IACA;;OAEG;IACH,QAAQ,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,CAAC,EAAE,MAAM,KAAK,GAAG,CAAC;CACtD,GACD;IACA;;;OAGG;IACH,OAAO,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,CAAC,EAAE,MAAM,KAAK,MAAM,GAAG,MAAM,EAAE,GAAG,eAAe,GAAG,eAAe,EAAE,CAAC;CACzG,CACH,CAAC;AAOF,iBAAS,mBAAmB,CAAC,KAAK,EAAE,SAAS,WAE5C;AA8DD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,UAAU;;;;;;yBAvES,SAAS,aAAa,MAAM,KAAG,eAAe,EAAE;;;;;;;;;;;;;;yBAqCjD,SAAS,aAAa,MAAM,KAAG,eAAe,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAuJ3C,CAAC;AAErC,MAAM,MAAM,WAAW,GAAG,MAAM,OAAO,UAAU,CAAC"}
+{"version":3,"file":"local-apps.d.ts","sourceRoot":"","sources":["../../src/local-apps.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAEhD,MAAM,WAAW,eAAe;IAC/B;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,MAAM,QAAQ,GAAG;IACtB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,QAAQ,EAAE,YAAY,CAAC;IACvB;;OAEG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB;;OAEG;IACH,kBAAkB,EAAE,CAAC,KAAK,EAAE,SAAS,KAAK,OAAO,CAAC;CAClD,GAAG,CACD;IACA;;OAEG;IACH,QAAQ,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,CAAC,EAAE,MAAM,KAAK,GAAG,CAAC;CACtD,GACD;IACA;;;OAGG;IACH,OAAO,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,CAAC,EAAE,MAAM,KAAK,MAAM,GAAG,MAAM,EAAE,GAAG,eAAe,GAAG,eAAe,EAAE,CAAC;CACzG,CACH,CAAC;AA0BF,iBAAS,mBAAmB,CAAC,KAAK,EAAE,SAAS,WAE5C;AAuGD;;;;;;;;;;GAUG;AACH,eAAO,MAAM,UAAU;;;;;;yBAhHS,SAAS,aAAa,MAAM,KAAG,eAAe,EAAE;;;;;;oCA4HlD,SAAS;yBAhEX,SAAS,KAAG,eAAe,EAAE;;;;;;;;;;;;;;yBAvB1B,SAAS,aAAa,MAAM,KAAG,eAAe,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA6M3C,CAAC;AAErC,MAAM,MAAM,WAAW,GAAG,MAAM,OAAO,UAAU,CAAC"}

package/dist/src/model-data.d.ts

@@ -37,6 +37,10 @@ export interface ModelData {
             bits?: number;
             load_in_4bit?: boolean;
             load_in_8bit?: boolean;
+            /**
+             * awq, gptq, aqlm, marlin, … Used by vLLM
+             */
+            quant_method?: string;
         };
         tokenizer_config?: TokenizerConfig;
         adapter_transformers?: {

package/dist/src/model-data.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"model-data.d.ts","sourceRoot":"","sources":["../../src/model-data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAChD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAExD;;GAEG;AACH,MAAM,WAAW,SAAS;IACzB;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IACX;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB;;OAEG;IACH,MAAM,CAAC,EAAE;QACR,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;QACzB;;WAEG;QACH,QAAQ,CAAC,EAAE;YACV;;eAEG;YACH,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;SACpB,CAAC;QACF,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,mBAAmB,CAAC,EAAE;YACrB,IAAI,CAAC,EAAE,MAAM,CAAC;YACd,YAAY,CAAC,EAAE,OAAO,CAAC;YACvB,YAAY,CAAC,EAAE,OAAO,CAAC;SACvB,CAAC;QACF,gBAAgB,CAAC,EAAE,eAAe,CAAC;QACnC,oBAAoB,CAAC,EAAE;YACtB,UAAU,CAAC,EAAE,MAAM,CAAC;YACpB,WAAW,CAAC,EAAE,MAAM,CAAC;SACrB,CAAC;QACF,SAAS,CAAC,EAAE;YACX,WAAW,CAAC,EAAE,MAAM,CAAC;SACrB,CAAC;QACF,OAAO,CAAC,EAAE;YACT,KAAK,CAAC,EAAE;gBACP,IAAI,CAAC,EAAE,MAAM,CAAC;aACd,CAAC;YACF,YAAY,CAAC,EAAE,MAAM,CAAC;SACtB,CAAC;QACF,WAAW,CAAC,EAAE;YACb,qBAAqB,CAAC,EAAE,MAAM,CAAC;YAC/B,iBAAiB,CAAC,EAAE,MAAM,CAAC;YAC3B,gBAAgB,CAAC,EAAE,MAAM,CAAC;SAC1B,CAAC;QACF,IAAI,CAAC,EAAE;YACN,uBAAuB,CAAC,EAAE,MAAM,CAAC;YACjC,SAAS,CAAC,EAAE,MAAM,CAAC;SACnB,CAAC;KACF,CAAC;IACF;;OAEG;IACH,IAAI,EAAE,MAAM,EAAE,CAAC;IACf;;OAEG;IACH,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;IACpC;;OAEG;IACH,YAAY,CAAC,EAAE,YAAY,GAAG,SAAS,CAAC;IACxC;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAChC;;;;;OAKG;IACH,UAAU,CAAC,EAAE,aAAa,EAAE,GAAG,SAAS,CAAC;IACzC;;;;;;;;;OASG;IACH,QAAQ,CAAC,EAAE;QACV,SAAS,CAAC,EACP,OAAO,GACP;YACA,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;SACpC,CAAC;QACL,UAAU,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;KAC/B,CAAC;IACF;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE;QACb,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACnC,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,EAAE,OAAO,CAAC;KACjB,CAAC;IACF,IAAI,CAAC,EAAE;QACN,KAAK,EAAE,MAAM,CAAC;QACd,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,cAAc,CAAC,EAAE,MAAM,CAAC;KACxB,CAAC;CACF;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAChC;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IACnB;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;OAEG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACnB"}
+{"version":3,"file":"model-data.d.ts","sourceRoot":"","sources":["../../src/model-data.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAChD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAExD;;GAEG;AACH,MAAM,WAAW,SAAS;IACzB;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IACX;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB;;OAEG;IACH,MAAM,CAAC,EAAE;QACR,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;QACzB;;WAEG;QACH,QAAQ,CAAC,EAAE;YACV;;eAEG;YACH,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;SACpB,CAAC;QACF,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,mBAAmB,CAAC,EAAE;YACrB,IAAI,CAAC,EAAE,MAAM,CAAC;YACd,YAAY,CAAC,EAAE,OAAO,CAAC;YACvB,YAAY,CAAC,EAAE,OAAO,CAAC;YACvB;;eAEG;YACH,YAAY,CAAC,EAAE,MAAM,CAAC;SACtB,CAAC;QACF,gBAAgB,CAAC,EAAE,eAAe,CAAC;QACnC,oBAAoB,CAAC,EAAE;YACtB,UAAU,CAAC,EAAE,MAAM,CAAC;YACpB,WAAW,CAAC,EAAE,MAAM,CAAC;SACrB,CAAC;QACF,SAAS,CAAC,EAAE;YACX,WAAW,CAAC,EAAE,MAAM,CAAC;SACrB,CAAC;QACF,OAAO,CAAC,EAAE;YACT,KAAK,CAAC,EAAE;gBACP,IAAI,CAAC,EAAE,MAAM,CAAC;aACd,CAAC;YACF,YAAY,CAAC,EAAE,MAAM,CAAC;SACtB,CAAC;QACF,WAAW,CAAC,EAAE;YACb,qBAAqB,CAAC,EAAE,MAAM,CAAC;YAC/B,iBAAiB,CAAC,EAAE,MAAM,CAAC;YAC3B,gBAAgB,CAAC,EAAE,MAAM,CAAC;SAC1B,CAAC;QACF,IAAI,CAAC,EAAE;YACN,uBAAuB,CAAC,EAAE,MAAM,CAAC;YACjC,SAAS,CAAC,EAAE,MAAM,CAAC;SACnB,CAAC;KACF,CAAC;IACF;;OAEG;IACH,IAAI,EAAE,MAAM,EAAE,CAAC;IACf;;OAEG;IACH,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;IACpC;;OAEG;IACH,YAAY,CAAC,EAAE,YAAY,GAAG,SAAS,CAAC;IACxC;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAChC;;;;;OAKG;IACH,UAAU,CAAC,EAAE,aAAa,EAAE,GAAG,SAAS,CAAC;IACzC;;;;;;;;;OASG;IACH,QAAQ,CAAC,EAAE;QACV,SAAS,CAAC,EACP,OAAO,GACP;YACA,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;SACpC,CAAC;QACL,UAAU,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;KAC/B,CAAC;IACF;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE;QACb,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACnC,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,EAAE,OAAO,CAAC;KACjB,CAAC;IACF,IAAI,CAAC,EAAE;QACN,KAAK,EAAE,MAAM,CAAC;QACd,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,cAAc,CAAC,EAAE,MAAM,CAAC;KACxB,CAAC;CACF;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAChC;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IACnB;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;OAEG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACnB"}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@huggingface/tasks",
   "packageManager": "pnpm@8.10.5",
-  "version": "0.12.0",
+  "version": "0.12.1",
   "description": "List of ML tasks for huggingface.co/tasks",
   "repository": "https://github.com/huggingface/huggingface.js.git",
   "publishConfig": {

package/src/local-apps.ts

@@ -58,11 +58,30 @@ export type LocalApp = {
 	  }
 );
 
-// eslint-disable-next-line @typescript-eslint/no-unused-vars
-function isGgufModel(model: ModelData) {
+function isGgufModel(model: ModelData): boolean {
 	return model.tags.includes("gguf");
 }
 
+function isAwqModel(model: ModelData): boolean {
+	return model.config?.quantization_config?.quant_method === "awq";
+}
+
+function isGptqModel(model: ModelData): boolean {
+	return model.config?.quantization_config?.quant_method === "gptq";
+}
+
+function isAqlmModel(model: ModelData): boolean {
+	return model.config?.quantization_config?.quant_method === "aqlm";
+}
+
+function isMarlinModel(model: ModelData): boolean {
+	return model.config?.quantization_config?.quant_method === "marlin";
+}
+
+function isTransformersModel(model: ModelData): boolean {
+	return model.tags.includes("transformers");
+}
+
 function isLlamaCppGgufModel(model: ModelData) {
 	return !!model.gguf?.context_length;
 }
@@ -127,6 +146,47 @@ const snippetLocalAI = (model: ModelData, filepath?: string): LocalAppSnippet[]
 	];
 };
 
+const snippetVllm = (model: ModelData): LocalAppSnippet[] => {
+	const runCommand = [
+		"",
+		"# Call the server using curl:",
+		`curl -X POST "http://localhost:8000/v1/chat/completions" \\ `,
+		`	-H "Content-Type: application/json" \\ `,
+		`	--data '{`,
+		`		"model": "${model.id}"`,
+		`		"messages": [`,
+		`			{"role": "user", "content": "Hello!"}`,
+		`		]`,
+		`	}'`,
+	];
+	return [
+		{
+			title: "Install from pip",
+			setup: ["# Install vLLM from pip:", "pip install vllm"].join("\n"),
+			content: ["# Load and run the model:", `vllm serve "${model.id}"`, ...runCommand].join("\n"),
+		},
+		{
+			title: "Use Docker images",
+			setup: [
+				"# Deploy with docker on Linux:",
+				`docker run --runtime nvidia --gpus all \\`,
+				`	--name my_vllm_container \\`,
+				`	-v ~/.cache/huggingface:/root/.cache/huggingface \\`,
+				` 	--env "HUGGING_FACE_HUB_TOKEN=<secret>" \\`,
+				`	-p 8000:8000 \\`,
+				`	--ipc=host \\`,
+				`	vllm/vllm-openai:latest \\`,
+				`	--model ${model.id}`,
+			].join("\n"),
+			content: [
+				"# Load and run the model:",
+				`docker exec -it my_vllm_container bash -c "vllm serve ${model.id}"`,
+				...runCommand,
+			].join("\n"),
+		},
+	];
+};
+
 /**
  * Add your new local app here.
  *
@@ -146,6 +206,19 @@ export const LOCAL_APPS = {
 		displayOnModelPage: isLlamaCppGgufModel,
 		snippet: snippetLlamacpp,
 	},
+	vllm: {
+		prettyLabel: "vLLM",
+		docsUrl: "https://docs.vllm.ai",
+		mainTask: "text-generation",
+		displayOnModelPage: (model: ModelData) =>
+			isAwqModel(model) ||
+			isGptqModel(model) ||
+			isAqlmModel(model) ||
+			isMarlinModel(model) ||
+			isGgufModel(model) ||
+			isTransformersModel(model),
+		snippet: snippetVllm,
+	},
 	lmstudio: {
 		prettyLabel: "LM Studio",
 		docsUrl: "https://lmstudio.ai",

package/src/model-data.ts

@@ -38,6 +38,10 @@ export interface ModelData {
 			bits?: number;
 			load_in_4bit?: boolean;
 			load_in_8bit?: boolean;
+			/**
+			 * awq, gptq, aqlm, marlin, … Used by vLLM
+			 */
+			quant_method?: string;
 		};
 		tokenizer_config?: TokenizerConfig;
 		adapter_transformers?: {