@huggingface/tasks 0.19.66 → 0.19.68

package/dist/esm/tasks/image-text-to-video/inference.d.ts

@@ -0,0 +1,78 @@
+/**
+ * Inference code generated from the JSON schema spec in ./spec
+ *
+ * Using src/scripts/inference-codegen
+ */
+/**
+ * Inputs for Image Text To Video inference. Either inputs (image) or prompt (in parameters)
+ * must be provided, or both.
+ */
+export interface ImageTextToVideoInput {
+    /**
+     * The input image data as a base64-encoded string. If no `parameters` are provided, you can
+     * also provide the image data as a raw bytes payload. Either this or prompt must be
+     * provided.
+     */
+    inputs?: Blob;
+    /**
+     * Additional inference parameters for Image Text To Video
+     */
+    parameters?: ImageTextToVideoParameters;
+    [property: string]: unknown;
+}
+/**
+ * Additional inference parameters for Image Text To Video
+ */
+export interface ImageTextToVideoParameters {
+    /**
+     * For diffusion models. A higher guidance scale value encourages the model to generate
+     * videos closely linked to the text prompt at the expense of lower image quality.
+     */
+    guidance_scale?: number;
+    /**
+     * One prompt to guide what NOT to include in video generation.
+     */
+    negative_prompt?: string;
+    /**
+     * The num_frames parameter determines how many video frames are generated.
+     */
+    num_frames?: number;
+    /**
+     * The number of denoising steps. More denoising steps usually lead to a higher quality
+     * video at the expense of slower inference.
+     */
+    num_inference_steps?: number;
+    /**
+     * The text prompt to guide the video generation. Either this or inputs (image) must be
+     * provided.
+     */
+    prompt?: string;
+    /**
+     * Seed for the random number generator.
+     */
+    seed?: number;
+    /**
+     * The size in pixel of the output video frames.
+     */
+    target_size?: TargetSize;
+    [property: string]: unknown;
+}
+/**
+ * The size in pixel of the output video frames.
+ */
+export interface TargetSize {
+    height: number;
+    width: number;
+    [property: string]: unknown;
+}
+/**
+ * Outputs of inference for the Image Text To Video task
+ */
+export interface ImageTextToVideoOutput {
+    /**
+     * The generated video returned as raw bytes in the payload.
+     */
+    video: unknown;
+    [property: string]: unknown;
+}
+//# sourceMappingURL=inference.d.ts.map

package/dist/esm/tasks/image-text-to-video/inference.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../../../src/tasks/image-text-to-video/inference.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH;;;GAGG;AACH,MAAM,WAAW,qBAAqB;IACrC;;;;OAIG;IACH,MAAM,CAAC,EAAE,IAAI,CAAC;IACd;;OAEG;IACH,UAAU,CAAC,EAAE,0BAA0B,CAAC;IACxC,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,0BAA0B;IAC1C;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;OAEG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;OAEG;IACH,WAAW,CAAC,EAAE,UAAU,CAAC;IACzB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,UAAU;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B;AACD;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACtC;;OAEG;IACH,KAAK,EAAE,OAAO,CAAC;IACf,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;CAC5B"}

package/dist/esm/tasks/image-text-to-video/inference.js

@@ -0,0 +1 @@
+export {};

package/dist/esm/tasks/index.d.ts

@@ -10,6 +10,8 @@ export type * from "./image-to-image/inference.js";
 export type { ImageToTextInput, ImageToTextOutput, ImageToTextParameters } from "./image-to-text/inference.js";
 export type * from "./image-segmentation/inference.js";
 export type { ImageToVideoInput, ImageToVideoOutput, ImageToVideoParameters } from "./image-to-video/inference.js";
+export type { ImageTextToImageInput, ImageTextToImageOutput, ImageTextToImageParameters, } from "./image-text-to-image/inference.js";
+export type { ImageTextToVideoInput, ImageTextToVideoOutput, ImageTextToVideoParameters, } from "./image-text-to-video/inference.js";
 export type * from "./object-detection/inference.js";
 export type * from "./depth-estimation/inference.js";
 export type * from "./question-answering/inference.js";

package/dist/esm/tasks/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/tasks/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAkDpD,mBAAmB,qCAAqC,CAAC;AACzD,mBAAmB,6CAA6C,CAAC;AACjE,YAAY,EACX,mBAAmB,EACnB,0BAA0B,EAC1B,mCAAmC,EACnC,oBAAoB,EACpB,4BAA4B,EAC5B,2BAA2B,EAC3B,0BAA0B,EAC1B,gCAAgC,EAChC,+BAA+B,GAC/B,MAAM,gCAAgC,CAAC;AACxC,mBAAmB,4CAA4C,CAAC;AAChE,mBAAmB,mCAAmC,CAAC;AACvD,mBAAmB,0BAA0B,CAAC;AAC9C,YAAY,EACX,wBAAwB,EACxB,yBAAyB,EACzB,gCAAgC,EAChC,6BAA6B,GAC7B,MAAM,qCAAqC,CAAC;AAC7C,mBAAmB,+BAA+B,CAAC;AACnD,YAAY,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AAC/G,mBAAmB,mCAAmC,CAAC;AACvD,YAAY,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAC;AACnH,mBAAmB,iCAAiC,CAAC;AACrD,mBAAmB,iCAAiC,CAAC;AACrD,mBAAmB,mCAAmC,CAAC;AACvD,mBAAmB,oCAAoC,CAAC;AACxD,mBAAmB,8BAA8B,CAAC;AAClD,mBAAmB,yCAAyC,CAAC;AAC7D,YAAY,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AAC/G,YAAY,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AAC/G,YAAY,EAAE,sBAAsB,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AACnH,mBAAmB,qCAAqC,CAAC;AACzD,YAAY,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAC;AACtF,YAAY,EACX,6BAA6B,EAC7B,uBAAuB,EACvB,wBAAwB,EACxB,+BAA+B,EAC/B,4BAA4B,GAC5B,MAAM,oCAAoC,CAAC;AAC5C,YAAY,EACX,gCAAgC,EAChC,gCAAgC,EAChC,mBAAmB,EACnB,oBAAoB,EACpB,2BAA2B,EAC3B,qCAAqC,EACrC,kCAAkC,EAClC,yBAAyB,EACzB,uCAAuC,EACvC,0BAA0B,GAC1B,MAAM,gCAAgC,CAAC;AACxC,mBAAmB,qCAAqC,CAAC;AACzD,mBAAmB,0CAA0C,CAAC;AAC9D,mBAAmB,yCAAyC,CAAC;AAC7D,mBAAmB,+CAA+C,CAAC;AACnE,YAAY,EACX,WAAW,EACX,4BAA4B,EAC5B,6BAA6B,EAC7B,oCAAoC,GACpC,MAAM,2CAA2C,CAAC;AAEnD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAC7D;;GAEG;AACH,eAAO,MAAM,qBAAqB,EAAE,MAAM,CAAC,YAAY,EAAE,eAAe,EAAE,CAgEzE,CAAC;AAoBF,eAAO,MAAM,UAAU,EAAE,MAAM,CAAC,YAAY,EAAE,QAAQ,GAAG,SAAS,CAwDxD,CAAC;AAEX,MAAM,WAAW,WAAW;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,EAAE,EAAE,MAAM,CAAC;CACX;AAED,MAAM,MAAM,aAAa,GACtB;IACA,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,OAAO,CAAC;CACb,GACD;IACA,IAAI,EAAE,KAAK,CAAC;QACX,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,MAAM,CAAC;KACd,CAAC,CAAC;IACH,IAAI,EAAE,OAAO,CAAC;CACb,GACD;IACA,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,KAAK,CAAC;CACX,GACD;IACA,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC;IAClB,IAAI,EAAE,SAAS,CAAC;CACf,GACD;IACA,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;CACZ,GACD;IACA,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,KAAK,CAAC;QACb,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;QACd,IAAI,EAAE,MAAM,CAAC;KACb,CAAC,CAAC;IACH,IAAI,EAAE,kBAAkB,CAAC;CACxB,CAAC;AAEL,MAAM,WAAW,QAAQ;IACxB,MAAM,EAAE,aAAa,EAAE,CAAC;IACxB,OAAO,EAAE,aAAa,EAAE,CAAC;CACzB;AAED,MAAM,WAAW,QAAQ;IACxB,QAAQ,EAAE,WAAW,EAAE,CAAC;IACxB,IAAI,EAAE,QAAQ,CAAC;IACf,EAAE,EAAE,YAAY,CAAC;IACjB,WAAW,CAAC,EAAE,YAAY,CAAC;IAC3B,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,eAAe,EAAE,CAAC;IAC7B,OAAO,EAAE,WAAW,EAAE,CAAC;IACvB,MAAM,EAAE,WAAW,EAAE,CAAC;IACtB,MAAM,EAAE,WAAW,EAAE,CAAC;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,SAAS,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,MAAM,cAAc,GAAG,IAAI,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,GAAG,WAAW,CAAC,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/tasks/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAoDpD,mBAAmB,qCAAqC,CAAC;AACzD,mBAAmB,6CAA6C,CAAC;AACjE,YAAY,EACX,mBAAmB,EACnB,0BAA0B,EAC1B,mCAAmC,EACnC,oBAAoB,EACpB,4BAA4B,EAC5B,2BAA2B,EAC3B,0BAA0B,EAC1B,gCAAgC,EAChC,+BAA+B,GAC/B,MAAM,gCAAgC,CAAC;AACxC,mBAAmB,4CAA4C,CAAC;AAChE,mBAAmB,mCAAmC,CAAC;AACvD,mBAAmB,0BAA0B,CAAC;AAC9C,YAAY,EACX,wBAAwB,EACxB,yBAAyB,EACzB,gCAAgC,EAChC,6BAA6B,GAC7B,MAAM,qCAAqC,CAAC;AAC7C,mBAAmB,+BAA+B,CAAC;AACnD,YAAY,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AAC/G,mBAAmB,mCAAmC,CAAC;AACvD,YAAY,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAC;AACnH,YAAY,EACX,qBAAqB,EACrB,sBAAsB,EACtB,0BAA0B,GAC1B,MAAM,oCAAoC,CAAC;AAC5C,YAAY,EACX,qBAAqB,EACrB,sBAAsB,EACtB,0BAA0B,GAC1B,MAAM,oCAAoC,CAAC;AAC5C,mBAAmB,iCAAiC,CAAC;AACrD,mBAAmB,iCAAiC,CAAC;AACrD,mBAAmB,mCAAmC,CAAC;AACvD,mBAAmB,oCAAoC,CAAC;AACxD,mBAAmB,8BAA8B,CAAC;AAClD,mBAAmB,yCAAyC,CAAC;AAC7D,YAAY,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AAC/G,YAAY,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AAC/G,YAAY,EAAE,sBAAsB,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AACnH,mBAAmB,qCAAqC,CAAC;AACzD,YAAY,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAC;AACtF,YAAY,EACX,6BAA6B,EAC7B,uBAAuB,EACvB,wBAAwB,EACxB,+BAA+B,EAC/B,4BAA4B,GAC5B,MAAM,oCAAoC,CAAC;AAC5C,YAAY,EACX,gCAAgC,EAChC,gCAAgC,EAChC,mBAAmB,EACnB,oBAAoB,EACpB,2BAA2B,EAC3B,qCAAqC,EACrC,kCAAkC,EAClC,yBAAyB,EACzB,uCAAuC,EACvC,0BAA0B,GAC1B,MAAM,gCAAgC,CAAC;AACxC,mBAAmB,qCAAqC,CAAC;AACzD,mBAAmB,0CAA0C,CAAC;AAC9D,mBAAmB,yCAAyC,CAAC;AAC7D,mBAAmB,+CAA+C,CAAC;AACnE,YAAY,EACX,WAAW,EACX,4BAA4B,EAC5B,6BAA6B,EAC7B,oCAAoC,GACpC,MAAM,2CAA2C,CAAC;AAEnD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAC7D;;GAEG;AACH,eAAO,MAAM,qBAAqB,EAAE,MAAM,CAAC,YAAY,EAAE,eAAe,EAAE,CAkEzE,CAAC;AAoBF,eAAO,MAAM,UAAU,EAAE,MAAM,CAAC,YAAY,EAAE,QAAQ,GAAG,SAAS,CA0DxD,CAAC;AAEX,MAAM,WAAW,WAAW;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,EAAE,EAAE,MAAM,CAAC;CACX;AAED,MAAM,MAAM,aAAa,GACtB;IACA,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,OAAO,CAAC;CACb,GACD;IACA,IAAI,EAAE,KAAK,CAAC;QACX,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,MAAM,CAAC;KACd,CAAC,CAAC;IACH,IAAI,EAAE,OAAO,CAAC;CACb,GACD;IACA,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,KAAK,CAAC;CACX,GACD;IACA,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC;IAClB,IAAI,EAAE,SAAS,CAAC;CACf,GACD;IACA,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;CACZ,GACD;IACA,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,KAAK,CAAC;QACb,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;QACd,IAAI,EAAE,MAAM,CAAC;KACb,CAAC,CAAC;IACH,IAAI,EAAE,kBAAkB,CAAC;CACxB,CAAC;AAEL,MAAM,WAAW,QAAQ;IACxB,MAAM,EAAE,aAAa,EAAE,CAAC;IACxB,OAAO,EAAE,aAAa,EAAE,CAAC;CACzB;AAED,MAAM,WAAW,QAAQ;IACxB,QAAQ,EAAE,WAAW,EAAE,CAAC;IACxB,IAAI,EAAE,QAAQ,CAAC;IACf,EAAE,EAAE,YAAY,CAAC;IACjB,WAAW,CAAC,EAAE,YAAY,CAAC;IAC3B,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,eAAe,EAAE,CAAC;IAC7B,OAAO,EAAE,WAAW,EAAE,CAAC;IACvB,MAAM,EAAE,WAAW,EAAE,CAAC;IACtB,MAAM,EAAE,WAAW,EAAE,CAAC;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,SAAS,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,MAAM,cAAc,GAAG,IAAI,CAAC,QAAQ,EAAE,IAAI,GAAG,OAAO,GAAG,WAAW,CAAC,CAAC"}

package/dist/esm/tasks/index.js

@@ -12,6 +12,8 @@ import imageFeatureExtraction from "./image-feature-extraction/data.js";
 import imageToImage from "./image-to-image/data.js";
 import imageToText from "./image-to-text/data.js";
 import imageTextToText from "./image-text-to-text/data.js";
+import imageTextToImage from "./image-text-to-image/data.js";
+import imageTextToVideo from "./image-text-to-video/data.js";
 import imageSegmentation from "./image-segmentation/data.js";
 import imageToVideo from "./image-to-video/data.js";
 import maskGeneration from "./mask-generation/data.js";
@@ -62,6 +64,8 @@ export const TASKS_MODEL_LIBRARIES = {
     "image-feature-extraction": ["timm", "transformers"],
     "image-segmentation": ["transformers", "transformers.js"],
     "image-text-to-text": ["transformers"],
+    "image-text-to-image": ["diffusers"],
+    "image-text-to-video": ["diffusers"],
     "image-to-image": ["diffusers", "transformers", "transformers.js"],
     "image-to-text": ["transformers", "transformers.js"],
     "image-to-video": ["diffusers"],
@@ -147,6 +151,8 @@ export const TASKS_DATA = {
     "image-segmentation": getData("image-segmentation", imageSegmentation),
     "image-to-image": getData("image-to-image", imageToImage),
     "image-text-to-text": getData("image-text-to-text", imageTextToText),
+    "image-text-to-image": getData("image-text-to-image", imageTextToImage),
+    "image-text-to-video": getData("image-text-to-video", imageTextToVideo),
     "image-to-text": getData("image-to-text", imageToText),
     "image-to-video": getData("image-to-video", imageToVideo),
     "keypoint-detection": getData("keypoint-detection", keypointDetection),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@huggingface/tasks",
-  "version": "0.19.66",
+  "version": "0.19.68",
   "description": "List of ML tasks for huggingface.co/tasks",
   "repository": "https://github.com/huggingface/huggingface.js.git",
   "publishConfig": {

package/src/dataset-libraries.ts

@@ -83,6 +83,12 @@ export const DATASET_LIBRARIES_UI_ELEMENTS = {
 		repoUrl: "https://github.com/duckdb/duckdb",
 		docsUrl: "https://huggingface.co/docs/hub/datasets-duckdb",
 	},
+	datadesigner: {
+		prettyLabel: "NeMo Data Designer",
+		repoName: "datadesigner",
+		repoUrl: "https://github.com/NVIDIA-NeMo/DataDesigner",
+		docsUrl: "https://nvidia-nemo.github.io/DataDesigner/",
+	},
 } satisfies Record<string, DatasetLibraryUiElement>;
 
 /// List of the dataset libraries supported by the Hub

package/src/hardware.ts

@@ -66,6 +66,10 @@ export const SKUS = {
 				tflops: 30.29,
 				memory: [24],
 			},
+			GB10: {
+				tflops: 29.71,
+				memory: [128],
+			},
 			"RTX PRO 6000 WS": {
 				tflops: 126,
 				memory: [96],
@@ -308,7 +312,7 @@ export const SKUS = {
 			},
 			"RTX 3050 Mobile": {
 				tflops: 7.639,
-				memory: [6],
+				memory: [4, 6],
 			},
 			"RTX 2060": {
 				tflops: 12.9,

package/src/local-apps.ts

@@ -517,13 +517,6 @@ export const LOCAL_APPS = {
 			model.tags.includes("coreml") && model.tags.includes("joyfusion") && model.pipeline_tag === "text-to-image",
 		deeplink: (model) => new URL(`https://joyfusion.app/import_from_hf?repo_id=${model.id}`),
 	},
-	invoke: {
-		prettyLabel: "Invoke",
-		docsUrl: "https://github.com/invoke-ai/InvokeAI",
-		mainTask: "text-to-image",
-		displayOnModelPage: (model) => model.library_name === "diffusers" && model.pipeline_tag === "text-to-image",
-		deeplink: (model) => new URL(`https://models.invoke.ai/huggingface/${model.id}`),
-	},
 	ollama: {
 		prettyLabel: "Ollama",
 		docsUrl: "https://ollama.com",

package/src/model-libraries-snippets.ts

@@ -331,6 +331,19 @@ output = model.generate(text)
 sf.write("simple.mp3", output, 44100)`,
 ];
 
+export const dia2 = (model: ModelData): string[] => [
+	`from dia2 import Dia2, GenerationConfig, SamplingConfig
+
+dia = Dia2.from_repo("${model.id}", device="cuda", dtype="bfloat16")
+config = GenerationConfig(
+    cfg_scale=2.0,
+    audio=SamplingConfig(temperature=0.8, top_k=50),
+    use_cuda_graph=True,
+)
+result = dia.generate("[S1] Hello Dia2!", config=config, output_wav="hello.wav", verbose=True)
+`,
+];
+
 export const describe_anything = (model: ModelData): string[] => [
 	`# pip install git+https://github.com/NVlabs/describe-anything
 from huggingface_hub import snapshot_download

package/src/model-libraries.ts

@@ -187,6 +187,25 @@ export const MODEL_LIBRARIES_UI_ELEMENTS = {
 		filter: false,
 		countDownloads: `path:"boltzgen1_diverse.ckpt"`,
 	},
+	cancertathomev2: {
+		prettyLabel: "Cancer@HomeV2",
+		repoName: "Cancer@HomeV2",
+		repoUrl: "https://huggingface.co/OpenPeerAI/CancerAtHomeV2",
+		filter: false,
+		countDownloads: `path:"run.py"`,
+	},
+	cartesia_pytorch: {
+		prettyLabel: "Cartesia Pytorch",
+		repoName: "Cartesia Pytorch",
+		repoUrl: "https://github.com/cartesia-ai/cartesia_pytorch",
+		snippets: snippets.cartesia_pytorch,
+	},
+	cartesia_mlx: {
+		prettyLabel: "Cartesia MLX",
+		repoName: "Cartesia MLX",
+		repoUrl: "https://github.com/cartesia-ai/cartesia_mlx",
+		snippets: snippets.cartesia_mlx,
+	},
 	champ: {
 		prettyLabel: "Champ",
 		repoName: "Champ",
@@ -201,6 +220,13 @@ export const MODEL_LIBRARIES_UI_ELEMENTS = {
 		countDownloads: `path:"tokenizer.json"`,
 		filter: false,
 	},
+	chaossim: {
+		prettyLabel: "ChaosSIM",
+		repoName: "ChaosSIM",
+		repoUrl: "https://huggingface.co/OpenPeerAI/ChaosSIM/",
+		countDownloads: `path:"ChaosSim.nb"`,
+		filter: false,
+	},
 	chat_tts: {
 		prettyLabel: "ChatTTS",
 		repoName: "ChatTTS",
@@ -215,6 +241,20 @@ export const MODEL_LIBRARIES_UI_ELEMENTS = {
 		repoUrl: "https://github.com/amazon-science/chronos-forecasting",
 		snippets: snippets.chronos_forecasting,
 	},
+	clara: {
+		prettyLabel: "Clara",
+		repoName: "Clara",
+		filter: false,
+		repoUrl: "https://github.com/nvidia/clara",
+		countDownloads: `path_extension:"ckpt" OR path:"config.json"`,
+	},
+	clipscope: {
+		prettyLabel: "clipscope",
+		repoName: "clipscope",
+		repoUrl: "https://github.com/Lewington-pitsos/clipscope",
+		filter: false,
+		countDownloads: `path_extension:"pt"`,
+	},
 	"cloud-agents": {
 		prettyLabel: "Cloud Agents",
 		repoName: "Cloud Agents",
@@ -222,6 +262,20 @@ export const MODEL_LIBRARIES_UI_ELEMENTS = {
 		filter: false,
 		countDownloads: `path:"setup.py"`,
 	},
+	cosyvoice: {
+		prettyLabel: "CosyVoice",
+		repoName: "CosyVoice",
+		repoUrl: "https://github.com/FunAudioLLM/CosyVoice",
+		filter: false,
+		countDownloads: `path_extension:"onnx" OR path_extension:"pt"`,
+	},
+	cotracker: {
+		prettyLabel: "CoTracker",
+		repoName: "CoTracker",
+		repoUrl: "https://github.com/facebookresearch/co-tracker",
+		filter: false,
+		countDownloads: `path_extension:"pth"`,
+	},
 	colpali: {
 		prettyLabel: "ColPali",
 		repoName: "ColPali",
@@ -293,6 +347,13 @@ export const MODEL_LIBRARIES_UI_ELEMENTS = {
 		snippets: snippets.dia,
 		filter: false,
 	},
+	dia2: {
+		prettyLabel: "Dia2",
+		repoName: "Dia2",
+		repoUrl: "https://github.com/nari-labs/dia2",
+		snippets: snippets.dia2,
+		filter: false,
+	},
 	"diff-interpretation-tuning": {
 		prettyLabel: "Diff Interpretation Tuning",
 		repoName: "Diff Interpretation Tuning",
@@ -322,44 +383,18 @@ export const MODEL_LIBRARIES_UI_ELEMENTS = {
 		repoUrl: "https://github.com/argmaxinc/DiffusionKit",
 		snippets: snippets.diffusionkit,
 	},
+	"docking-at-home": {
+		prettyLabel: "Docking@Home",
+		repoName: "Docking@Home",
+		repoUrl: "https://huggingface.co/OpenPeerAI/DockingAtHOME",
+		filter: false,
+		countDownloads: `path:"setup.py"`,
+	},
 	doctr: {
 		prettyLabel: "docTR",
 		repoName: "doctr",
 		repoUrl: "https://github.com/mindee/doctr",
 	},
-	cartesia_pytorch: {
-		prettyLabel: "Cartesia Pytorch",
-		repoName: "Cartesia Pytorch",
-		repoUrl: "https://github.com/cartesia-ai/cartesia_pytorch",
-		snippets: snippets.cartesia_pytorch,
-	},
-	cartesia_mlx: {
-		prettyLabel: "Cartesia MLX",
-		repoName: "Cartesia MLX",
-		repoUrl: "https://github.com/cartesia-ai/cartesia_mlx",
-		snippets: snippets.cartesia_mlx,
-	},
-	clipscope: {
-		prettyLabel: "clipscope",
-		repoName: "clipscope",
-		repoUrl: "https://github.com/Lewington-pitsos/clipscope",
-		filter: false,
-		countDownloads: `path_extension:"pt"`,
-	},
-	cosyvoice: {
-		prettyLabel: "CosyVoice",
-		repoName: "CosyVoice",
-		repoUrl: "https://github.com/FunAudioLLM/CosyVoice",
-		filter: false,
-		countDownloads: `path_extension:"onnx" OR path_extension:"pt"`,
-	},
-	cotracker: {
-		prettyLabel: "CoTracker",
-		repoName: "CoTracker",
-		repoUrl: "https://github.com/facebookresearch/co-tracker",
-		filter: false,
-		countDownloads: `path_extension:"pth"`,
-	},
 	edsnlp: {
 		prettyLabel: "EDS-NLP",
 		repoName: "edsnlp",

package/src/pipelines.ts

@@ -557,6 +557,14 @@ export const PIPELINE_DATA = {
 		name: "Image-Text-to-Text",
 		modality: "multimodal",
 	},
+	"image-text-to-image": {
+		name: "Image-Text-to-Image",
+		modality: "multimodal",
+	},
+	"image-text-to-video": {
+		name: "Image-Text-to-Video",
+		modality: "multimodal",
+	},
 	"visual-question-answering": {
 		name: "Visual Question Answering",
 		subtasks: [

package/src/snippets/inputs.ts

@@ -94,6 +94,16 @@ const inputsImageToVideo = () => `{
     "prompt": "The cat starts to dance"
 }`;
 
+const inputsImageTextToImage = () => `{
+    "image": "cat.png",
+    "prompt": "Turn the cat into a tiger."
+}`;
+
+const inputsImageTextToVideo = () => `{
+    "image": "cat.png",
+    "prompt": "The cat starts to dance"
+}`;
+
 const inputsImageSegmentation = () => `"cats.jpg"`;
 
 const inputsObjectDetection = () => `"cats.jpg"`;
@@ -130,6 +140,8 @@ const modelInputSnippets: {
 	"image-to-text": inputsImageToText,
 	"image-to-image": inputsImageToImage,
 	"image-to-video": inputsImageToVideo,
+	"image-text-to-image": inputsImageTextToImage,
+	"image-text-to-video": inputsImageTextToVideo,
 	"image-segmentation": inputsImageSegmentation,
 	"object-detection": inputsObjectDetection,
 	"question-answering": inputsQuestionAnswering,

package/src/tasks/image-text-to-image/about.md

@@ -0,0 +1,73 @@
+## Use Cases
+
+### Instruction-based Image Editing
+
+Image-text-to-image models can be used to edit images based on natural language instructions. For example, you can provide an image of a summer landscape and the instruction "Make it winter, add snow" to generate a winter version of the same scene.
+
+### Style Transfer
+
+These models can apply artistic styles or transformations to images based on text descriptions. For instance, you can transform a photo into a painting style by providing prompts like "Make it look like a Van Gogh painting" or "Convert to watercolor style."
+
+### Image Variations
+
+Generate variations of an existing image by providing different text prompts. This is useful for creative workflows where you want to explore different versions of the same image with specific modifications.
+
+### Guided Image Generation
+
+Use a reference image along with text prompts to guide the generation process. This allows for more controlled image generation compared to text-to-image models alone, as the reference image provides structural guidance.
+
+### Image Inpainting and Outpainting
+
+Fill in missing or masked parts of an image based on text descriptions, or extend an image beyond its original boundaries with text-guided generation.
+
+## Task Variants
+
+### Instruction-based Editing
+
+Models that follow natural language instructions to edit images, which can perform complex edits like object removal, color changes, and compositional modifications.
+
+### Reference-guided Generation
+
+Models that use a reference image to guide the generation process while incorporating text prompts to control specific attributes or modifications.
+
+### Conditional Image-to-Image
+
+Models that perform specific transformations based on text conditions, such as changing weather conditions, time of day, or seasonal variations.
+
+## Inference
+
+You can use the Diffusers library to interact with image-text-to-image models.
+
+```python
+import torch
+from diffusers import Flux2Pipeline
+from diffusers.utils import load_image
+
+repo_id = "black-forest-labs/FLUX.2-dev"
+device = "cuda:0"
+torch_dtype = torch.bfloat16
+
+pipe = Flux2Pipeline.from_pretrained(
+    repo_id, torch_dtype=torch_dtype
+)
+pipe.enable_model_cpu_offload() #no need to do cpu offload for >80G VRAM carts like H200, B200, etc. and do a `pipe.to(device)` instead
+
+prompt = "Realistic macro photograph of a hermit crab using a soda can as its shell, partially emerging from the can, captured with sharp detail and natural colors, on a sunlit beach with soft shadows and a shallow depth of field, with blurred ocean waves in the background. The can has the text `BFL Diffusers` on it and it has a color gradient that start with #FF5733 at the top and transitions to #33FF57 at the bottom."
+
+#cat_image = load_image("https://huggingface.co/spaces/zerogpu-aoti/FLUX.1-Kontext-Dev-fp8-dynamic/resolve/main/cat.png")
+image = pipe(
+    prompt=prompt,
+    #image=[cat_image] #multi-image input
+    generator=torch.Generator(device=device).manual_seed(42),
+    num_inference_steps=50,
+    guidance_scale=4,
+).images[0]
+
+image.save("flux2_output.png")
+```
+
+## Useful Resources
+
+- [FLUX.2 Model Card](https://huggingface.co/black-forest-labs/FLUX.2-dev)
+- [Diffusers documentation on Image-to-Image](https://huggingface.co/docs/diffusers/using-diffusers/img2img)
+- [ControlNet for Conditional Image Generation](https://huggingface.co/docs/diffusers/using-diffusers/controlnet)

package/src/tasks/image-text-to-image/data.ts

@@ -0,0 +1,54 @@
+import type { TaskDataCustom } from "../index.js";
+
+const taskData: TaskDataCustom = {
+	datasets: [],
+	demo: {
+		inputs: [
+			{
+				filename: "image-text-to-image-input.jpeg",
+				type: "img",
+			},
+			{
+				label: "Input",
+				content: "A city above clouds, pastel colors, Victorian style",
+				type: "text",
+			},
+		],
+		outputs: [
+			{
+				filename: "image-text-to-image-output.png",
+				type: "img",
+			},
+		],
+	},
+	metrics: [
+		{
+			description:
+				"The Fréchet Inception Distance (FID) calculates the distance between distributions between synthetic and real samples. A lower FID score indicates better similarity between the distributions of real and generated images.",
+			id: "FID",
+		},
+		{
+			description:
+				"CLIP Score measures the similarity between the generated image and the text prompt using CLIP embeddings. A higher score indicates better alignment with the text prompt.",
+			id: "CLIP",
+		},
+	],
+	models: [
+		{
+			description: "A powerful model for image-text-to-image generation.",
+			id: "black-forest-labs/FLUX.2-dev",
+		},
+	],
+	spaces: [
+		{
+			description: "An application for image-text-to-image generation.",
+			id: "black-forest-labs/FLUX.2-dev",
+		},
+	],
+	summary:
+		"Image-text-to-image models take an image and a text prompt as input and generate a new image based on the reference image and text instructions. These models are useful for image editing, style transfer, image variations, and guided image generation tasks.",
+	widgetModels: ["black-forest-labs/FLUX.2-dev"],
+	youtubeId: undefined,
+};
+
+export default taskData;

package/src/tasks/image-text-to-image/inference.ts

@@ -0,0 +1,75 @@
+/**
+ * Inference code generated from the JSON schema spec in ./spec
+ *
+ * Using src/scripts/inference-codegen
+ */
+/**
+ * Inputs for Image Text To Image inference. Either inputs (image) or prompt (in parameters)
+ * must be provided, or both.
+ */
+export interface ImageTextToImageInput {
+	/**
+	 * The input image data as a base64-encoded string. If no `parameters` are provided, you can
+	 * also provide the image data as a raw bytes payload. Either this or prompt must be
+	 * provided.
+	 */
+	inputs?: Blob;
+	/**
+	 * Additional inference parameters for Image Text To Image
+	 */
+	parameters?: ImageTextToImageParameters;
+	[property: string]: unknown;
+}
+/**
+ * Additional inference parameters for Image Text To Image
+ */
+export interface ImageTextToImageParameters {
+	/**
+	 * For diffusion models. A higher guidance scale value encourages the model to generate
+	 * images closely linked to the text prompt at the expense of lower image quality.
+	 */
+	guidance_scale?: number;
+	/**
+	 * One prompt to guide what NOT to include in image generation.
+	 */
+	negative_prompt?: string;
+	/**
+	 * For diffusion models. The number of denoising steps. More denoising steps usually lead to
+	 * a higher quality image at the expense of slower inference.
+	 */
+	num_inference_steps?: number;
+	/**
+	 * The text prompt to guide the image generation. Either this or inputs (image) must be
+	 * provided.
+	 */
+	prompt?: string;
+	/**
+	 * Seed for the random number generator.
+	 */
+	seed?: number;
+	/**
+	 * The size in pixels of the output image. This parameter is only supported by some
+	 * providers and for specific models. It will be ignored when unsupported.
+	 */
+	target_size?: TargetSize;
+	[property: string]: unknown;
+}
+/**
+ * The size in pixels of the output image. This parameter is only supported by some
+ * providers and for specific models. It will be ignored when unsupported.
+ */
+export interface TargetSize {
+	height: number;
+	width: number;
+	[property: string]: unknown;
+}
+/**
+ * Outputs of inference for the Image Text To Image task
+ */
+export interface ImageTextToImageOutput {
+	/**
+	 * The generated image returned as raw bytes in the payload.
+	 */
+	image: unknown;
+	[property: string]: unknown;
+}

package/src/tasks/image-text-to-image/spec/input.json

@@ -0,0 +1,59 @@
+{
+	"$id": "/inference/schemas/image-text-to-image/input.json",
+	"$schema": "http://json-schema.org/draft-06/schema#",
+	"description": "Inputs for Image Text To Image inference. Either inputs (image) or prompt (in parameters) must be provided, or both.",
+	"title": "ImageTextToImageInput",
+	"type": "object",
+	"properties": {
+		"inputs": {
+			"type": "string",
+			"description": "The input image data as a base64-encoded string. If no `parameters` are provided, you can also provide the image data as a raw bytes payload. Either this or prompt must be provided.",
+			"comment": "type=binary"
+		},
+		"parameters": {
+			"description": "Additional inference parameters for Image Text To Image",
+			"$ref": "#/$defs/ImageTextToImageParameters"
+		}
+	},
+	"$defs": {
+		"ImageTextToImageParameters": {
+			"title": "ImageTextToImageParameters",
+			"type": "object",
+			"properties": {
+				"prompt": {
+					"type": "string",
+					"description": "The text prompt to guide the image generation. Either this or inputs (image) must be provided."
+				},
+				"guidance_scale": {
+					"type": "number",
+					"description": "For diffusion models. A higher guidance scale value encourages the model to generate images closely linked to the text prompt at the expense of lower image quality."
+				},
+				"negative_prompt": {
+					"type": "string",
+					"description": "One prompt to guide what NOT to include in image generation."
+				},
+				"num_inference_steps": {
+					"type": "integer",
+					"description": "For diffusion models. The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference."
+				},
+				"target_size": {
+					"type": "object",
+					"description": "The size in pixels of the output image. This parameter is only supported by some providers and for specific models. It will be ignored when unsupported.",
+					"properties": {
+						"width": {
+							"type": "integer"
+						},
+						"height": {
+							"type": "integer"
+						}
+					},
+					"required": ["width", "height"]
+				},
+				"seed": {
+					"type": "integer",
+					"description": "Seed for the random number generator."
+				}
+			}
+		}
+	}
+}