@framefields/node-vision 2.0.2

package/src/shared/config.ts

@@ -0,0 +1,88 @@
+import {
+	createOutputItemSchema,
+	SingleOutputGenericSchema,
+	VirtualMediaDataSchema,
+} from "@framefields/core";
+import {
+	ImageResultSchema,
+	MultiOutputGenericSchema,
+} from "@framefields/node-sdk";
+import { z } from "zod";
+
+/**
+ * What the node draws:
+ *  - passthrough — the child unchanged (vision signals still update)
+ *  - mask        — white subject silhouette on black
+ *  - matte       — the child cut out by the subject alpha (transparent background)
+ *  - crop        — the matte, punched in to the subject's bounds
+ *  - skeleton    — COCO-17 pose skeleton of the primary person
+ *  - boxes       — the child with tracked-object boxes
+ *  - tracking    — boxes plus track centers
+ */
+export const VISION_MODES = [
+	"passthrough",
+	"mask",
+	"matte",
+	"crop",
+	"skeleton",
+	"boxes",
+	"tracking",
+] as const;
+
+export type VisionMode = (typeof VISION_MODES)[number];
+
+export const VisionNodeConfigSchema = z
+	.object({
+		enableDetection: z.boolean().default(true),
+		enableSegmentation: z.boolean().default(false),
+		enablePose: z.boolean().default(false),
+		enableMatte: z.boolean().default(false),
+		classes: z.array(z.string()).optional(),
+		confidence: z.number().min(0).max(1).default(0.3),
+		variant: z.enum(["t", "s", "m"]).default("s"),
+		mode: z.enum(VISION_MODES).default("passthrough"),
+		/**
+		 * Source of the subject alpha for mask/matte/crop: `instance` (RTMDet-Ins, any COCO
+		 * class, overlapping parts merged) or `selfie` (Selfie Segmenter, people only, fastest).
+		 */
+		matteSource: z.enum(["instance", "selfie"]).default("instance"),
+		/** Frames a track may go unseen before it is dropped. */
+		maxMissedFrames: z.number().int().positive().default(15),
+		maskThreshold: z.number().min(0).max(1).default(0.5),
+		featherRadius: z.number().min(0).max(1).optional(),
+		/** Grow the subject into connected pixels that differ from the frame-border backdrop. */
+		keyBackground: z.boolean().default(false),
+		backgroundKeyThreshold: z.number().min(0).max(255).default(70),
+		modelsDir: z.string().optional(),
+		baseUrl: z.string().optional(),
+		visionBundle: z.custom<unknown>().optional(),
+	})
+	.strict();
+
+export type VisionNodeConfig = z.infer<typeof VisionNodeConfigSchema>;
+
+export const VisionOperationSchema = VisionNodeConfigSchema.extend({
+	op: z.literal("Vision"),
+	dataType: z.enum(["Image", "Video", "SVG", "GIF", "Caption"]).optional(),
+	inputs: z.record(z.string(), z.unknown()).optional(),
+	metadata: z.record(z.string(), z.unknown()).optional(),
+});
+
+export type VisionOperation = z.infer<typeof VisionOperationSchema>;
+
+export const ImageVisionResultSchema = ImageResultSchema;
+export type ImageVisionResult = z.infer<typeof ImageVisionResultSchema>;
+
+const VisualOutputSchema = z.union([
+	createOutputItemSchema(z.literal("Video"), VirtualMediaDataSchema),
+	createOutputItemSchema(z.literal("Image"), VirtualMediaDataSchema),
+	createOutputItemSchema(z.literal("SVG"), VirtualMediaDataSchema),
+	createOutputItemSchema(z.literal("GIF"), VirtualMediaDataSchema),
+]);
+
+export const VideoVisionResultSchema =
+	SingleOutputGenericSchema(VisualOutputSchema);
+export type VideoVisionResult = z.infer<typeof VideoVisionResultSchema>;
+
+export const VisionResultSchema = MultiOutputGenericSchema(VisualOutputSchema);
+export type VisionResult = z.infer<typeof VisionResultSchema>;

package/src/shared/index.ts

@@ -0,0 +1 @@
+export * from "./config.js";

package/tsconfig.json

@@ -0,0 +1,11 @@
+{
+	"extends": "@framefields/tsconfig/base.json",
+	"compilerOptions": {
+		"lib": ["ES2022", "ESNext", "DOM", "DOM.Iterable"],
+		"types": ["webgpu"],
+		"target": "esnext",
+		"noEmit": true
+	},
+	"include": ["src/**/*"],
+	"exclude": ["dist", "build", "node_modules"]
+}

package/tsdown.config.ts

@@ -0,0 +1,13 @@
+import { defineConfig } from "tsdown";
+
+export default defineConfig({
+	entry: {
+		index: "src/index.ts",
+		renderer: "src/renderers/index.ts",
+	},
+	format: ["esm"],
+	dts: true,
+	clean: true,
+	sourcemap: true,
+	treeshake: true,
+});