@foresthubai/workflow-core 0.4.3 → 0.4.4

package/src/deploy/requirements.ts

@@ -1,41 +1,156 @@
-import type { Workflow } from "../workflow";
-import { NodeRegistry, isNodeUsedAsTool } from "../node";
-import { isParameterActive } from "../parameter";
-
-/**
- * Model ids that nodes reference but the workflow does not declare in `models`.
- * A `modelSelect` accepts exactly two sources — declared custom models and the
- * static catalog — so any referenced id that isn't a declared model is a catalog
- * model: it carries no declared config, yet still needs a provider/credential
- * supplied at deploy.
- *
- * This is the one deploy demand the workflow's resource arrays can't express:
- * channels/memory/declared-models are enumerable directly from
- * `workflow.{channels,memory,models}`, but catalog model ids live only on the
- * nodes that pick them — hence the walk. Spans every canvas (main + function
- * bodies) and honors parameter activation, so a model behind an inactive
- * `modelSelect` (pruned on serialize, never deployed) is not counted.
- */
-export function getReferencedCatalogModelIds(workflow: Workflow): string[] {
-  const declaredModel = new Set(Object.keys(workflow.models));
-  const catalogIds = new Set<string>();
-
-  for (const canvas of Object.values(workflow.canvases)) {
-    for (const node of canvas.nodes) {
-      const def = NodeRegistry.getByType(node.type);
-      if (!def) continue;
-      const args = node.arguments as Record<string, unknown>;
-      const isToolInput = isNodeUsedAsTool(node.id, node, canvas.edges);
-      for (const param of def.parameters) {
-        if (param.type !== "modelSelect") continue;
-        if (!isParameterActive(param, args, isToolInput)) continue;
-        const id = args[param.id];
-        if (typeof id === "string" && id !== "" && !declaredModel.has(id)) {
-          catalogIds.add(id);
-        }
-      }
-    }
-  }
-
-  return [...catalogIds];
-}
+import type { Workflow } from "../workflow";
+import { NodeRegistry, isNodeUsedAsTool } from "../node";
+import { isParameterActive } from "../parameter";
+
+/**
+ * Model ids that nodes reference but the workflow does not declare in `models`.
+ * A `modelSelect` accepts exactly two sources — declared custom models and the
+ * static catalog — so any referenced id that isn't a declared model is a catalog
+ * model: it carries no declared config, yet still needs a provider/credential
+ * supplied at deploy.
+ *
+ * This is the one deploy demand the workflow's resource arrays can't express:
+ * channels/memory/declared-models are enumerable directly from
+ * `workflow.{channels,memory,models}`, but catalog model ids live only on the
+ * nodes that pick them — hence the walk. Spans every canvas (main + function
+ * bodies) and honors parameter activation, so a model behind an inactive
+ * `modelSelect` (pruned on serialize, never deployed) is not counted.
+ */
+export function getReferencedCatalogModelIds(workflow: Workflow): string[] {
+  const declaredModel = new Set(Object.keys(workflow.models));
+  const catalogIds = new Set<string>();
+
+  for (const canvas of Object.values(workflow.canvases)) {
+    for (const node of canvas.nodes) {
+      const def = NodeRegistry.getByType(node.type);
+      if (!def) continue;
+      const args = node.arguments as Record<string, unknown>;
+      const isToolInput = isNodeUsedAsTool(node.id, node, canvas.edges);
+      for (const param of def.parameters) {
+        if (param.type !== "modelSelect") continue;
+        if (!isParameterActive(param, args, isToolInput)) continue;
+        const id = args[param.id];
+        if (typeof id === "string" && id !== "" && !declaredModel.has(id)) {
+          catalogIds.add(id);
+        }
+      }
+    }
+  }
+
+  return [...catalogIds];
+}
+
+// The five hardware-channel families the engine has a driver for. UART is the
+// odd one out: it carries no per-channel sub-address (see `addressable`).
+export type HardwareFamily = "gpio" | "adc" | "dac" | "pwm" | "serial";
+
+// One hardware channel the workflow declares. `family` is derived from the
+// channel type; `addressable` is false only for serial/UART (every
+// gpio/adc/dac/pwm channel needs an `index` sub-address, UART does not).
+export interface HardwareChannel {
+  id: string;
+  label: string;
+  family: HardwareFamily;
+  addressable: boolean;
+}
+
+export interface MqttChannel {
+  id: string;
+  label: string;
+}
+
+// One custom/self-hosted model declared in workflow.models — needs an
+// ExternalResources provider entry (a device sidecar or a network endpoint).
+export interface CustomModel {
+  id: string;
+  label: string;
+}
+
+// What a workflow needs from its environment, derived from its content alone —
+// no operator input. Drives both input collection (which bindings to ask for)
+// and the spec resolver's completeness check. The component-set derivation the
+// migration doc names as must-not-be-duplicated lives here, shared by every
+// producer.
+export interface DeployRequirements {
+  // At least one Agent references a catalog model (not declared in
+  // workflow.models) — that model needs a provider key as engine env.
+  hasProviderModel: boolean;
+  // The workflow has a Retriever node. A standalone engine has no retriever, so
+  // the node cannot resolve — a producer may refuse to deploy.
+  hasRetriever: boolean;
+  // The workflow has a WebSearchTool node — needs a web-search key as engine env.
+  hasWebSearch: boolean;
+  // Every hardware channel, in declaration order; drives the device manifest +
+  // mapping + container device passthrough.
+  hardwareChannels: HardwareChannel[];
+  // Every MQTT channel; each becomes an ExternalResources entry + a mapping.
+  mqttChannels: MqttChannel[];
+  // Every declared custom model; each becomes an ExternalResources provider + a
+  // mapping (and a llama-server sidecar when bound on-device).
+  customModels: CustomModel[];
+}
+
+// Drift sentinel: a new ChannelType widens the switch input and breaks
+// compilation here until the new type is classified above.
+function assertNeverChannel(t: never): never {
+  throw new Error(`unhandled channel type: ${String(t)}`);
+}
+
+// deriveRequirements reads what a workflow demands of its environment. Pure —
+// no I/O, no operator input. Sorts every declared channel into the hardware /
+// MQTT pools the deploy artifacts need and walks nodes (main + function bodies)
+// for the catalog-model / retriever / web-search signals.
+export function deriveRequirements(workflow: Workflow): DeployRequirements {
+  const hardwareChannels: HardwareChannel[] = [];
+  const mqttChannels: MqttChannel[] = [];
+
+  for (const channel of Object.values(workflow.channels)) {
+    switch (channel.type) {
+      case "GPIOIN":
+      case "GPIOOUT":
+        hardwareChannels.push({ id: channel.id, label: channel.label, family: "gpio", addressable: true });
+        break;
+      case "ADC":
+        hardwareChannels.push({ id: channel.id, label: channel.label, family: "adc", addressable: true });
+        break;
+      case "DAC":
+        hardwareChannels.push({ id: channel.id, label: channel.label, family: "dac", addressable: true });
+        break;
+      case "PWM":
+        hardwareChannels.push({ id: channel.id, label: channel.label, family: "pwm", addressable: true });
+        break;
+      case "UART":
+        hardwareChannels.push({ id: channel.id, label: channel.label, family: "serial", addressable: false });
+        break;
+      case "MQTT":
+        mqttChannels.push({ id: channel.id, label: channel.label });
+        break;
+      case "LOG":
+        // Resolves to the ambient engine logger — no platform resource to bind, so
+        // it demands nothing of the deploy environment.
+        break;
+      default:
+        return assertNeverChannel(channel.type);
+    }
+  }
+
+  let hasRetriever = false;
+  let hasWebSearch = false;
+  for (const canvas of Object.values(workflow.canvases)) {
+    for (const node of canvas.nodes) {
+      if (node.type === "Retriever") hasRetriever = true;
+      else if (node.type === "WebSearchTool") hasWebSearch = true;
+    }
+  }
+
+  const customModels: CustomModel[] = Object.values(workflow.models).map((m) => ({ id: m.id, label: m.label }));
+
+  return {
+    hasProviderModel: getReferencedCatalogModelIds(workflow).length > 0,
+    hasRetriever,
+    hasWebSearch,
+    hardwareChannels,
+    mqttChannels,
+    customModels,
+  };
+}

package/src/deploy/spec.test.ts

@@ -0,0 +1,226 @@
+import { describe, it, expect } from "vitest";
+import { buildDeploymentSpec, assertDeployable, sidecarServiceName } from "./spec";
+import { deriveRequirements } from "./requirements";
+import type { DeploymentInputs } from "./inputs";
+import { MAIN_CANVAS_ID, type Workflow } from "../workflow";
+import type { Channel } from "../channel";
+import type { Model } from "../model";
+import type { DeploymentSchemas, EngineSchemas } from "../api";
+
+type Spec = DeploymentSchemas["DeploymentSpec"];
+
+// The engine is the component named "engine"; its config is transported as an
+// opaque blob, so cast it back to EngineConfig to inspect manifest/mapping/etc.
+function engineOf(spec: Spec): DeploymentSchemas["DeployComponent"] {
+  const e = spec.components.find((c) => c.name === "engine");
+  if (!e) throw new Error("expected engine component");
+  return e;
+}
+function engineConfigOf(spec: Spec): EngineSchemas["EngineConfig"] {
+  return engineOf(spec).config as EngineSchemas["EngineConfig"];
+}
+const llamaOf = (spec: Spec, modelId: string) => spec.components.find((c) => c.name === sidecarServiceName(modelId));
+
+function channel(id: string, type: Channel["type"], args: Record<string, unknown> = {}): Channel {
+  return { id, label: id, type, arguments: args };
+}
+
+const customModel: Model = { id: "local-llm", label: "Local", type: "LLMModel", arguments: {} };
+
+// One workflow exercising every resource kind: a cdev output (GPIO), a serial
+// device (cdev), a sysfs ADC (privileged), an MQTT channel, and an on-device
+// custom model. Empty main canvas — the resolver reads channels/models, not nodes.
+function fullWorkflow(): Workflow {
+  return {
+    canvases: { [MAIN_CANVAS_ID]: { nodes: [], edges: [], variables: {} } },
+    functions: {},
+    channels: {
+      led: channel("led", "GPIOOUT"),
+      serial0: channel("serial0", "UART"),
+      sensor: channel("sensor", "ADC"),
+      telemetry: channel("telemetry", "MQTT", { topic: "t/x" }),
+    },
+    memory: {},
+    models: { "local-llm": customModel },
+  };
+}
+
+const fullInputs: DeploymentInputs = {
+  hardware: {
+    led: { chipOrDevice: "/dev/gpiochip0", index: 17 },
+    serial0: { chipOrDevice: "/dev/ttyUSB0", baud: 9600 },
+    sensor: { chipOrDevice: "/sys/bus/iio/devices/iio:device0", index: 0 },
+  },
+  mqtt: { telemetry: { brokerUrl: "mqtt://broker.local:1883", username: "u", password: "p" } },
+  models: { "local-llm": { location: "device", modelFile: "model.gguf" } },
+};
+
+const meta = {
+  id: "dep-1",
+  status: "active" as const,
+  engineImage: "fh-engine:0.4.2",
+  llamaServerImage: "ghcr.io/ggml-org/llama.cpp:server-b8589",
+};
+
+describe("buildDeploymentSpec", () => {
+  it("resolves a full workflow into a contract-shaped spec", () => {
+    const { spec } = buildDeploymentSpec(fullWorkflow(), fullInputs, meta);
+
+    expect(spec.schemaVersion).toBe(1);
+    expect(spec.id).toBe("dep-1");
+    expect(spec.status).toBe("active");
+
+    const engine = engineOf(spec);
+    expect(engine.image).toBe("fh-engine:0.4.2");
+    expect(engine.pull).toBe("never"); // built locally, in no registry
+    expect(engineConfigOf(spec).workflow.schemaVersion).toBeGreaterThanOrEqual(1);
+  });
+
+  it("splits the device manifest by family and maps every resource", () => {
+    const config = engineConfigOf(buildDeploymentSpec(fullWorkflow(), fullInputs, meta).spec);
+    const m = config.manifest!;
+    expect(Object.keys(m.gpios ?? {})).toHaveLength(1);
+    expect(Object.keys(m.serials ?? {})).toHaveLength(1);
+    expect(Object.keys(m.adcs ?? {})).toHaveLength(1);
+
+    // Mapping keyed by every workflow logical id; each ref resolves to a built resource.
+    expect(Object.keys(config.mapping ?? {}).sort()).toEqual(["led", "local-llm", "sensor", "serial0", "telemetry"]);
+    expect(config.mapping!.sensor).toEqual({ ref: expect.any(String), index: 0 });
+  });
+
+  it("resolves cdev nodes into devices and forces privileged for sysfs families", () => {
+    const engine = engineOf(buildDeploymentSpec(fullWorkflow(), fullInputs, meta).spec);
+    // GPIO + serial are cdev → granted; ADC (sysfs) is not a node → privileged instead.
+    expect(engine.devices?.sort()).toEqual(["/dev/gpiochip0", "/dev/ttyUSB0"]);
+    expect(engine.privileged).toBe(true);
+    // A nonroot image reaching root-owned nodes/sysfs runs as root.
+    expect(engine.user).toBe("0:0");
+  });
+
+  it("emits a llama-server component and points the engine's provider at the sidecar", () => {
+    const { spec } = buildDeploymentSpec(fullWorkflow(), fullInputs, meta);
+    expect(llamaOf(spec, "local-llm")).toMatchObject({
+      name: sidecarServiceName("local-llm"),
+      image: "ghcr.io/ggml-org/llama.cpp:server-b8589",
+      command: ["--model", "/models/model.gguf", "--host", "0.0.0.0", "--port", "8080", "--ctx-size", "4096"],
+      volumes: ["./models:/models:ro"],
+    });
+    // The external-resource provider URL must point at the sidecar service name.
+    const ext = engineConfigOf(spec).externalResources!;
+    const provider = Object.values(ext).find((r) => r.type === "selfhosted");
+    expect(provider).toMatchObject({ url: `http://${sidecarServiceName("local-llm")}:8080` });
+  });
+
+  it("omits privileged and the llama sidecar when neither applies", () => {
+    const wf: Workflow = {
+      canvases: { [MAIN_CANVAS_ID]: { nodes: [], edges: [], variables: {} } },
+      functions: {},
+      channels: { led: channel("led", "GPIOOUT") },
+      memory: {},
+      models: {},
+    };
+    const { spec } = buildDeploymentSpec(wf, { hardware: { led: { chipOrDevice: "/dev/gpiochip0", index: 1 } }, mqtt: {}, models: {} }, meta);
+    const engine = engineOf(spec);
+    expect(engine.privileged).toBeUndefined();
+    expect(spec.components).toHaveLength(1); // engine only, no sidecar
+    expect(engine.devices).toEqual(["/dev/gpiochip0"]);
+  });
+
+  it("uses a network model's own endpoint and runs no sidecar", () => {
+    const wf: Workflow = {
+      canvases: { [MAIN_CANVAS_ID]: { nodes: [], edges: [], variables: {} } },
+      functions: {},
+      channels: {},
+      memory: {},
+      models: { "local-llm": customModel },
+    };
+    const inputs: DeploymentInputs = {
+      hardware: {},
+      mqtt: {},
+      models: { "local-llm": { location: "network", url: "https://infer.example/v1", apiKey: "k" } },
+    };
+    const { spec, resourceSecrets } = buildDeploymentSpec(wf, inputs, meta);
+    expect(spec.components).toHaveLength(1); // engine only, no sidecar
+    // The provider config in the spec is secret-free; the apiKey is pulled out.
+    const ext = engineConfigOf(spec).externalResources!;
+    const [ref, provider] = Object.entries(ext).find(([, r]) => r.type === "selfhosted")!;
+    expect(provider).toEqual({ type: "selfhosted", url: "https://infer.example/v1" });
+    expect(resourceSecrets[ref]).toEqual({ apiKey: "k" });
+  });
+
+  it("pulls MQTT/endpoint secrets out of the spec, keyed by resource ref", () => {
+    const { spec, resourceSecrets } = buildDeploymentSpec(fullWorkflow(), fullInputs, meta);
+    const ext = engineConfigOf(spec).externalResources!;
+    // The stored connection carries metadata but never the password.
+    const [mqttRef, mqttConn] = Object.entries(ext).find(([, r]) => r.type === "mqtt")!;
+    expect(mqttConn).not.toHaveProperty("password");
+    expect(mqttConn).toMatchObject({ username: "u" });
+    expect(resourceSecrets[mqttRef]).toEqual({ password: "p" });
+    // Whole spec serialized carries no secret value.
+    expect(JSON.stringify(spec)).not.toContain('"p"');
+  });
+});
+
+describe("buildDeploymentSpec custom components", () => {
+  const bareWf: Workflow = {
+    canvases: { [MAIN_CANVAS_ID]: { nodes: [], edges: [], variables: {} } },
+    functions: {},
+    channels: {},
+    memory: {},
+    models: {},
+  };
+  const bareInputs: DeploymentInputs = { hardware: {}, mqtt: {}, models: {} };
+
+  it("merges custom components after engine (and llama)", () => {
+    const grafana = { name: "grafana", image: "grafana/grafana:11.3.0", ports: ["3000:3000"] };
+    const { spec } = buildDeploymentSpec(bareWf, bareInputs, meta, [grafana]);
+    expect(spec.components.map((c) => c.name)).toEqual(["engine", "grafana"]);
+  });
+
+  it("rejects a custom name colliding with the engine", () => {
+    expect(() => buildDeploymentSpec(bareWf, bareInputs, meta, [{ name: "engine", image: "x" }])).toThrow(
+      /duplicate component name "engine"/,
+    );
+  });
+
+  it("rejects a custom name colliding with a generated llama sidecar", () => {
+    const dup = { name: sidecarServiceName("local-llm"), image: "x" };
+    expect(() => buildDeploymentSpec(fullWorkflow(), fullInputs, meta, [dup])).toThrow(/duplicate component name/);
+  });
+
+  it("allows the same image under different names", () => {
+    const a = { name: "graf-a", image: "grafana/grafana:11.3.0" };
+    const b = { name: "graf-b", image: "grafana/grafana:11.3.0" };
+    const { spec } = buildDeploymentSpec(bareWf, bareInputs, meta, [a, b]);
+    expect(spec.components.map((c) => c.name)).toEqual(["engine", "graf-a", "graf-b"]);
+  });
+
+  it("rejects two customs sharing a name", () => {
+    const a = { name: "dash", image: "x" };
+    const b = { name: "dash", image: "y" };
+    expect(() => buildDeploymentSpec(bareWf, bareInputs, meta, [a, b])).toThrow(/duplicate component name "dash"/);
+  });
+});
+
+describe("assertDeployable", () => {
+  it("throws listing every unbound resource", () => {
+    const req = deriveRequirements(fullWorkflow());
+    expect(() => assertDeployable(req, { hardware: {}, mqtt: {}, models: {} })).toThrow(/device path[\s\S]*broker URL[\s\S]*model/);
+  });
+
+  it("rejects a non-GGUF on-device model file", () => {
+    const req = deriveRequirements(fullWorkflow());
+    const inputs = { ...fullInputs, models: { "local-llm": { location: "device" as const, modelFile: "model.bin" } } };
+    expect(() => assertDeployable(req, inputs)).toThrow(/\.gguf/);
+  });
+});
+
+describe("deriveRequirements", () => {
+  it("classifies channels into hardware families and mqtt", () => {
+    const req = deriveRequirements(fullWorkflow());
+    expect(req.hardwareChannels.map((c) => c.family).sort()).toEqual(["adc", "gpio", "serial"]);
+    expect(req.mqttChannels.map((c) => c.id)).toEqual(["telemetry"]);
+    expect(req.customModels.map((c) => c.id)).toEqual(["local-llm"]);
+    expect(req.hardwareChannels.find((c) => c.family === "serial")?.addressable).toBe(false);
+  });
+});