@readymadeit/mcp 0.0.0-stage → 0.1.0

package/bundle/hkp-frontend/src/runtime/browser/services/Delay.tsx

@@ -0,0 +1,110 @@
+import { AppInstance, ServiceClass } from "hkp-frontend/src/types";
+import ServiceBase from "./ServiceBase";
+import {
+  convertToFloat32Array,
+  convertToUint8Array,
+  isFloatRingBuffer,
+} from "../../rest/Data";
+
+const serviceId = "hookup.to/service/delay";
+const serviceName = "Delay";
+
+type State = {
+  delayTimeMs: number;
+};
+
+class Delay extends ServiceBase<State> {
+  _delayProcessor: ReturnType<typeof DelayProcessor>;
+
+  constructor(
+    app: AppInstance,
+    board: string,
+    descriptor: ServiceClass,
+    id: string,
+  ) {
+    super(app, board, descriptor, id, {
+      delayTimeMs: 0.5, // Default delay time in seconds
+    });
+    this._delayProcessor = DelayProcessor(44100); // Assuming a sample rate of 44100 Hz
+  }
+
+  async configure(config: any) {
+    if (config.time) {
+      this._delayProcessor.setDelayTime(config.time * 1000); // Convert to milliseconds
+      this.state.delayTimeMs = config.time;
+    }
+    return config;
+  }
+
+  async process(params: any) {
+    if (isFloatRingBuffer(params)) {
+      const array: Uint8Array = params.array;
+      const floatArray = convertToFloat32Array(array);
+      const processedArray = floatArray.map((x) =>
+        this._delayProcessor.process(x),
+      );
+
+      return {
+        ...params,
+        array: convertToUint8Array(processedArray),
+      };
+    }
+    return params;
+  }
+}
+
+function DelayProcessor(sampleRate: number, maxDelayMs = 2000) {
+  const maxSamples = Math.floor((maxDelayMs / 1000) * sampleRate);
+  const buffer = new Float32Array(maxSamples);
+  let writeIndex = 0;
+
+  let delayTimeMs = 500;
+  let feedback = 0.5;
+  let mix = 0.5;
+
+  function setDelayTime(ms: number) {
+    delayTimeMs = Math.max(0, Math.min(ms, maxDelayMs));
+  }
+
+  function setFeedback(fb: number) {
+    feedback = Math.max(0, Math.min(fb, 0.999)); // Avoid runaway
+  }
+
+  function setMix(m: number) {
+    mix = Math.max(0, Math.min(m, 1));
+  }
+
+  function process(input: number): number {
+    const delaySamples = Math.floor((delayTimeMs / 1000) * sampleRate);
+    const readIndex = (writeIndex - delaySamples + maxSamples) % maxSamples;
+
+    const delayed = buffer[readIndex];
+    const output = input * (1 - mix) + delayed * mix;
+
+    buffer[writeIndex] = input + delayed * feedback;
+
+    writeIndex = (writeIndex + 1) % maxSamples;
+
+    return output;
+  }
+
+  return {
+    setDelayTime,
+    setFeedback,
+    setMix,
+    process,
+  };
+}
+
+const descriptor = {
+  serviceName,
+  serviceId,
+  create: (
+    app: AppInstance,
+    board: string,
+    descriptor: ServiceClass,
+    id: string,
+  ) => new Delay(app, board, descriptor, id),
+};
+
+export default descriptor;

package/bundle/hkp-frontend/src/runtime/browser/services/Detect.ts

@@ -0,0 +1,214 @@
+/**
+ * Service Documentation
+ * Service ID: hookup.to/service/detect
+ * Service Name: Detect
+ * Modes: face
+ * Key Config: mode, minConfidence
+ * IO: in=image (Blob, { image: Blob }, ImageBitmap, image/canvas/video element)
+ *     -> out={ ...input, faces: [{ score, box, keypoints }], count, frame }
+ */
+
+import { AppInstance, ServiceClass } from "hkp-frontend/src/types";
+import ServiceBase from "./ServiceBase";
+import DetectUI from "./DetectUI";
+import { DETECT_MODES, DetectMode } from "./detect-modes";
+import type { FaceDetector } from "./detect/mediapipe";
+
+const serviceId = "hookup.to/service/detect";
+const serviceName = "Detect";
+
+type State = {
+  mode: DetectMode;
+  /** Detections scoring below this are left out, 0–1. */
+  minConfidence: number;
+};
+
+export type Box = { x: number; y: number; width: number; height: number };
+export type Face = {
+  score: number;
+  /** In pixels of the frame. */
+  box: Box;
+  /** Eyes, nose, mouth and ears, in pixels of the frame. */
+  keypoints: Array<{ x: number; y: number }>;
+};
+export type Status = "idle" | "loading" | "ready" | "error";
+
+type Drawable = Exclude<TexImageSource, VideoFrame | OffscreenCanvas>;
+
+function isDrawable(value: unknown): value is Drawable {
+  return (
+    (typeof ImageBitmap !== "undefined" && value instanceof ImageBitmap) ||
+    (typeof HTMLImageElement !== "undefined" && value instanceof HTMLImageElement) ||
+    (typeof HTMLCanvasElement !== "undefined" && value instanceof HTMLCanvasElement) ||
+    (typeof HTMLVideoElement !== "undefined" && value instanceof HTMLVideoElement) ||
+    (typeof ImageData !== "undefined" && value instanceof ImageData)
+  );
+}
+
+/** The image in an input: the input itself, or its `image` field (what Camera sends). */
+export function imageOf(input: unknown): Blob | Drawable | null {
+  const candidate =
+    input instanceof Blob || isDrawable(input)
+      ? input
+      : input && typeof input === "object"
+        ? (input as { image?: unknown }).image
+        : undefined;
+  if (candidate instanceof Blob || isDrawable(candidate)) {
+    return candidate;
+  }
+  return null;
+}
+
+function sizeOf(image: Drawable): { width: number; height: number } {
+  if (typeof HTMLVideoElement !== "undefined" && image instanceof HTMLVideoElement) {
+    return { width: image.videoWidth, height: image.videoHeight };
+  }
+  if (typeof HTMLImageElement !== "undefined" && image instanceof HTMLImageElement) {
+    return { width: image.naturalWidth, height: image.naturalHeight };
+  }
+  return { width: image.width, height: image.height };
+}
+
+/**
+ * Finds things in an image — faces, for now — and reports what it found on
+ * every frame, none included. Whether a finding matters (a face that has just
+ * appeared, say) is the pipeline's to decide: Changes, Filter, Debounce.
+ *
+ * Runs MediaPipe Tasks Vision in the page, from files shipped with the app
+ * (`detect/mediapipe.ts`). The model is loaded when the service is first
+ * configured, so the first frame does not wait for it.
+ */
+class Detect extends ServiceBase<State> {
+  __detector: Promise<FaceDetector> | null = null;
+  __status: Status = "idle";
+  __confidence = 0.5;
+
+  constructor(app: AppInstance, board: string, descriptor: ServiceClass, id: string) {
+    super(app, board, descriptor, id, { mode: "face", minConfidence: 0.5 });
+  }
+
+  /** Live, not saved: what the panel shows before any notification. */
+  get status(): Status {
+    return this.__status;
+  }
+
+  private setStatus(status: Status, error?: string) {
+    this.__status = status;
+    this.app.notify(this, { status, ...(error ? { error } : {}) });
+  }
+
+  private detector(): Promise<FaceDetector> {
+    if (!this.__detector) {
+      this.setStatus("loading");
+      this.__confidence = this.state.minConfidence;
+      this.__detector = import("./detect/mediapipe")
+        .then(({ createFaceDetector }) => createFaceDetector(this.state.minConfidence))
+        .then((detector) => {
+          this.setStatus("ready");
+          return detector;
+        })
+        .catch((err) => {
+          // Forgotten, so the next frame tries again rather than failing for good.
+          this.__detector = null;
+          const message = err?.message ?? String(err);
+          this.setStatus("error", message);
+          this.pushErrorNotification(`Detect: could not load the model — ${message}`);
+          throw err;
+        });
+    }
+    return this.__detector;
+  }
+
+  configure(config: Partial<State>) {
+    if (config.mode !== undefined && DETECT_MODES.includes(config.mode)) {
+      this.state.mode = config.mode;
+      this.app.notify(this, { mode: config.mode });
+    }
+    if (config.minConfidence !== undefined) {
+      const value = Math.min(1, Math.max(0, Number(config.minConfidence)));
+      if (Number.isFinite(value)) {
+        this.state.minConfidence = value;
+        this.app.notify(this, { minConfidence: value });
+      }
+    }
+    // Loading starts with the first configure — a board restore configures
+    // every service — so the model is there by the first frame. Not awaited:
+    // a restore waits for each configure, and would wait for the model too.
+    void this.applyOptions();
+  }
+
+  private async applyOptions() {
+    const detector = await this.detector().catch(() => null);
+    if (detector && this.__confidence !== this.state.minConfidence) {
+      this.__confidence = this.state.minConfidence;
+      await detector.setOptions({ minDetectionConfidence: this.state.minConfidence });
+    }
+  }
+
+  async process(params: any): Promise<any> {
+    const image = imageOf(params);
+    if (!image) {
+      // Nothing to look at is not a frame without faces: say so, pass nothing.
+      this.app.notify(this, { error: "No image in the input" });
+      return null;
+    }
+    let detector: FaceDetector;
+    try {
+      detector = await this.detector();
+    } catch {
+      return null;
+    }
+
+    let bitmap: ImageBitmap | null = null;
+    try {
+      // Inside the try: a Blob that is not an image a browser can decode is a
+      // frame this service reports it cannot read, not a failed pipeline.
+      bitmap = image instanceof Blob ? await createImageBitmap(image) : null;
+      const source = bitmap ?? (image as Drawable);
+      const frame = sizeOf(source);
+      const result = detector.detect(source);
+      const faces: Face[] = result.detections.map((detection) => ({
+        score: detection.categories[0]?.score ?? 0,
+        box: {
+          x: detection.boundingBox?.originX ?? 0,
+          y: detection.boundingBox?.originY ?? 0,
+          width: detection.boundingBox?.width ?? 0,
+          height: detection.boundingBox?.height ?? 0,
+        },
+        keypoints: detection.keypoints.map((point) => ({
+          x: point.x * frame.width,
+          y: point.y * frame.height,
+        })),
+      }));
+      this.app.notify(this, {
+        error: null,
+        count: faces.length,
+        found: { faces, frame, image: image instanceof Blob ? image : null },
+      });
+      const carried =
+        params && typeof params === "object" && !(params instanceof Blob) && !isDrawable(params)
+          ? params
+          : { image: params };
+      return { ...carried, faces, count: faces.length, frame };
+    } catch (err: any) {
+      this.pushErrorNotification(`Detect: ${err?.message ?? err}`);
+      return null;
+    } finally {
+      bitmap?.close();
+    }
+  }
+
+  destroy() {
+    const pending = this.__detector;
+    this.__detector = null;
+    void pending?.then((detector) => detector.close()).catch(() => {});
+  }
+}
+
+export default {
+  serviceName,
+  serviceId,
+  create: (app: AppInstance, board: string, descriptor: ServiceClass, id: string) =>
+    new Detect(app, board, descriptor, id),
+  createUI: DetectUI,
+};

package/bundle/hkp-frontend/src/runtime/browser/services/DetectUI.tsx

@@ -0,0 +1,86 @@
+import { useState } from "react";
+
+import { ServiceUIProps } from "hkp-frontend/src/types";
+import ServiceUI from "hkp-frontend/src/ui-components/service/ServiceUI";
+import PillRadioGroup from "hkp-frontend/src/ui-components/PillRadioGroup";
+import Slider from "hkp-frontend/src/ui-components/Slider";
+import { DETECT_LABELS, DetectMode } from "./detect-modes";
+import type { Status } from "./Detect";
+import DetectionPreview, { Found } from "./detect/DetectionPreview";
+
+const PREVIEW_WIDTH = 260;
+
+const STATUS_TEXT: { [status in Status]: string } = {
+  idle: "Model not loaded",
+  loading: "Loading model…",
+  ready: "Ready",
+  error: "Model failed to load",
+};
+
+export default function DetectUI(props: ServiceUIProps) {
+  const { service } = props;
+  const [mode, setMode] = useState<DetectMode>("face");
+  const [minConfidence, setMinConfidence] = useState(0.5);
+  const [status, setStatus] = useState<Status>(
+    () => (service as any).status ?? "idle",
+  );
+  const [error, setError] = useState<string | null>(null);
+  const [found, setFound] = useState<Found | null>(null);
+
+  const update = (state: any) => {
+    if (state.mode !== undefined) {
+      setMode(state.mode);
+    }
+    if (state.minConfidence !== undefined) {
+      setMinConfidence(state.minConfidence);
+    }
+    if (state.status !== undefined) {
+      setStatus(state.status);
+    }
+    if (state.error !== undefined) {
+      setError(state.error);
+    }
+    if (state.found !== undefined) {
+      setFound(state.found);
+    }
+  };
+
+  return (
+    <ServiceUI
+      {...props}
+      onInit={update}
+      onNotification={update}
+      initialSize={{ width: 290, height: undefined }}
+    >
+      <div className="flex flex-col gap-2" style={{ textAlign: "left" }}>
+        <PillRadioGroup
+          title="Find"
+          options={DETECT_LABELS}
+          value={mode}
+          onChange={(next) => service.configure({ mode: next })}
+        />
+        <Slider
+          title="Min confidence"
+          value={minConfidence}
+          min={0}
+          max={1}
+          step={0.05}
+          onChange={(value: number) => {
+            setMinConfidence(value);
+            service.configure({ minConfidence: value });
+          }}
+        />
+        <div className="flex justify-between text-xs opacity-70">
+          <span>{STATUS_TEXT[status]}</span>
+          {found && (
+            <span>
+              {found.faces.length} {found.faces.length === 1 ? "face" : "faces"}
+            </span>
+          )}
+        </div>
+        {error && <div className="text-xs text-red-600 break-words">{error}</div>}
+        {found && <DetectionPreview found={found} width={PREVIEW_WIDTH} />}
+      </div>
+    </ServiceUI>
+  );
+}

package/bundle/hkp-frontend/src/runtime/browser/services/Download.ts

@@ -0,0 +1,120 @@
+/**
+ * Service Documentation
+ * Service ID: hookup.to/service/download
+ * Service Name: Download
+ * Runtime: browser
+ * Key Config: filename, mimeType
+ * IO: in=string | Uint8Array | ArrayBuffer | JSON -> out=the same input
+ *
+ * Saves whatever reaches it as a file, through the browser's own download:
+ * text as it is, bytes as they are, anything else as its JSON. The input
+ * travels on unchanged, so a download can sit in the middle of a pipeline as
+ * well as at its end.
+ *
+ * `filename` is what the file is offered as; a board that saves several
+ * different things configures it before each one (e.g. `{{item.name}}.sql`).
+ * `mimeType` left empty is decided by the input: text/plain for text,
+ * application/octet-stream for bytes, application/json for anything else.
+ */
+import { AppInstance, ServiceClass } from "hkp-frontend/src/types";
+import { needsUpdate } from "hkp-frontend/src/ui-components/service/ServiceUI";
+import ServiceBase from "./ServiceBase";
+import DownloadUI from "./DownloadUI";
+
+const serviceId = "hookup.to/service/download";
+const serviceName = "Download";
+
+type State = {
+  filename: string;
+  mimeType: string;
+};
+
+/** The input as a file's content, with the type it would be saved as. */
+function fileContent(
+  input: unknown,
+  mimeType: string,
+): { part: BlobPart; type: string } {
+  if (typeof input === "string") {
+    return { part: input, type: mimeType || "text/plain;charset=utf-8" };
+  }
+  if (input instanceof Uint8Array || input instanceof ArrayBuffer) {
+    return {
+      part: input as BlobPart,
+      type: mimeType || "application/octet-stream",
+    };
+  }
+  return {
+    part: JSON.stringify(input, null, 2),
+    type: mimeType || "application/json",
+  };
+}
+
+class Download extends ServiceBase<State> {
+  constructor(
+    app: AppInstance,
+    board: string,
+    descriptor: ServiceClass,
+    id: string,
+  ) {
+    super(app, board, descriptor, id, { filename: "download.txt", mimeType: "" });
+  }
+
+  configure(config: any): void {
+    if (
+      typeof config.filename === "string" &&
+      needsUpdate(config.filename, this.state.filename)
+    ) {
+      this.state.filename = config.filename;
+      this.app.notify(this, { filename: this.state.filename });
+    }
+    if (
+      typeof config.mimeType === "string" &&
+      needsUpdate(config.mimeType, this.state.mimeType)
+    ) {
+      this.state.mimeType = config.mimeType;
+      this.app.notify(this, { mimeType: this.state.mimeType });
+    }
+  }
+
+  process(input: any): any {
+    if (input === undefined || input === null) {
+      return input;
+    }
+    const filename = this.state.filename.trim() || "download.txt";
+    const { part, type } = fileContent(input, this.state.mimeType);
+    const blob = new Blob([part], { type });
+    try {
+      const url = URL.createObjectURL(blob);
+      const link = document.createElement("a");
+      link.href = url;
+      link.download = filename;
+      link.style.display = "none";
+      document.body.appendChild(link);
+      link.click();
+      link.remove();
+      // The download has started from the link; the URL is only needed until
+      // the browser has read it.
+      setTimeout(() => URL.revokeObjectURL(url), 1000);
+      this.app.notify(this, { saved: filename, bytes: blob.size, error: "" });
+    } catch (err) {
+      const error = `could not save ${filename}: ${
+        err instanceof Error ? err.message : String(err)
+      }`;
+      this.app.log(this, "error", "service.failed", { message: error });
+      this.app.notify(this, { error });
+    }
+    return input;
+  }
+}
+
+export default {
+  serviceName,
+  serviceId,
+  create: (
+    app: AppInstance,
+    board: string,
+    descriptor: ServiceClass,
+    id: string,
+  ) => new Download(app, board, descriptor, id),
+  createUI: DownloadUI,
+};

package/bundle/hkp-frontend/src/runtime/browser/services/DownloadUI.tsx

@@ -0,0 +1,61 @@
+import { useState } from "react";
+
+import { ServiceUIProps } from "hkp-frontend/src/types";
+import ServiceUI, {
+  needsUpdate,
+} from "hkp-frontend/src/ui-components/service/ServiceUI";
+import InputField from "hkp-frontend/src/components/shared/InputField";
+
+/**
+ * The Download service's panel: what the file is called, what type it is
+ * offered as, and what was saved last.
+ */
+export default function DownloadUI(props: ServiceUIProps) {
+  const [filename, setFilename] = useState("");
+  const [mimeType, setMimeType] = useState("");
+  const [saved, setSaved] = useState("");
+  const [error, setError] = useState("");
+
+  const update = (config: any) => {
+    if (config?.filename !== undefined && needsUpdate(config.filename, filename)) {
+      setFilename(config.filename);
+    }
+    if (config?.mimeType !== undefined && needsUpdate(config.mimeType, mimeType)) {
+      setMimeType(config.mimeType);
+    }
+    if (typeof config?.saved === "string") {
+      setSaved(`saved ${config.saved} (${config.bytes} bytes)`);
+    }
+    if (typeof config?.error === "string") {
+      setError(config.error);
+    }
+  };
+
+  const configure = (config: any) => props.service.configure(config);
+
+  return (
+    <ServiceUI {...props} onInit={update} onNotification={update}>
+      <div
+        className="flex flex-col my-2 gap-2"
+        style={{ minWidth: 240, textAlign: "left" }}
+      >
+        <InputField
+          value={filename}
+          label="Filename"
+          onChange={(value) => configure({ filename: value })}
+        />
+        <InputField
+          value={mimeType}
+          label="Type"
+          onChange={(value) => configure({ mimeType: value })}
+        />
+        {saved || error ? (
+          <div className="text-xs">
+            <span className="opacity-60">{saved}</span>
+            {error ? <span style={{ color: "#ef4444" }}> {error}</span> : null}
+          </div>
+        ) : null}
+      </div>
+    </ServiceUI>
+  );
+}