@signalcore/ui 0.1.0 → 1.0.0-beta.1

package/src/useDictation.ts

@@ -0,0 +1,83 @@
+import { useCallback, useEffect, useRef, useState } from "react";
+
+/** Turns a recording into words: the host's (its server's speech to text). */
+export type Dictate = (audio: Blob) => Promise<string>;
+
+/** Where a dictation is: nothing, the microphone open, the words on their way, or refused. */
+export type Dictation = "idle" | "listening" | "writing" | "failed";
+
+/**
+ * Dictation in the composer: a press opens the microphone, the next one closes it and hands the
+ * recording to the host's `dictate`, whose words go to `onWords`. Without `dictate`, or in a
+ * browser that cannot record, it is not `available`. Leaving mid-recording stops the microphone.
+ */
+export function useDictation(
+  dictate: Dictate | undefined,
+  onWords: (words: string) => void,
+): { available: boolean; state: Dictation; toggle: () => void } {
+  const [state, setState] = useState<Dictation>("idle");
+  const recorder = useRef<MediaRecorder | null>(null);
+  const said = useRef(onWords);
+  said.current = onWords;
+  const available =
+    dictate !== undefined &&
+    typeof MediaRecorder !== "undefined" &&
+    typeof navigator !== "undefined" &&
+    navigator.mediaDevices !== undefined;
+
+  useEffect(
+    () => () => {
+      const open = recorder.current;
+      recorder.current = null;
+      if (open !== null && open.state !== "inactive") {
+        open.onstop = null;
+        open.stop();
+        open.stream.getTracks().forEach((track) => {
+          track.stop();
+        });
+      }
+    },
+    [],
+  );
+
+  const start = useCallback(async () => {
+    if (dictate === undefined) return;
+    let stream: MediaStream;
+    try {
+      stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+    } catch {
+      setState("failed");
+      return;
+    }
+    const open = new MediaRecorder(stream);
+    const chunks: Blob[] = [];
+    open.ondataavailable = (event) => {
+      if (event.data.size > 0) chunks.push(event.data);
+    };
+    open.onstop = () => {
+      stream.getTracks().forEach((track) => {
+        track.stop();
+      });
+      recorder.current = null;
+      setState("writing");
+      dictate(new Blob(chunks, { type: open.mimeType || "audio/webm" }))
+        .then((words) => {
+          if (words !== "") said.current(words);
+          setState("idle");
+        })
+        .catch(() => {
+          setState("failed");
+        });
+    };
+    recorder.current = open;
+    open.start();
+    setState("listening");
+  }, [dictate]);
+
+  const toggle = useCallback(() => {
+    if (state === "listening") recorder.current?.stop();
+    else if (state === "idle" || state === "failed") void start();
+  }, [state, start]);
+
+  return { available, state, toggle };
+}

package/src/useLive.ts

@@ -0,0 +1,115 @@
+import { useEffect, useEffectEvent, useRef, useState } from "react";
+
+import type { Client } from "./client";
+import { frameOf, type Frame, type UpFrame } from "./frames";
+
+export interface LiveHandlers {
+  /** Every frame the socket sends, as it comes. */
+  onFrame: (frame: Frame) => void;
+  /** The socket is open; `again` when it had been before, so what was stored meanwhile can be read. */
+  onOpen?: ((again: boolean) => void) | undefined;
+}
+
+/** A conversation's socket: whether it is open, and the way to speak up it. */
+export interface Live {
+  open: boolean;
+  /** Sends a frame up; false when the socket is not open. */
+  send: (frame: UpFrame) => boolean;
+}
+
+/** How long to wait before trying again after the nth failure in a row: 1s, 2s, 4s, 8s, then 10s. */
+export function retryDelay(failures: number): number {
+  return Math.min(1000 * 2 ** failures, 10_000);
+}
+
+/**
+ * The socket of conversation `conversationId`, held for as long as the component lives: frames
+ * down to `onFrame`, `send` up. A turn runs on the server apart from the socket, so a socket that
+ * drops opens again, waiting longer each time; `onOpen` then says it is not the first time, and a
+ * turn still running is told again from its start. A socket that could not open may have carried
+ * a token that expired: the next try asks the client for a fresh one, once until one opens. The
+ * raw layer: what a host draws its own things over.
+ */
+export function useLive(
+  client: Client,
+  conversationId: string,
+  { onFrame, onOpen }: LiveHandlers,
+): Live {
+  const [open, setOpen] = useState(false);
+  const socket = useRef<WebSocket | null>(null);
+  // The handlers as the host last gave them, without the socket opening again for each render.
+  const frame = useEffectEvent(onFrame);
+  const opened = useEffectEvent((again: boolean) => onOpen?.(again));
+
+  useEffect(() => {
+    let current: WebSocket | null = null;
+    let failures = 0;
+    let before = false;
+    let stopped = false;
+    // Whether the next try asks for a fresh credential: once after a socket that never opened.
+    let fresh = false;
+    let renewed = false;
+    let retry: ReturnType<typeof setTimeout> | undefined;
+
+    const again = () => {
+      setOpen(false);
+      retry = setTimeout(connect, retryDelay(failures));
+      failures += 1;
+    };
+
+    const open = (url: string) => {
+      const live = new WebSocket(url);
+      let opens = false;
+      current = live;
+      socket.current = live;
+      live.onopen = () => {
+        opens = true;
+        failures = 0;
+        renewed = false;
+        setOpen(true);
+        opened(before);
+        before = true;
+      };
+      live.onmessage = (event: MessageEvent) => {
+        const next = frameOf(event.data);
+        if (next) frame(next);
+      };
+      live.onclose = () => {
+        if (stopped) return;
+        fresh = !opens && !renewed;
+        again();
+      };
+    };
+
+    function connect() {
+      renewed ||= fresh;
+      client.liveUrl(conversationId, { fresh }).then(
+        (url) => {
+          if (!stopped) open(url);
+        },
+        // No address (the host's backend gave no token): tried again like a socket that dropped.
+        () => {
+          if (!stopped) again();
+        },
+      );
+      fresh = false;
+    }
+
+    connect();
+    return () => {
+      stopped = true;
+      clearTimeout(retry);
+      socket.current = null;
+      current?.close();
+    };
+  }, [client, conversationId]);
+
+  const send = (up: UpFrame): boolean => {
+    const live = socket.current;
+    if (live?.readyState !== WebSocket.OPEN) return false;
+    live.send(JSON.stringify(up));
+    return true;
+  };
+
+  return { open, send };
+}

package/src/usePolled.ts

@@ -0,0 +1,60 @@
+import { useEffect, useRef, useState } from "react";
+
+import { ApiError } from "./client";
+
+/** What a read gave: the value once it came, why it failed, or that it is no longer there. */
+export interface Polled<T> {
+  value: T | null;
+  problem: unknown;
+  gone: boolean;
+}
+
+/** How often something still moving (a run under way) is read again. */
+export const POLL_MS = 3000;
+
+/**
+ * Reads `load` when `key` changes, and again every `POLL_MS` while `moving` says what it gave is
+ * still moving (a run under way). What was read stays while a new read is on its way, so nothing
+ * flickers; a 404 reads as gone.
+ */
+export function usePolled<T>(
+  load: () => Promise<T>,
+  key: string,
+  moving: (value: T) => boolean,
+): Polled<T> {
+  const [polled, setPolled] = useState<Polled<T> & { key: string }>({
+    key,
+    value: null,
+    problem: null,
+    gone: false,
+  });
+  const latest = useRef({ load, moving });
+  useEffect(() => {
+    latest.current = { load, moving };
+  });
+
+  useEffect(() => {
+    let stopped = false;
+    let timer: ReturnType<typeof setTimeout> | undefined;
+    const read = async () => {
+      try {
+        const value = await latest.current.load();
+        if (stopped) return;
+        setPolled({ key, value, problem: null, gone: false });
+        if (latest.current.moving(value)) timer = setTimeout(() => void read(), POLL_MS);
+      } catch (error) {
+        if (stopped) return;
+        const gone = error instanceof ApiError && error.status === 404;
+        setPolled((last) => ({ ...last, key, problem: gone ? null : error, gone }));
+      }
+    };
+    void read();
+    return () => {
+      stopped = true;
+      clearTimeout(timer);
+    };
+  }, [key]);
+
+  // Another key's value is not this one's: until its own comes, nothing is shown.
+  return polled.key === key ? polled : { value: null, problem: null, gone: false };
+}

package/src/voice/Orb.tsx

@@ -0,0 +1,45 @@
+import { lazy, Suspense, type ComponentType } from "react";
+
+import { joined, useParts } from "../kit/context";
+import { DEFAULT_ORB, ORB_SIZE, type OrbProps, type OrbStyle } from "./orbs";
+
+/**
+ * Each style is a module of its own, loaded when an orb of that style is first drawn: a host that
+ * never calls loads none of them, and one that does loads the one it uses (a fragment shader
+ * each, and the little WebGL they share).
+ */
+const STYLES: Readonly<Record<OrbStyle, ComponentType<OrbProps>>> = {
+  wave: lazy(() => import("./wave")),
+  "fluid-dots": lazy(() => import("./fluidDots")),
+};
+
+/**
+ * The orb of a voice conversation with an agent: a shape that shows where it stands and how loud
+ * it is, in one of two styles (`wave`, the default; `fluid-dots`). Both follow the level while
+ * listening and speaking, rest when nothing happens, and stand still under
+ * `prefers-reduced-motion`, one frame per state. They are shaders: each paints its own colours,
+ * on a transparent canvas. Where WebGL is missing or lost, a still disc in a ring takes its place.
+ */
+export function Orb({ style = DEFAULT_ORB, ...props }: OrbProps) {
+  // A style the package does not have (a host's data, say) is drawn as the default.
+  const Style = STYLES[style] ?? STYLES[DEFAULT_ORB];
+  return (
+    <Suspense fallback={<Waiting {...props} />}>
+      <Style {...props} />
+    </Suspense>
+  );
+}
+
+/** What stands where the orb will be while its style loads: its box, named, with nothing in it. */
+function Waiting({ state, size = ORB_SIZE, label, className }: Omit<OrbProps, "style">) {
+  const part = useParts();
+  return (
+    <div
+      role="img"
+      aria-label={label}
+      part="orb"
+      className={joined(part("sc-orb", `sc-orb--${state}`, "sc-orb--loading"), className)}
+      style={{ width: size, height: size }}
+    />
+  );
+}

package/src/voice/StillOrb.tsx

@@ -0,0 +1,29 @@
+import { joined, useParts } from "../kit/context";
+import { ORB_SIZE, type OrbProps } from "./orbs";
+
+/**
+ * An orb that does not move: a disc inside a ring, which says the state by its outline (thin at
+ * rest, solid while listening, dashed while thinking, double while speaking) and by the words
+ * around it, never by movement. It is what the shader styles are where WebGL is not there (no
+ * context, a shader that does not compile, a context the browser took away).
+ */
+export function StillOrb({
+  state,
+  size = ORB_SIZE,
+  label,
+  className,
+}: Pick<OrbProps, "state" | "size" | "label" | "className">) {
+  const part = useParts();
+  return (
+    <div
+      role="img"
+      aria-label={label}
+      part="orb"
+      className={joined(part("sc-orb", `sc-orb--${state}`, "sc-orb--still"), className)}
+      style={{ width: size, height: size }}
+    >
+      <span aria-hidden className={part("sc-orb__ring")} />
+      <span aria-hidden className={part("sc-orb__core")} />
+    </div>
+  );
+}

package/src/voice/WebGLOrb.tsx

@@ -0,0 +1,98 @@
+import { useEffect, useRef, useState } from "react";
+
+import { joined, useParts } from "../kit/context";
+import { animate, drawStill, motionOf, type Motion } from "./motion";
+import { ORB_SIZE, type OrbProps } from "./orbs";
+import { StillOrb } from "./StillOrb";
+import { useReducedMotion } from "./useReducedMotion";
+import { createRenderer, inkOf, type Renderer, type Shader } from "./webgl";
+
+/** The canvas is drawn at this fraction of its size and scaled up by the browser: the shaders are soft. */
+const RENDER_SCALE = 0.75;
+
+/** The level as it is now: given, or read off its meter. */
+const read = (level: OrbProps["level"]): number => (typeof level === "function" ? level() : level);
+
+type WebGLOrbProps = Omit<OrbProps, "style"> & { shader: Shader };
+
+/**
+ * The body both shader styles share: a box that carries the orb's name and a canvas the shader
+ * paints on, frame by frame, from refs. Nothing here renders again in React for a frame: the level
+ * and the state are read by the loop (`animate`) when a frame is due.
+ *
+ * The loop runs only while something moves. It asks for no frame while the tab is hidden, and
+ * stops three seconds after the state became `idle`; a new state, the tab coming back or a new
+ * size starts it again. Under `prefers-reduced-motion` there is no loop at all: one frame per
+ * state, at a fixed moment. Where WebGL is missing, the shader does not compile or the browser
+ * takes the context away, the orb is the still one, with the same name. A host's colour
+ * (`--sc-orb-color`) is read off the canvas with each state, and the shader paints in it.
+ */
+export function WebGLOrb({
+  shader,
+  state,
+  level,
+  size = ORB_SIZE,
+  label,
+  className,
+}: WebGLOrbProps) {
+  const part = useParts();
+  const still = useReducedMotion();
+  const [broken, setBroken] = useState(false);
+  const surface = useRef<HTMLCanvasElement>(null);
+  const renderer = useRef<Renderer | undefined>(undefined);
+  const loud = useRef(level);
+  const motion = useRef<Motion | null>(null);
+
+  useEffect(() => {
+    loud.current = level;
+  });
+
+  // The program is made when the orb appears (once for its shader and size), not for each state.
+  useEffect(() => {
+    const canvas = surface.current;
+    if (!canvas) return undefined;
+    const made = createRenderer(canvas, shader.fragment, Math.round(size * RENDER_SCALE), () => {
+      setBroken(true);
+    });
+    if (!made) {
+      setBroken(true);
+      return undefined;
+    }
+    renderer.current = made;
+    return () => {
+      made.dispose();
+      renderer.current = undefined;
+    };
+  }, [shader, size]);
+
+  useEffect(() => {
+    const gl = renderer.current;
+    const canvas = surface.current;
+    if (!gl || !canvas || broken) return undefined;
+    // The host's colour (`--sc-orb-color`), read again with each state: a theme changed since
+    // shows by the next one.
+    gl.ink(inkOf(canvas));
+    if (still) {
+      drawStill(gl, state);
+      return undefined;
+    }
+    motion.current ??= motionOf(state, read(loud.current));
+    return animate(gl, shader, state, () => read(loud.current), motion.current);
+  }, [broken, shader, size, state, still]);
+
+  if (broken) return <StillOrb state={state} size={size} label={label} className={className} />;
+  return (
+    <div
+      role="img"
+      aria-label={label}
+      part="orb"
+      className={joined(
+        part("sc-orb", `sc-orb--${state}`, ...(still ? (["sc-orb--still"] as const) : [])),
+        className,
+      )}
+      style={{ width: size, height: size }}
+    >
+      <canvas ref={surface} aria-hidden style={{ width: size, height: size }} />
+    </div>
+  );
+}

package/src/voice/fluidDots.tsx

@@ -0,0 +1,171 @@
+import type { OrbProps } from "./orbs";
+import { WebGLOrb } from "./WebGLOrb";
+import { CUTOUT, INK, INK_UNIFORMS, type Shader } from "./webgl";
+
+/**
+ * Ported from siri-wave by 40973894 (21st.dev), MIT. Shaders kept verbatim except the uniforms
+ * added below.
+ *
+ * The `fluid-dots` style of the orb: six metaballs that merge, scatter and gather. Ours is
+ * `uLevel`, how loud the microphone or the voice is, 0 to 1: the dots shine from 0.7 of their
+ * brightness in silence to all of it at full voice. How fast they move follows the energy of
+ * the state (0.4 of the original's pace at rest, all of it while listening and speaking, see
+ * `pace`), summed on the clock the shader is given so that a change of state bends the motion.
+ *
+ * The last line of `main` is ours too: the host's colour, when it gives one (`INK`, its uniforms
+ * `INK_UNIFORMS`), and what is near black turning transparent (`CUTOUT`).
+ */
+const FRAGMENT = `precision highp float;
+uniform vec2 iResolution; uniform float iTime;
+uniform float uLevel; // signalcore: the signal
+${INK_UNIFORMS}
+const float TAU = 6.28318530718;
+const int   N   = 6;
+const float SMOOTH_K = 0.08;
+const float INTENSITY  = 0.0025;
+const float FALLOFF_P  = 1.35;
+const float FADE_START = 0.02;
+const float FADE_END   = 0.56;
+const float ABERR = 0.005;
+const vec3  SPECTRAL = vec3(0.0, 0.5, 1.0) * ABERR;
+const float HUE_SPEED = 0.06;
+const float COLOR_K   = 0.5;
+const float SAT       = 0.01;
+const float HUE_SPAN  = 0.667;
+const float MERGE_PERIOD = 6.0;
+const float T_MOVE   = 1.25;
+const float STAGGER  = 0.33;
+const float HOLD     = 0.0;
+const float W = 4.6;
+const float L = 3.2;
+const float PIERCE  = 0.12;
+const float RECOIL  = 0.035;
+const float REC_LAG = 0.11;
+const float GATHER_PERIOD = 12.0;
+const float GATHER_START  = 9.2;
+const float GATHER_HOLD   = 0.8;
+const float GATHER_R      = 0.008;
+const float GATHER_DIM    = 0.85;
+const float GATHER_IN     = 1.8;
+const float GATHER_IN_L   = 7.5;
+const float BURST_W = 6.5;
+const float BURST_L = 4.0;
+const float CHARGE_T     = 0.30;
+const float CHARGE_SHRK  = 0.18;
+const float CHARGE_GLOW  = 0.35;
+const float FLASH_GAIN   = 1.2;
+const float FLASH_DECAY  = 7.0;
+
+float hash11(float n){ return fract(sin(n*127.1 + 311.7)*43758.5453); }
+float settleWL(float tau, float w, float l){
+    if(tau <= 0.0) return 0.0;
+    return 1.0 - exp(-l*tau)*cos(w*tau);
+}
+float settle(float tau){ return settleWL(tau, W, L); }
+float settleCrit(float tau, float l){
+    if(tau <= 0.0) return 0.0;
+    return 1.0 - exp(-l*tau)*(1.0 + l*tau);
+}
+float smin(float a, float b, float k){
+    float h = max(k - abs(a - b), 0.0) / k;
+    return min(a, b) - h*h*k*0.25;
+}
+vec3 hue2rgb(float h){
+    h = fract(h);
+    float r = clamp(abs(h*6.0 - 3.0) - 1.0, 0.0, 1.0);
+    float g = clamp(2.0 - abs(h*6.0 - 2.0), 0.0, 1.0);
+    float b = clamp(2.0 - abs(h*6.0 - 4.0), 0.0, 1.0);
+    return vec3(r, g, b);
+}
+float dotR(float fi, float seed, float t){
+    return 0.036 + 0.010*sin(t*1.3 + seed*TAU) + 0.005*sin(t*2.4 + fi*1.3);
+}
+float dotSD(vec2 p, vec2 pos, float r, float t, float fi, float shapeDamp){
+    vec2 d = p - pos;
+    float sq = 0.075 * (0.5 + 0.5*sin(t*0.9 + fi*2.0)) * shapeDamp;
+    float ca = cos(t*0.35 + fi), sa = sin(t*0.35 + fi);
+    d = mat2(ca,-sa,sa,ca) * d;
+    d *= vec2(1.0+sq, 1.0-sq);
+    return length(d) - r;
+}
+vec3 scene(vec2 p, float t){
+    float k  = floor(t/MERGE_PERIOD);
+    float u  = fract(t/MERGE_PERIOD);
+    float te = u * MERGE_PERIOD;
+    float tg = mod(t, GATHER_PERIOD);
+    float g  = settleCrit((tg - GATHER_START) * GATHER_IN, GATHER_IN_L)
+             - settleWL(tg - GATHER_START - GATHER_HOLD, BURST_W, BURST_L);
+    float gC = clamp(g, 0.0, 1.0);
+    float tb     = tg - (GATHER_START + GATHER_HOLD);
+    float charge = smoothstep(-CHARGE_T, 0.0, min(tb, 0.0)) * gC;
+    float flash  = tb > 0.0 ? exp(-tb * FLASH_DECAY) : 0.0;
+    float gBright = mix(1.0, GATHER_DIM, gC) * (1.0 + CHARGE_GLOW*charge + FLASH_GAIN*flash);
+    vec3  total3 = vec3(1e5);
+    vec3  cAcc   = vec3(0.0);
+    float wAcc   = 1e-6;
+    for(int i=0; i<N; i++){
+        float fi   = float(i);
+        float seed = hash11(fi);
+        float ang = fi/float(N)*TAU + t*0.35;
+        vec2 dir  = vec2(cos(ang), sin(ang));
+        float R = 0.17 + 0.010*sin(t*1.0) + 0.007*sin(t*1.3 + seed*TAU);
+        float pairId   = mod(fi, 3.0);
+        float moverLow = mod(k + pairId, 2.0);
+        float isMover  = (fi < 2.5) ? step(moverLow, 0.5) : step(0.5, moverLow);
+        float goStart  = pairId * STAGGER;
+        float retStart = 3.0*STAGGER + HOLD + pairId * STAGGER;
+        float m   = (settle(te - goStart)           - settle(te - retStart))           * isMover;
+        float rec = (settle(te - goStart - REC_LAG) - settle(te - retStart - REC_LAG)) * (1.0 - isMover);
+        float rSelf = dotR(fi, seed, t);
+        rSelf = mix(rSelf, 0.036, gC);
+        rSelf *= 1.0 - CHARGE_SHRK * charge;
+        float fj    = mod(fi + 3.0, 6.0);
+        float rPart = dotR(fj, hash11(fj), t);
+        float deep   = -(R + RECOIL) - PIERCE * rPart;
+        float radial = mix(R, deep, m) + RECOIL * rec;
+        radial = mix(radial, GATHER_R, g);
+        vec2  pos    = radial * dir;
+        float sdR = dotSD(p - SPECTRAL.r*dir, pos, rSelf, t, fi, 1.0 - gC);
+        float sdG = dotSD(p - SPECTRAL.g*dir, pos, rSelf, t, fi, 1.0 - gC);
+        float sdB = dotSD(p - SPECTRAL.b*dir, pos, rSelf, t, fi, 1.0 - gC);
+        total3 = vec3( smin(total3.r, sdR, SMOOTH_K),
+                       smin(total3.g, sdG, SMOOTH_K),
+                       smin(total3.b, sdB, SMOOTH_K) );
+        float hue = fract(fi/float(N) + t*HUE_SPEED) * HUE_SPAN;
+        vec3 dotCol = mix(vec3(1.0), hue2rgb(hue), SAT);
+        float w = exp(-sdG * COLOR_K);
+        cAcc += w * dotCol;
+        wAcc += w;
+    }
+    vec3 sd3    = max(total3, vec3(0.0)) + 1e-4;
+    vec3 core3  = clamp(INTENSITY / pow(sd3, vec3(FALLOFF_P)), 0.0, 1.0);
+    vec3 edge3  = 1.0 - smoothstep(vec3(FADE_START), vec3(FADE_END), sd3);
+    vec3 bright = core3 * edge3 * gBright;
+    return bright * (cAcc / wAcc);
+}
+void mainImage(out vec4 fragColor, in vec2 fragCoord){
+    vec2 res = iResolution.xy;
+    vec2 p = (2.0*fragCoord - res) / min(res.x, res.y);
+    float t = iTime;
+    p /= 1.0 + 0.03*sin(t*1.0);
+    vec3 col = scene(p, t);
+    col *= 1.0 + 0.05*sin(t*1.0 + 1.0);
+    col *= 0.7 + 0.3*clamp(uLevel, 0.0, 1.0); // signalcore: brighter with the voice
+    col = pow(col, vec3(1.0/1.2));
+    col = min(col, 1.0);
+    float n = fract(sin(dot(fragCoord, vec2(12.9898,78.233)))*43758.5453);
+    col += (n - 0.5)/255.0;
+    fragColor = vec4(col, 1.0);
+}
+void main(){ mainImage(gl_FragColor, gl_FragCoord.xy); ${INK} ${CUTOUT} }`;
+
+/** At rest the dots move at 0.4 of their pace; listening or speaking, at all of it. */
+export const FLUID_DOTS: Shader = {
+  fragment: FRAGMENT,
+  pace: (energy) => 0.4 + 0.6 * energy,
+};
+
+/** The `fluid-dots` style of the orb: six metaballs that merge, scatter and gather. */
+export default function FluidDotsOrb(props: OrbProps) {
+  return <WebGLOrb {...props} shader={FLUID_DOTS} />;
+}