@signalcore/ui 0.1.0 → 1.0.0-beta.1

package/src/voice/motion.ts

@@ -0,0 +1,112 @@
+/**
+ * How a shader orb moves: the level and the energy of its state eased toward where they are
+ * going, its clock paced by the energy, a frame drawn each time the browser has one, and no frame
+ * asked for when nothing moves. Nothing here knows React; `WebGLOrb` runs it.
+ */
+
+import type { OrbState } from "./orbs";
+import type { Renderer, Shader } from "./webgl";
+
+/** How long an orb at rest keeps moving before it stops asking the browser for frames. */
+export const IDLE_STOP_MS = 3000;
+
+/** The longest step of time one frame may take: a tab coming back does not make the orb jump. */
+const LONGEST_STEP = 0.1;
+
+/** The moment a still frame is drawn at, and where a moving orb starts: a settled picture. */
+export const STILL_TIME = 1.0;
+
+/** How loud the still frame of a state that is not rest is drawn, so its shape still says it. */
+export const STILL_LEVEL = 0.45;
+
+/** How much the shader is asked to be moved by the signal in each state. */
+export const ENERGY: Readonly<Record<OrbState, number>> = {
+  idle: 0,
+  thinking: 0.5,
+  listening: 1,
+  speaking: 1,
+};
+
+/** How fast the level and the energy move toward where they are going, per second. */
+const LEVEL_RATE = 14;
+const ENERGY_RATE = 3;
+
+/** `from` moved toward `to` at `rate` per second: a value that does not jump. */
+function approach(from: number, to: number, rate: number, step: number): number {
+  return from + (to - from) * (1 - Math.exp(-rate * step));
+}
+
+/**
+ * Where an orb's motion is: its clock, its eased level and energy, and the moment of its last
+ * frame. One per orb, kept across its states, so a new state bends the motion instead of starting
+ * it over.
+ */
+export interface Motion {
+  time: number;
+  level: number;
+  energy: number;
+  last: number;
+}
+
+/** The motion of an orb that appears in `state`, already as loud as `level`. */
+export function motionOf(state: OrbState, level: number): Motion {
+  return { time: STILL_TIME, level, energy: ENERGY[state], last: 0 };
+}
+
+/** One frame of `state` at a fixed moment: what the orb shows under reduced motion, one per state. */
+export function drawStill(renderer: Renderer, state: OrbState): void {
+  renderer.draw({
+    time: STILL_TIME,
+    level: state === "idle" ? 0 : STILL_LEVEL,
+    energy: ENERGY[state],
+  });
+}
+
+/**
+ * Draws `state` frame by frame from `motion` on (which it moves), reading `level` on each frame.
+ * It asks for no frame while the tab is hidden, and stops three seconds after it came to rest
+ * (`idle`); the tab coming back asks again. The result stops it.
+ */
+export function animate(
+  renderer: Renderer,
+  shader: Shader,
+  state: OrbState,
+  level: () => number,
+  motion: Motion,
+): () => void {
+  let frame = 0;
+  let restingSince: number | null = null;
+
+  const tick = (stamp: number) => {
+    frame = 0;
+    const step = Math.min(LONGEST_STEP, Math.max(0, (stamp - motion.last) / 1000));
+    motion.last = stamp;
+    motion.level = approach(motion.level, level(), LEVEL_RATE, step);
+    motion.energy = approach(motion.energy, ENERGY[state], ENERGY_RATE, step);
+    motion.time += step * shader.pace(motion.energy);
+    renderer.draw({ time: motion.time, level: motion.level, energy: motion.energy });
+    if (state === "idle") {
+      restingSince ??= stamp;
+      if (stamp - restingSince >= IDLE_STOP_MS) return;
+    }
+    frame = requestAnimationFrame(tick);
+  };
+  const run = () => {
+    if (frame === 0 && !document.hidden) frame = requestAnimationFrame(tick);
+  };
+  const onVisibility = () => {
+    if (document.hidden) {
+      cancelAnimationFrame(frame);
+      frame = 0;
+    } else {
+      run();
+    }
+  };
+
+  document.addEventListener("visibilitychange", onVisibility);
+  run();
+  return () => {
+    document.removeEventListener("visibilitychange", onVisibility);
+    cancelAnimationFrame(frame);
+  };
+}

package/src/voice/orbs.ts

@@ -0,0 +1,39 @@
+/**
+ * The orb: the shape a voice conversation with an agent takes, where it stands and how loud it
+ * is. Its styles, its states and its props; the drawing is `Orb`'s.
+ */
+
+/**
+ * The ways the orb can be drawn: `wave`, the default, the iOS voice waveform; `fluid-dots`, six
+ * dots that merge and scatter. Both are shaders, and each paints its own colours.
+ */
+export const ORB_STYLES = ["wave", "fluid-dots"] as const;
+
+export type OrbStyle = (typeof ORB_STYLES)[number];
+
+/** The one drawn when nobody chose. */
+export const DEFAULT_ORB: OrbStyle = "wave";
+
+/** Where the conversation stands, as the orb shows it: at rest, or the agent listening, thinking or speaking. */
+export type OrbState = "idle" | "listening" | "thinking" | "speaking";
+
+/** The diameter when nobody says: room for the orb and the glow it gives off. */
+export const ORB_SIZE = 200;
+
+export interface OrbProps {
+  /** Where the conversation stands. The orb looks different in each. */
+  state: OrbState;
+  /**
+   * How loud the microphone (while `listening`) or the voice (while `speaking`) is, from 0 to 1;
+   * 0 in the other states. A number, or a reader the orb calls on each frame it draws (a call's
+   * meter), so it follows the sound without a render per frame. The orb eases it on its own.
+   */
+  level: number | (() => number);
+  /** The diameter in pixels. Default: 200. */
+  size?: number | undefined;
+  /** How it is drawn: `"wave"` (the default) or `"fluid-dots"`. */
+  style?: OrbStyle | undefined;
+  /** What the orb is, for those who cannot see it: its `aria-label`. */
+  label: string;
+  className?: string | undefined;
+}

package/src/voice/useReducedMotion.ts

@@ -0,0 +1,24 @@
+import { useSyncExternalStore } from "react";
+
+const LESS_MOTION = "(prefers-reduced-motion: reduce)";
+
+const motionQuery = (): MediaQueryList | null =>
+  typeof window !== "undefined" && typeof window.matchMedia === "function"
+    ? window.matchMedia(LESS_MOTION)
+    : null;
+
+/**
+ * Whether the person asked their system for less motion, followed while the page is open.
+ * Without `matchMedia` (a server, an old browser) there is no such request.
+ */
+export function useReducedMotion(): boolean {
+  return useSyncExternalStore(
+    (changed) => {
+      const query = motionQuery();
+      query?.addEventListener("change", changed);
+      return () => query?.removeEventListener("change", changed);
+    },
+    () => motionQuery()?.matches ?? false,
+    () => false,
+  );
+}

package/src/voice/wave.tsx

@@ -0,0 +1,129 @@
+import type { OrbProps } from "./orbs";
+import { WebGLOrb } from "./WebGLOrb";
+import { CUTOUT, INK, INK_UNIFORMS, type Shader } from "./webgl";
+
+/**
+ * Ported from siri-wave by 40973894 (21st.dev), MIT. Shaders kept verbatim except the uniforms
+ * added below.
+ *
+ * The `wave` style of the orb: the iOS voice waveform, a bright line with chromatic bands. The
+ * original simulated three frequency bands (`low`, `mid`, `high`) with slow sines. Two uniforms
+ * are ours, and everything marked `signalcore:` in the source below is where they are used:
+ *
+ * - `uLevel`, how loud the microphone or the voice is, 0 to 1. The bands follow it.
+ * - `uEnergy`, 0 at rest, 0.5 while thinking, 1 while listening or speaking: how much the bands
+ *   follow the level instead of the original's simulation. At rest the wave breathes on the
+ *   simulation, slowly (the clock runs at 0.35, see `pace`).
+ *
+ * The last line of `main` is ours too: the host's colour, when it gives one (`INK`, its uniforms
+ * `INK_UNIFORMS`), and what is near black turning transparent (`CUTOUT`).
+ */
+const FRAGMENT = `precision highp float;
+uniform vec2 iResolution; uniform float iTime;
+uniform float uLevel; uniform float uEnergy; // signalcore: the signal
+${INK_UNIFORMS}
+const float PI = 3.14159265359;
+const float AMPLITUDE   = 0.32;
+const float FREQ        = 1.1;
+const float ABER_FREQ   = 1.0;
+const float SPEED       = 2.4;
+const float WAVE_SCALE  = 0.6;
+const float ABERRATION  = 2.6;
+const float THICKNESS   = 3.0;
+const float INTENSITY   = 2.;
+const float FALLOFF     = 1.7;
+const float EDGE_MASK   = 0.4;
+const float EDGE_INSET  = 0.0;
+const float BAND_FILL   = 30000.0;
+const float BAND_THICK  = 0.08;
+const float SOFTNESS    = 2.5;
+const float LOW_AMP     = 6.0;
+const float LOW_INT     = 1.5;
+const float MID_ABER    = 0.8;
+const float MID_ABAMP   = 0.05;
+const float MID_BAND    = 20.0;
+const float MID_SOFT    = 0.4;
+const float HIGH_ABER   = 0.5;
+const float HIGH_ABAMP  = 0.06;
+const float RESOLVED    = 1.0;
+const float UNRES_SCALE = 0.14;
+
+vec3 spectral4(int s){
+    float x = float(s);
+    return clamp(vec3(abs(x-3.0)-1.0, 2.0-abs(x-2.0), 2.0-abs(x-4.0)), 0.0, 1.0);
+}
+
+void mainImage(out vec4 fragColor, in vec2 fragCoord){
+    vec2 R = iResolution.xy;
+    float aspect = R.x / R.y;
+    vec2 p = (fragCoord + 0.5) * 2.0 / R - 1.0;
+    p.x *= aspect;
+    float yScreen = p.y;
+    p /= max(WAVE_SCALE, 0.1);
+
+    float t   = iTime;
+    float lvl = clamp(uLevel, 0.0, 1.0); // signalcore: the bands the level drives, one shape each
+    float low  = mix(clamp(0.45 + 0.45*sin(t*0.8)*sin(t*0.37+1.0), 0.0, 1.0), lvl, uEnergy);
+    float mid  = mix(clamp(0.40 + 0.40*sin(t*1.7+2.0)*sin(t*0.53), 0.0, 1.0), lvl*(0.7 + 0.3*sin(t*5.1+2.0)), uEnergy);
+    float high = mix(clamp(0.30 + 0.30*sin(t*2.9+4.0)*sin(t*0.71+2.0), 0.0, 1.0), lvl*lvl*(0.7 + 0.3*sin(t*8.3+4.0)), uEnergy);
+
+    float res   = clamp(RESOLVED, 0.0, 1.0);
+    float drift = mod(t, 20.0*PI) * SPEED;
+
+    float xN  = p.x / max(aspect, 1.0);
+    float env = cos(PI*0.5 * min(abs(0.9*xN), 1.0));
+    env *= env;
+
+    float A1    = AMPLITUDE + 0.01*low*LOW_AMP;
+    float A2    = A1 + mid*MID_ABAMP + high*HIGH_ABAMP;
+    float AB    = (ABERRATION + mid*MID_ABER + high*HIGH_ABER)*res;
+    float th    = mix(0.1, 0.01*THICKNESS, res);
+    float inten = mix(0.1, 0.01*(INTENSITY + low*LOW_INT), res);
+    float soft  = 0.01*res*max(0.0, SOFTNESS + mid*MID_SOFT);
+
+    float dUnres = max(length(p) - mix(0.14, UNRES_SCALE, res), 0.0);
+    float yMain = A1 * env * res * sin(p.x*FREQ + drift);
+
+    float bandFillTh = max(BAND_THICK, 1e-4);
+    float bandAmt    = 1e-4 * BAND_FILL * inten;
+    vec3 num = vec3(0.0), den = vec3(0.0);
+    for(int s = 0; s < 4; s++){
+        vec3 hue = mix(vec3(1.0), spectral4(s), res);
+        den += hue;
+        float ab = mix(-AB, AB, float(s)/3.0);
+        float yL = A2 * env * res * sin(p.x*ABER_FREQ + drift + ab);
+        float d   = mix(dUnres, abs(p.y - yL), res);
+        float lor = mix(1.0/(1.0 + (0.02*d)*(0.02*d)), 1.0, res);
+        float line = inten / (sqrt(d*d + soft*soft) + th);
+        float lo = min(yMain, yL), hi = max(yMain, yL);
+        float dBand = max(0.0, max(p.y - hi, lo - p.y));
+        float band  = bandAmt / (dBand + bandFillTh);
+        num += hue * lor * (line + band);
+    }
+    vec3 col = num / den;
+
+    float dM    = mix(dUnres, abs(p.y - yMain), res);
+    float lorM  = mix(1.0/(1.0 + (0.02*dM)*(0.02*dM)), 1.0, res);
+    float boost = (1.0 - res) * (14.0*low + 4.0);
+    col += 0.5 * inten * (lorM + boost) / (sqrt(dM*dM + soft*soft) + th);
+
+    col = pow(max(col, 0.0), vec3(1.5));
+    float emT = clamp((abs(yScreen) - 1.0 + EDGE_INSET) / (-max(EDGE_MASK, 1e-4)), 0.0, 1.0);
+    float em  = emT*emT*(3.0 - 2.0*emT);
+    float gauss = exp(-pow(xN*FALLOFF, 2.0));
+    col *= mix(1.0, em*gauss, res);
+    col *= res;
+    fragColor = vec4(col, 1.0);
+}
+void main(){ mainImage(gl_FragColor, gl_FragCoord.xy); ${INK} ${CUTOUT} }`;
+
+/** At rest the wave breathes at a third of its pace; once something is heard it runs its own. */
+export const WAVE: Shader = {
+  fragment: FRAGMENT,
+  pace: (energy) => 0.35 + 0.65 * Math.min(1, energy * 2),
+};
+
+/** The `wave` style of the orb: the iOS voice waveform. */
+export default function WaveOrb(props: OrbProps) {
+  return <WebGLOrb {...props} shader={WAVE} />;
+}

package/src/voice/webgl.ts

@@ -0,0 +1,209 @@
+/**
+ * The WebGL under the `wave` and `fluid-dots` orbs: one fullscreen triangle drawn by one
+ * fragment shader, and its uniforms (the time, the signal, the host's colour). Nothing here knows
+ * React, a call or a state; the frame loop is `motion`'s.
+ */
+
+/** A style of orb drawn by a fragment shader. */
+export interface Shader {
+  /** GLSL ES 1.00 source of the fragment shader; it reads the uniforms named in `Uniforms`. */
+  fragment: string;
+  /**
+   * How fast the shader's clock runs at an energy, 0 (idle) to 1 (listening, speaking): 1 is the
+   * shader's own pace. The clock is summed frame by frame on this side, so a change of pace bends
+   * the motion instead of making it jump, as a time multiplied inside the shader would.
+   */
+  pace(energy: number): number;
+}
+
+/** What one frame is drawn from. */
+export interface Uniforms {
+  /** `iTime`: seconds on the shader's own clock, already paced. */
+  time: number;
+  /** `uLevel`: how loud the microphone or the voice is, 0 to 1. */
+  level: number;
+  /** `uEnergy`: 0 at rest, 0.5 while thinking, 1 while listening or speaking. */
+  energy: number;
+}
+
+/** A colour as the shader takes it: red, green and blue, each 0 to 1. */
+export type Rgb = readonly [number, number, number];
+
+export interface Renderer {
+  draw(values: Uniforms): void;
+  /** Paints the next frames in `ink` (`uInk`), or (null) in the shader's own colours. */
+  ink(ink: Rgb | null): void;
+  dispose(): void;
+}
+
+const VERTEX = `attribute vec2 aPos; void main(){ gl_Position=vec4(aPos,0.0,1.0); }`;
+
+/**
+ * The line every shader ends its `main` with: the shaders paint light on black, and an orb sits on
+ * any background, so what is near black becomes transparent instead. The canvas is premultiplied,
+ * so the colour goes with its alpha. There is no black square around the orb.
+ */
+export const CUTOUT = `gl_FragColor.a *= smoothstep(0.02, 0.12, max(gl_FragColor.r, max(gl_FragColor.g, gl_FragColor.b))); gl_FragColor.rgb *= gl_FragColor.a;`;
+
+/** The uniforms of the host's colour, which every shader declares beside its own. */
+export const INK_UNIFORMS = `uniform vec3 uInk; uniform float uInked; // signalcore: the host's colour`;
+
+/**
+ * The line before `CUTOUT`: with the host's colour (`uInked` 1), the shader's light is painted
+ * in it, as bright as it was (its brightest channel); without, the shader's own colours stay.
+ */
+export const INK = `gl_FragColor.rgb = mix(gl_FragColor.rgb, uInk * max(gl_FragColor.r, max(gl_FragColor.g, gl_FragColor.b)), uInked);`;
+
+/**
+ * What the shader paints so that, blended by exclusion over `surface`, its brightest light shows
+ * as `color`: the colour itself over black, its opposite over white (exclusion turns it over),
+ * and between them, channel by channel, what turns into it as far as it can; a channel of the
+ * surface near the middle turns nothing over, and keeps the colour.
+ */
+export function inkOver(color: Rgb, surface: Rgb): Rgb {
+  const channel = (wanted: number, under: number): number => {
+    const turn = 1 - 2 * under;
+    if (Math.abs(turn) < 0.2) return wanted;
+    return Math.min(1, Math.max(0, (wanted - under) / turn));
+  };
+  return [
+    channel(color[0], surface[0]),
+    channel(color[1], surface[1]),
+    channel(color[2], surface[2]),
+  ];
+}
+
+/** A pixel to read colours off: any CSS colour, as the browser paints it. */
+let swatch: CanvasRenderingContext2D | null | undefined;
+
+/** A CSS colour as red, green, blue and alpha, each 0 to 1; null where it can't be read. */
+export function rgbaOf(css: string): readonly [number, number, number, number] | null {
+  if (swatch === undefined) {
+    swatch = document.createElement("canvas").getContext("2d", { willReadFrequently: true });
+  }
+  if (swatch === null || css === "") return null;
+  swatch.clearRect(0, 0, 1, 1);
+  swatch.fillStyle = "#000";
+  swatch.fillStyle = css;
+  swatch.fillRect(0, 0, 1, 1);
+  const [r = 0, g = 0, b = 0, a = 0] = swatch.getImageData(0, 0, 1, 1).data;
+  return [r / 255, g / 255, b / 255, a / 255];
+}
+
+/**
+ * The host's colour for an orb's canvas, as its shader paints it, from what the stylesheet puts
+ * on the canvas: the colour (`color`: `--sc-orb-color`, transparent when none) and the surface it
+ * blends with (`outline-color`; a page's own, light, where the surface is see-through). Null:
+ * the shader's own colours.
+ */
+export function inkOf(canvas: HTMLCanvasElement): Rgb | null {
+  const style = getComputedStyle(canvas);
+  const color = rgbaOf(style.color);
+  if (color === null || color[3] === 0) return null;
+  const surface = rgbaOf(style.outlineColor);
+  const under: Rgb =
+    surface === null || surface[3] < 0.5 ? [1, 1, 1] : [surface[0], surface[1], surface[2]];
+  return inkOver([color[0], color[1], color[2]], under);
+}
+
+function compile(gl: WebGLRenderingContext, type: number, source: string): WebGLShader {
+  const shader = gl.createShader(type);
+  if (!shader) throw new Error("no shader");
+  gl.shaderSource(shader, source);
+  gl.compileShader(shader);
+  if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {
+    const log = gl.getShaderInfoLog(shader);
+    gl.deleteShader(shader);
+    throw new Error(log ?? "shader compile error");
+  }
+  return shader;
+}
+
+/** The program of a fragment shader, linked: the triangle's vertex shader and `fragment`. */
+function link(gl: WebGLRenderingContext, fragment: string): WebGLProgram {
+  const program = gl.createProgram();
+  const vertex = compile(gl, gl.VERTEX_SHADER, VERTEX);
+  const pixels = compile(gl, gl.FRAGMENT_SHADER, fragment);
+  gl.attachShader(program, vertex);
+  gl.attachShader(program, pixels);
+  gl.linkProgram(program);
+  // Linked, the program keeps what it needs: the shaders can go.
+  gl.deleteShader(vertex);
+  gl.deleteShader(pixels);
+  if (!gl.getProgramParameter(program, gl.LINK_STATUS)) {
+    const log = gl.getProgramInfoLog(program);
+    gl.deleteProgram(program);
+    throw new Error(log ?? "shader link error");
+  }
+  return program;
+}
+
+/**
+ * Sets `canvas` up to draw `fragment` on a square of `dim` pixels, compiled once: the program, the
+ * triangle and the uniforms' places are kept for every frame after. Nothing when WebGL is not
+ * there or the shader does not compile on this device, and `onLost` is called if the browser
+ * takes the context away later (a GPU reset, too many contexts on the page).
+ */
+export function createRenderer(
+  canvas: HTMLCanvasElement,
+  fragment: string,
+  dim: number,
+  onLost: () => void,
+): Renderer | undefined {
+  const gl = canvas.getContext("webgl", { alpha: true, premultipliedAlpha: true });
+  if (!gl) return undefined;
+
+  let program: WebGLProgram;
+  try {
+    program = link(gl, fragment);
+  } catch {
+    return undefined;
+  }
+  gl.useProgram(program);
+
+  const buffer = gl.createBuffer();
+  gl.bindBuffer(gl.ARRAY_BUFFER, buffer);
+  // One triangle bigger than the screen: three vertices cover it, with no seam across a quad.
+  gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([-1, -1, 3, -1, -1, 3]), gl.STATIC_DRAW);
+  const position = gl.getAttribLocation(program, "aPos");
+  gl.enableVertexAttribArray(position);
+  gl.vertexAttribPointer(position, 2, gl.FLOAT, false, 0, 0);
+
+  canvas.width = dim;
+  canvas.height = dim;
+  gl.viewport(0, 0, dim, dim);
+  gl.clearColor(0, 0, 0, 0);
+  // A uniform a shader does not use is optimised out and has no place: setting it does nothing.
+  gl.uniform2f(gl.getUniformLocation(program, "iResolution"), dim, dim);
+  const time = gl.getUniformLocation(program, "iTime");
+  const level = gl.getUniformLocation(program, "uLevel");
+  const energy = gl.getUniformLocation(program, "uEnergy");
+  const inkColor = gl.getUniformLocation(program, "uInk");
+  const inked = gl.getUniformLocation(program, "uInked");
+  gl.uniform1f(inked, 0);
+
+  const lost = (event: Event) => {
+    event.preventDefault();
+    onLost();
+  };
+  canvas.addEventListener("webglcontextlost", lost);
+
+  return {
+    draw(values) {
+      gl.uniform1f(time, values.time);
+      gl.uniform1f(level, values.level);
+      gl.uniform1f(energy, values.energy);
+      gl.clear(gl.COLOR_BUFFER_BIT);
+      gl.drawArrays(gl.TRIANGLES, 0, 3);
+    },
+    ink(ink) {
+      if (ink !== null) gl.uniform3f(inkColor, ink[0], ink[1], ink[2]);
+      gl.uniform1f(inked, ink === null ? 0 : 1);
+    },
+    dispose() {
+      canvas.removeEventListener("webglcontextlost", lost);
+      gl.deleteProgram(program);
+      gl.deleteBuffer(buffer);
+    },
+  };
+}

package/src/widget/css.d.ts

@@ -0,0 +1,5 @@
+// A stylesheet as text, which the widget puts into its shadow root (Vite's `?inline`).
+declare module "*.css?inline" {
+  const css: string;
+  export default css;
+}