@signalcore/ui 0.1.0 → 1.0.0-beta.1

package/src/blocks.ts

@@ -0,0 +1,249 @@
+/**
+ * What an agent can draw: a surface, a tree of blocks from one kit, with a text twin for channels
+ * that have no screen. The agent composes each screen from these pieces only; the renderer draws
+ * every piece in the host's style, and nothing the agent sends runs as code. Written by hand, one
+ * to one with `v3/api/app/ui/blocks.py`, the one source (the API validates every surface against
+ * it at `show`). Keys the API leaves out when unset are optional here.
+ */
+
+import type { AutomationView } from "./automation/types";
+import type { FileBlock } from "./types";
+
+export type Tone = "info" | "success" | "warning" | "danger";
+
+// --- Display ------------------------------------------------------------------------------
+
+/** Words, as Markdown. */
+export interface TextBlock {
+  type: "text";
+  text: string;
+}
+
+/** Label and value pairs: a summary a person reads at a glance. */
+export interface FieldsBlock {
+  type: "fields";
+  items: { label: string; value: string }[];
+}
+
+/** Rows under named columns; a row's cells come in the columns' order. */
+export interface TableBlock {
+  type: "table";
+  columns: string[];
+  rows: string[][];
+}
+
+export interface Entry {
+  id: string;
+  title: string;
+  subtitle?: string;
+  /** At its right: a price, a time, a status. */
+  meta?: string;
+  /** A small picture at its left (https). */
+  image?: string;
+}
+
+/** Things in a column; with `action`, picking one sends it with `{item: id}`. */
+export interface ListBlock {
+  type: "list";
+  items: Entry[];
+  action?: string;
+}
+
+/** One number that matters, under its label, with a line of context in its tone. */
+export interface StatBlock {
+  type: "stat";
+  label: string;
+  value: string;
+  hint?: string;
+  tone?: Tone;
+}
+
+export interface ChartPoint {
+  label: string;
+  value: number;
+}
+
+/**
+ * Figures over a few labels: bars to compare them (by state, by who), a line to follow them in
+ * time (by day, by week), the labels in their order.
+ */
+export interface ChartBlock {
+  type: "chart";
+  kind: "bar" | "line";
+  points: ChartPoint[];
+}
+
+/** A line that stands out: what the person should notice. */
+export interface NoteBlock {
+  type: "note";
+  tone: Tone;
+  text: string;
+}
+
+export interface ImageBlock {
+  type: "image";
+  /** https only. */
+  url: string;
+  alt: string;
+  caption?: string;
+}
+
+/**
+ * One of an agent's automations, as its page shows it, read only: the agent names it (`agent`,
+ * its name or id, and `name`), and the API fills in `view` before the surface is shown or kept,
+ * with its newest 5 runs; null when there is no such automation.
+ */
+export interface AutomationBlock {
+  type: "automation";
+  agent: string;
+  name: string;
+  view: AutomationView | null;
+}
+
+// --- Action -------------------------------------------------------------------------------
+
+/** One thing the person can do: pressing it sends `action`, or, with `url`, opens that page. */
+export interface ButtonBlock {
+  type: "button";
+  label: string;
+  action?: string;
+  url?: string;
+  tone: "primary" | "neutral" | "danger";
+}
+
+export interface Option {
+  id: string;
+  label: string;
+  hint?: string;
+}
+
+/**
+ * Pick one of a few, at once; picking sends `action` with `{option: id}`. With `other`, a last
+ * row takes an answer in the person's own words instead: `{text}`.
+ */
+export interface ChoiceBlock {
+  type: "choice";
+  options: Option[];
+  action: string;
+  other?: boolean;
+}
+
+// --- Input --------------------------------------------------------------------------------
+
+export type FieldKind =
+  | "text"
+  | "textarea"
+  | "email"
+  | "phone"
+  | "url"
+  | "number"
+  | "date"
+  | "time"
+  | "datetime"
+  | "select"
+  | "multiselect"
+  | "checkbox"
+  | "range"
+  | "rating"
+  /** Files the person uploads to the conversation as they pick them; the value is their ids. */
+  | "file"
+  /** A value typed once and never stored by the conversation (`ask_person` only). */
+  | "secret";
+
+/** What a field holds: words, a number, yes or no, option ids; dates and times as ISO text. */
+export type FieldValue = string | number | boolean | string[] | null;
+
+/**
+ * One thing to fill. Its `kind` picks the control; the rest narrows it: `options` for a select or
+ * multiselect; `min`, `max`, `step` and `unit` for a number, range or rating (stars up to `max`,
+ * 5 by default); `dates` (the only days to pick, "YYYY-MM-DD"), `earliest` and `latest` for a
+ * date or datetime; `times` (the only slots, "HH:MM") for a time or datetime; `accept`,
+ * `multiple` and `max_mb` for a file.
+ */
+export interface FormField {
+  name: string;
+  /** None: the field stands alone, its question above it. */
+  label?: string;
+  kind: FieldKind;
+  required: boolean;
+  placeholder?: string;
+  /** A quiet line under it. */
+  help?: string;
+  options?: Option[];
+  min?: number;
+  max?: number;
+  step?: number;
+  unit?: string;
+  dates?: string[];
+  earliest?: string;
+  latest?: string;
+  times?: string[];
+  /** The kinds of file it takes: types, families ("image/*"), extensions (".xlsx"); none: any. */
+  accept?: string[];
+  /** More than one file. */
+  multiple?: boolean;
+  /** The most a file may weigh, in MB (50, the ceiling, by default). */
+  max_mb?: number;
+  /** What it starts with. */
+  value?: FieldValue;
+}
+
+/**
+ * Fields to fill; sending them sends `action` with the values by field name: words, a number,
+ * true or false, a date "YYYY-MM-DD", a time "HH:MM", a moment "YYYY-MM-DDTHH:MM", a
+ * multiselect's option ids, a file field's file ids.
+ */
+export interface FormBlock {
+  type: "form";
+  fields: FormField[];
+  /** None: the host's own word for sending (`KitMessages.send`). */
+  submit?: string;
+  action: string;
+}
+
+// --- Layout -------------------------------------------------------------------------------
+
+export interface DividerBlock {
+  type: "divider";
+}
+
+/** Blocks under a title: one thing, set apart. */
+export interface CardBlock {
+  type: "card";
+  title?: string;
+  subtitle?: string;
+  /** At its head's start: an app's logo, a product's picture (https). */
+  image?: string;
+  children: Block[];
+}
+
+/** Blocks laid out together: down, or across (wrapping on a narrow screen). */
+export interface StackBlock {
+  type: "stack";
+  direction: "column" | "row";
+  children: Block[];
+}
+
+export type Block =
+  | CardBlock
+  | StackBlock
+  | DividerBlock
+  | TextBlock
+  | FieldsBlock
+  | TableBlock
+  | ListBlock
+  | StatBlock
+  | ChartBlock
+  | NoteBlock
+  | ImageBlock
+  | AutomationBlock
+  | FileBlock
+  | ButtonBlock
+  | ChoiceBlock
+  | FormBlock;
+
+/** What the agent shows: blocks, and the words a channel without a screen gets instead. */
+export interface Surface {
+  blocks: Block[];
+  text: string;
+}

package/src/browser.ts

@@ -0,0 +1,334 @@
+/**
+ * The conversation's browser, apart from how it is drawn: its screen as pictures to watch, the
+ * person's pointer and keys as the input it takes, and where it is in a few words.
+ */
+
+import type { BrowserFrame, BrowserInput, Message } from "./types";
+
+/** The action that hands the person the browser; the turn waits on it until they give it back. */
+export const HAND_OVER = "browser_hand_over";
+
+/**
+ * The browser's screen as it comes: the latest picture, and whoever watches. Kept out of the
+ * conversation's state, so a picture repaints a canvas and nothing else.
+ */
+export interface Screen {
+  /** A new picture: each watcher hears it. */
+  show: (frame: BrowserFrame) => void;
+  /** The browser closed: no picture any more. */
+  clear: () => void;
+  /** Hears each picture from now on, the latest at once; returns how to stop. */
+  watch: (listener: (frame: BrowserFrame) => void) => () => void;
+}
+
+export function createScreen(): Screen {
+  let latest: BrowserFrame | null = null;
+  const listeners = new Set<(frame: BrowserFrame) => void>();
+  return {
+    show: (frame) => {
+      latest = frame;
+      for (const listener of listeners) listener(frame);
+    },
+    clear: () => {
+      latest = null;
+    },
+    watch: (listener) => {
+      listeners.add(listener);
+      if (latest) listener(latest);
+      return () => {
+        listeners.delete(listener);
+      };
+    },
+  };
+}
+
+/**
+ * The hand-over the turn waits on (its step's id) when the browser it handed over is no longer
+ * there to give back: the person can still go on.
+ */
+export function handOverWaiting(
+  messages: readonly Message[],
+  awaiting: string | null,
+): string | null {
+  if (awaiting === null) return null;
+  const step = messages.flatMap((message) => message.steps).find((one) => one.id === awaiting);
+  return step?.action === HAND_OVER ? awaiting : null;
+}
+
+/** "es.wikipedia.org" and the rest of the address apart, for an address bar; nothing for a blank page. */
+export function siteOf(url: string): { host: string; rest: string } {
+  try {
+    const { protocol, host, pathname, search } = new URL(url);
+    if (!protocol.startsWith("http")) return { host: "", rest: "" };
+    return { host, rest: pathname === "/" && !search ? "" : `${pathname}${search}` };
+  } catch {
+    return { host: "", rest: "" };
+  }
+}
+
+export interface Size {
+  width: number;
+  height: number;
+}
+
+/** Where a pointer event is on the page: scaled from the element as shown, kept inside it. */
+export function pointOf(
+  event: { clientX: number; clientY: number },
+  box: { left: number; top: number; width: number; height: number },
+  size: Size,
+): { x: number; y: number } {
+  const along = (at: number, from: number, shown: number, real: number) =>
+    shown > 0 ? Math.round(Math.min(Math.max((at - from) / shown, 0), 1) * real) : 0;
+  return {
+    x: along(event.clientX, box.left, box.width, size.width),
+    y: along(event.clientY, box.top, box.height, size.height),
+  };
+}
+
+/** DevTools' mask of the keys held with another: Alt 1, Ctrl 2, Meta 4, Shift 8. */
+export function modifiersOf(event: {
+  altKey: boolean;
+  ctrlKey: boolean;
+  metaKey: boolean;
+  shiftKey: boolean;
+}): number {
+  return (
+    (event.altKey ? 1 : 0) |
+    (event.ctrlKey ? 2 : 0) |
+    (event.metaKey ? 4 : 0) |
+    (event.shiftKey ? 8 : 0)
+  );
+}
+
+type KeyInput = Extract<BrowserInput, { kind: "key" }>;
+
+/**
+ * A key as the browser takes it; nothing for a key that stays here: Tab (the focus leaves the
+ * screen) and paste (the text comes in its own event, from this computer's clipboard).
+ */
+export function keyOf(
+  event: Pick<
+    KeyboardEvent,
+    "key" | "code" | "keyCode" | "altKey" | "ctrlKey" | "metaKey" | "shiftKey"
+  >,
+  action: "down" | "up",
+): KeyInput | null {
+  if (event.key === "Tab") return null;
+  if ((event.ctrlKey || event.metaKey) && event.key.toLowerCase() === "v") return null;
+  return {
+    kind: "key",
+    action,
+    key: event.key.slice(0, 32),
+    code: event.code.slice(0, 32),
+    keyCode: Math.min(Math.max(event.keyCode, 0), 255),
+    modifiers: modifiersOf(event),
+  };
+}
+
+const BUTTONS: Readonly<Record<number, "left" | "middle" | "right">> = {
+  0: "left",
+  1: "middle",
+  2: "right",
+};
+const MOVE_EVERY_MS = 40; // where the pointer is, 25 times a second at most
+// A trackpad fires the wheel 60 times a second, and each one waits on the hosted browser: what
+// turns within this long goes as one, so the page keeps up with the hand instead of trailing it.
+const WHEEL_EVERY_MS = 50;
+const LINE_PX = 16; // a wheel that scrolls by lines, in pixels
+const MAX_DELTA = 10_000;
+const MAX_PASTE = 4_000;
+
+const clamp = (value: number, limit: number) => Math.min(Math.max(value, -limit), limit);
+
+/**
+ * Sends what the person does on `element` (the screen) to `send`, in the page's pixels; the
+ * returned function stops it. The pointer goes as down, move and up (drags and double clicks
+ * work as on the page; a drag that leaves the screen still ends), the wheel as wheel (the page
+ * around does not scroll; the turns of `WHEEL_EVERY_MS` summed into one), keys while the screen
+ * has the focus, and text pasted there as text. A key still held when the focus goes is let go.
+ * The context menu is the page's.
+ */
+export function capture(
+  element: HTMLElement,
+  size: Size,
+  send: (input: BrowserInput) => void,
+): () => void {
+  const held = new Map<string, KeyInput>();
+  let lastMove = Number.NEGATIVE_INFINITY;
+  const at = (event: { clientX: number; clientY: number }) =>
+    pointOf(event, element.getBoundingClientRect(), size);
+  // The wheel's turns not sent yet, where the pointer was, and when the last went.
+  const turned = { x: 0, y: 0, dx: 0, dy: 0 };
+  let lastWheel = Number.NEGATIVE_INFINITY;
+  let wheelTimer: ReturnType<typeof setTimeout> | undefined;
+  const flushWheel = () => {
+    wheelTimer = undefined;
+    lastWheel = performance.now();
+    const { x, y, dx, dy } = turned;
+    turned.dx = 0;
+    turned.dy = 0;
+    if (dx !== 0 || dy !== 0) {
+      send({ kind: "wheel", x, y, dx: clamp(dx, MAX_DELTA), dy: clamp(dy, MAX_DELTA) });
+    }
+  };
+
+  const down = (event: PointerEvent) => {
+    const button = BUTTONS[event.button];
+    if (!button) return;
+    event.preventDefault(); // the page would select the screen, and keep the focus where it was
+    element.focus({ preventScroll: true });
+    element.setPointerCapture(event.pointerId);
+    const clicks = Math.min(Math.max(event.detail, 1), 3);
+    send({ kind: "mouse", action: "down", ...at(event), button, clicks });
+  };
+  const up = (event: PointerEvent) => {
+    const button = BUTTONS[event.button];
+    if (!button) return;
+    const clicks = Math.min(Math.max(event.detail, 1), 3);
+    send({ kind: "mouse", action: "up", ...at(event), button, clicks });
+  };
+  const move = (event: PointerEvent) => {
+    if (event.timeStamp - lastMove < MOVE_EVERY_MS) return;
+    lastMove = event.timeStamp;
+    send({ kind: "mouse", action: "move", ...at(event) });
+  };
+  const wheel = (event: WheelEvent) => {
+    event.preventDefault();
+    const unit = event.deltaMode === 1 ? LINE_PX : event.deltaMode === 2 ? size.height : 1;
+    Object.assign(turned, at(event));
+    turned.dx += event.deltaX * unit;
+    turned.dy += event.deltaY * unit;
+    if (wheelTimer !== undefined) return;
+    // The first turn goes at once; what follows it within the window, together at its end.
+    const wait = lastWheel + WHEEL_EVERY_MS - performance.now();
+    if (wait <= 0) flushWheel();
+    else wheelTimer = setTimeout(flushWheel, wait);
+  };
+  const keyed = (action: "down" | "up") => (event: KeyboardEvent) => {
+    if (event.isComposing) return;
+    const key = keyOf(event, action);
+    if (!key) return;
+    event.preventDefault();
+    if (action === "down") held.set(event.code || event.key, { ...key, action: "up" });
+    else held.delete(event.code || event.key);
+    send(key);
+  };
+  const paste = (event: ClipboardEvent) => {
+    const text = event.clipboardData?.getData("text/plain") ?? "";
+    if (text === "") return;
+    event.preventDefault();
+    send({ kind: "paste", text: text.slice(0, MAX_PASTE) });
+  };
+  const letGo = () => {
+    for (const key of held.values()) send(key);
+    held.clear();
+  };
+  const noMenu = (event: Event) => {
+    event.preventDefault();
+  };
+  const keyDown = keyed("down");
+  const keyUp = keyed("up");
+
+  element.addEventListener("pointerdown", down);
+  element.addEventListener("pointerup", up);
+  element.addEventListener("pointermove", move);
+  element.addEventListener("wheel", wheel, { passive: false });
+  element.addEventListener("keydown", keyDown);
+  element.addEventListener("keyup", keyUp);
+  element.addEventListener("paste", paste);
+  element.addEventListener("blur", letGo);
+  element.addEventListener("contextmenu", noMenu);
+  return () => {
+    clearTimeout(wheelTimer);
+    element.removeEventListener("pointerdown", down);
+    element.removeEventListener("pointerup", up);
+    element.removeEventListener("pointermove", move);
+    element.removeEventListener("wheel", wheel);
+    element.removeEventListener("keydown", keyDown);
+    element.removeEventListener("keyup", keyUp);
+    element.removeEventListener("paste", paste);
+    element.removeEventListener("blur", letGo);
+    element.removeEventListener("contextmenu", noMenu);
+  };
+}
+
+/**
+ * The bytes in a base64 string: the platform's own decoder where it has one, else a plain loop
+ * (`Uint8Array.from` with a mapper walks a frame's 100,000 characters through an iterator, a
+ * callback each, on the main thread, as the next picture is due).
+ */
+export function bytesOf(base64: string): Uint8Array<ArrayBuffer> {
+  const native = (Uint8Array as { fromBase64?: (text: string) => Uint8Array<ArrayBuffer> })
+    .fromBase64;
+  if (native) return native(base64);
+  const text = atob(base64);
+  const bytes = new Uint8Array(text.length);
+  for (let at = 0; at < text.length; at += 1) bytes[at] = text.charCodeAt(at);
+  return bytes;
+}
+
+/** Draws a picture on a canvas, unless a newer one comes while it is still being decoded. */
+export interface Painter {
+  paint: (jpeg: string) => void;
+  dispose: () => void;
+}
+
+/**
+ * Paints JPEGs (base64) on `canvas`, stretched to its size, v2's way: decoded off the main thread
+ * (`createImageBitmap`), one at a time, and drawn on the screen's next frame, so a picture never
+ * lands mid-paint. A picture that comes while another is being decoded waits, and a newer one
+ * replaces it, as a newer decoded one replaces one not drawn yet: a slow device skips to the
+ * present instead of falling behind. The canvas keeps the last picture until the next is ready,
+ * so it never flickers; one that does not decode is skipped. `onPaint` hears each one drawn.
+ */
+export function painterOn(canvas: HTMLCanvasElement, onPaint?: () => void): Painter {
+  let decoding = false;
+  let waiting: string | undefined;
+  let ready: ImageBitmap | undefined; // decoded, for the next frame
+  let frame = 0; // the animation frame asked for, if one is
+  let disposed = false;
+
+  const draw = () => {
+    frame = 0;
+    const bitmap = ready;
+    ready = undefined;
+    if (!bitmap) return;
+    canvas.getContext("2d")?.drawImage(bitmap, 0, 0, canvas.width, canvas.height);
+    bitmap.close();
+    onPaint?.();
+  };
+
+  const decode = async (jpeg: string): Promise<void> => {
+    decoding = true;
+    try {
+      const bitmap = await createImageBitmap(new Blob([bytesOf(jpeg)], { type: "image/jpeg" }));
+      if (disposed) {
+        bitmap.close();
+      } else {
+        ready?.close();
+        ready = bitmap;
+        if (frame === 0) frame = requestAnimationFrame(draw);
+      }
+    } catch {
+      // Cut off or damaged: the next picture replaces it.
+    }
+    decoding = false;
+    const next = waiting;
+    waiting = undefined;
+    if (next !== undefined && !disposed) await decode(next);
+  };
+
+  return {
+    paint: (jpeg) => {
+      if (decoding) waiting = jpeg;
+      else void decode(jpeg);
+    },
+    dispose: () => {
+      disposed = true;
+      waiting = undefined;
+      if (frame !== 0) cancelAnimationFrame(frame);
+      ready?.close();
+      ready = undefined;
+    },
+  };
+}