@hypit/hypit 0.0.1 → 0.1.0

package/packages/studio/src/ui/markers.ts

@@ -0,0 +1,152 @@
+import type { Range, StudioSnapshot } from "../shared.js";
+
+/** How many distinct depth tones the stylesheet defines. */
+export const INTENT_TONES = 4;
+
+/**
+ * A tone per authored intent, shared by the code pane, the timeline and the box
+ * drawn over the picture.
+ *
+ * Author intents are coloured by **nesting depth**, not by identity. Depth is the only
+ * difference between two Selections that means anything — an inner one is a
+ * narrower claim about the same speech — so two intents at the same depth read
+ * the same, and an inner pair reads as sitting inside its outer pair rather than
+ * as an unrelated thing.
+ *
+ * The three panes have to agree, or the colour would carry no meaning: an intent
+ * pair in the prose and the clip it binds are the same fact seen twice.
+ */
+export function intentTones(snapshot: StudioSnapshot): ReadonlyMap<string, number> {
+  const tones = new Map<string, number>();
+  for (const segment of snapshot.script?.segments ?? []) {
+    tones.set(segment.id, segment.depth % INTENT_TONES);
+  }
+  for (const selection of snapshot.script?.selections ?? []) {
+    tones.set(selection.id, selection.depth % INTENT_TONES);
+  }
+  // A Moment marks an instant, so it sits at whatever level it was written in
+  // rather than enclosing anything. One level in from its Segment is honest.
+  for (const moment of snapshot.script?.moments ?? []) tones.set(moment.id, 1 % INTENT_TONES);
+  // A clip is coloured by what placed it, so a cutaway and the words that call
+  // for it read as one thing. But several clips can share one intent - the rows
+  // of a board do - and colouring them alike would say they are the same clip.
+  // Siblings therefore step through the tones from their intent's own.
+  const seen = new Map<string, number>();
+  let spare = 0;
+  for (const track of snapshot.tracks) {
+    for (const clip of track.clips) {
+      if (tones.has(clip.authoredId)) continue;
+      const intent = clip.markerId === undefined ? undefined : tones.get(clip.markerId);
+      if (intent === undefined) {
+        // Nothing said to put this here, so it takes the next tone along.
+        tones.set(clip.authoredId, spare % INTENT_TONES);
+        spare += 1;
+        continue;
+      }
+      const index = seen.get(clip.markerId!) ?? 0;
+      seen.set(clip.markerId!, index + 1);
+      tones.set(clip.authoredId, (intent + index) % INTENT_TONES);
+    }
+  }
+  return tones;
+}
+
+/**
+ * The Script ranges whose speech the playhead is inside, outermost first.
+ *
+ * Segments and Selections alike, because a Segment is a level too: `<price>`
+ * encloses `@bags`, so drawing only the Selection would show a nesting with its
+ * outermost level missing.
+ *
+ * Derived from the Script tokens each range encloses rather than from the clips
+ * bound to it, because a range is a claim about speech whether or not anything
+ * was placed on it. `@amount` inside `@fee` is still inside `@fee` even when
+ * only the outer one drives a Media Item.
+ */
+export type ScriptSpan = {
+  readonly id: string;
+  readonly depth: number;
+  readonly range: Range;
+  readonly startFrame: number;
+  readonly endFrame: number;
+};
+
+/** Every Segment and Selection, placed on the timeline by the tokens it holds. */
+export function scriptSpans(snapshot: StudioSnapshot): readonly ScriptSpan[] {
+  const tokens = snapshot.script?.tokens ?? [];
+  const placed: ScriptSpan[] = [];
+  const spans = [
+    ...(snapshot.script?.segments ?? []).map((segment) => ({
+      id: segment.id, depth: segment.depth, range: segment.range,
+    })),
+    ...(snapshot.script?.selections ?? []).map((selection) => ({
+      id: selection.id,
+      depth: selection.depth,
+      range: { start: selection.open.start, end: selection.close.end },
+    })),
+  ];
+  for (const { id, depth, range } of spans) {
+    const within = tokens.filter((token) =>
+      token.range.start >= range.start && token.range.end <= range.end);
+    if (within.length === 0) continue;
+    placed.push({
+      id, depth, range,
+      startFrame: Math.min(...within.map((token) => token.startFrame)),
+      endFrame: Math.max(...within.map((token) => token.endFrame)),
+    });
+  }
+  return placed;
+}
+
+export function liveRanges(snapshot: StudioSnapshot, frame: number): readonly ScriptSpan[] {
+  // Outermost first: an inner outline is drawn last and sits inside its parent.
+  return scriptSpans(snapshot)
+    .filter((span) => frame >= span.startFrame && frame < span.endFrame)
+    .sort((left, right) => left.depth - right.depth);
+}
+
+/**
+ * The tightest Script range written around a source offset.
+ *
+ * Clicking marked prose should land inside the thing that was clicked, and the
+ * innermost intent is the most specific claim there. Resolving to a clip alone
+ * would send a click on `@amount` to whatever encloses it, because a marker that
+ * places nothing still means something.
+ */
+export function spanAtOffset(snapshot: StudioSnapshot, offset: number): ScriptSpan | undefined {
+  return scriptSpans(snapshot)
+    .filter((span) => offset >= span.range.start && offset <= span.range.end)
+    .sort((left, right) =>
+      (left.range.end - left.range.start) - (right.range.end - right.range.start))[0];
+}
+
+/**
+ * The authored intent under a source cursor. Unlike a rendered clip, this is
+ * an author entity in its own right: clicking a Selection or Moment in prose
+ * must select that marker, even when it has no consumer and therefore no clip.
+ */
+export type SourceIntent =
+  | { readonly kind: "selection"; readonly id: string; readonly range: Range }
+  | { readonly kind: "moment"; readonly id: string; readonly range: Range };
+
+export function intentAtOffset(snapshot: StudioSnapshot, offset: number): SourceIntent | undefined {
+  const intents: SourceIntent[] = [
+    ...(snapshot.script?.selections ?? []).map((selection) => ({
+      kind: "selection" as const,
+      id: selection.id,
+      range: { start: selection.open.start, end: selection.close.end },
+    })),
+    ...(snapshot.script?.moments ?? []).map((moment) => ({
+      kind: "moment" as const,
+      id: moment.id,
+      range: moment.range,
+    })),
+  ];
+  return intents
+    .filter((intent) => offset >= intent.range.start && offset <= intent.range.end)
+    .sort((left, right) => {
+      const span = (item: SourceIntent): number => item.range.end - item.range.start;
+      const width = span(left) - span(right);
+      return width !== 0 ? width : left.kind === "moment" ? -1 : 1;
+    })[0];
+}

package/packages/studio/src/ui/material-preview.ts

@@ -0,0 +1,176 @@
+import type { StudioMaterialPreview } from "../shared.js";
+
+type Storyboard = {
+  readonly imageUrl: string;
+  readonly count: number;
+  readonly columns: number;
+  readonly rows: number;
+  readonly tileWidth: number;
+  readonly tileHeight: number;
+};
+
+const generatedAudio = new Map<string, Promise<string | undefined>>();
+const resolvedAudio = new Map<string, string | undefined>();
+const storyboards = new Map<string, Promise<Storyboard | undefined>>();
+const resolvedStoryboards = new Map<string, Storyboard | undefined>();
+
+function previewUrl(preview: StudioMaterialPreview): string {
+  if (preview.source.kind === "artifact") return `/__studio/material/${preview.source.resource}`;
+  const query = new URLSearchParams({
+    module: preview.source.module,
+    version: preview.source.version,
+    surface: preview.source.surface,
+  });
+  return `/__studio/surface-preview?${query.toString()}`;
+}
+
+function positiveHeader(response: Response, name: string): number | undefined {
+  const value = Number(response.headers.get(name));
+  return Number.isFinite(value) && value > 0 ? value : undefined;
+}
+
+async function loadStoryboard(materialUrl: string): Promise<Storyboard | undefined> {
+  const storyboardUrl = materialUrl.replace("/__studio/material/", "/__studio/storyboard/");
+  if (storyboardUrl === materialUrl) return undefined;
+  const response = await fetch(storyboardUrl);
+  if (!response.ok) return undefined;
+  const count = positiveHeader(response, "x-hypit-storyboard-count");
+  const columns = positiveHeader(response, "x-hypit-storyboard-columns");
+  const rows = positiveHeader(response, "x-hypit-storyboard-rows");
+  const tileWidth = positiveHeader(response, "x-hypit-storyboard-tile-width");
+  const tileHeight = positiveHeader(response, "x-hypit-storyboard-tile-height");
+  if (count === undefined || columns === undefined || rows === undefined
+    || tileWidth === undefined || tileHeight === undefined) return undefined;
+  return {
+    imageUrl: URL.createObjectURL(await response.blob()),
+    count: Math.floor(count),
+    columns: Math.floor(columns),
+    rows: Math.floor(rows),
+    tileWidth: Math.floor(tileWidth),
+    tileHeight: Math.floor(tileHeight),
+  };
+}
+
+function storyboardFor(url: string): Promise<Storyboard | undefined> {
+  const held = storyboards.get(url);
+  if (held !== undefined) return held;
+  const pending = loadStoryboard(url).catch(() => undefined).then((storyboard) => {
+    resolvedStoryboards.set(url, storyboard);
+    return storyboard;
+  });
+  storyboards.set(url, pending);
+  return pending;
+}
+
+function renderStoryboard(target: HTMLElement, storyboard: Storyboard): void {
+  if (!target.isConnected) return;
+  const lane = target.closest<HTMLElement>(".lane");
+  if (lane === null) return;
+  const targetBox = target.getBoundingClientRect();
+  const laneBox = lane.getBoundingClientRect();
+  const visibleLeft = Math.max(targetBox.left, laneBox.left);
+  const visibleRight = Math.min(targetBox.right, laneBox.right);
+  if (visibleRight <= visibleLeft || targetBox.width < 1 || targetBox.height < 1) return;
+
+  const slotWidth = Math.max(1, targetBox.height * storyboard.tileWidth / storyboard.tileHeight);
+  const firstSlot = Math.floor((visibleLeft - targetBox.left) / slotWidth);
+  const lastSlot = Math.ceil((visibleRight - targetBox.left) / slotWidth);
+  const frames: HTMLElement[] = [];
+  for (let slot = firstSlot; slot < lastSlot; slot += 1) {
+    const left = slot * slotWidth;
+    const progress = Math.max(0, Math.min(1, (left + slotWidth / 2) / targetBox.width));
+    const index = Math.min(storyboard.count - 1, Math.floor(progress * storyboard.count));
+    const column = index % storyboard.columns;
+    const row = Math.floor(index / storyboard.columns);
+    const frame = document.createElement("span");
+    frame.className = "clip-video-frame";
+    frame.style.left = `${left}px`;
+    // Adjacent frames share the same fractional boundary. Adding a full CSS
+    // pixel here lets the next frame overpaint roughly 3% of a 9:16 thumbnail,
+    // which reads as a horizontal crop even though the atlas itself is exact.
+    frame.style.width = `${slotWidth}px`;
+    frame.style.backgroundImage = `url(${JSON.stringify(storyboard.imageUrl)})`;
+    frame.style.backgroundSize = `${storyboard.columns * slotWidth}px ${storyboard.rows * targetBox.height}px`;
+    frame.style.backgroundPosition = `${-column * slotWidth}px ${-row * targetBox.height}px`;
+    frames.push(frame);
+  }
+  target.replaceChildren(...frames);
+  target.classList.add("ready");
+}
+
+async function audioWaveform(url: string): Promise<string | undefined> {
+  const response = await fetch(url);
+  if (!response.ok) return undefined;
+  const audioContext = new AudioContext();
+  try {
+    const buffer = await audioContext.decodeAudioData(await response.arrayBuffer());
+    const width = 2048;
+    const height = 56;
+    const channels = Array.from({ length: buffer.numberOfChannels }, (_, index) => buffer.getChannelData(index));
+    const samplesPerColumn = Math.max(1, Math.floor(buffer.length / width));
+    const segments: string[] = [];
+    for (let x = 0; x < width; x += 1) {
+      const from = x * samplesPerColumn;
+      const to = Math.min(buffer.length, from + samplesPerColumn);
+      let peak = 0;
+      for (const channel of channels) {
+        for (let sample = from; sample < to; sample += 1) {
+          peak = Math.max(peak, Math.abs(channel[sample] ?? 0));
+        }
+      }
+      const amplitude = Math.max(1, peak * (height - 4));
+      const top = (height - amplitude) / 2;
+      segments.push(`M${x} ${top.toFixed(2)}V${(top + amplitude).toFixed(2)}`);
+    }
+    const svg = `<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 ${width} ${height}" preserveAspectRatio="none"><path d="${segments.join("")}" fill="none" stroke="rgba(255,255,255,.76)" stroke-width="1" vector-effect="non-scaling-stroke"/></svg>`;
+    return `data:image/svg+xml,${encodeURIComponent(svg)}`;
+  } finally {
+    await audioContext.close();
+  }
+}
+
+function audioPreview(url: string): Promise<string | undefined> {
+  const held = generatedAudio.get(url);
+  if (held !== undefined) return held;
+  const pending = audioWaveform(url).catch(() => undefined).then((waveform) => {
+    resolvedAudio.set(url, waveform);
+    return waveform;
+  });
+  generatedAudio.set(url, pending);
+  return pending;
+}
+
+/** Mount only a real selected artifact; failed decoding deliberately leaves the item plain. */
+export function mountMaterialPreview(target: HTMLElement, preview: StudioMaterialPreview): void {
+  const url = previewUrl(preview);
+  target.classList.add(`clip-material-${preview.kind}`);
+  if (preview.kind === "video") {
+    if (resolvedStoryboards.has(url)) {
+      const storyboard = resolvedStoryboards.get(url);
+      if (storyboard !== undefined) renderStoryboard(target, storyboard);
+      return;
+    }
+    void storyboardFor(url).then((storyboard) => {
+      if (storyboard !== undefined) renderStoryboard(target, storyboard);
+    });
+    return;
+  }
+  if (preview.kind === "image") {
+    target.style.backgroundImage = `url(${JSON.stringify(url)})`;
+    target.classList.add("ready");
+    return;
+  }
+  if (resolvedAudio.has(url)) {
+    const waveform = resolvedAudio.get(url);
+    if (waveform !== undefined) {
+      target.style.backgroundImage = `url(${JSON.stringify(waveform)})`;
+      target.classList.add("ready");
+    }
+    return;
+  }
+  void audioPreview(url).then((waveform) => {
+    if (waveform === undefined || !target.isConnected) return;
+    target.style.backgroundImage = `url(${JSON.stringify(waveform)})`;
+    target.classList.add("ready");
+  });
+}

package/packages/studio/src/ui/overlay.ts

@@ -0,0 +1,144 @@
+import type { Clip, StudioSnapshot } from "../shared.js";
+import { intentTones } from "./markers.js";
+import type { Store } from "./selection.js";
+
+const SVG_NS = "http://www.w3.org/2000/svg";
+
+/**
+ * Where a clip is actually drawn, in canvas pixels, or undefined when the
+ * picture has not mounted yet.
+ */
+/** A box on the canvas, in canvas pixels. */
+export type Box = {
+  readonly xPx: number; readonly yPx: number;
+  readonly widthPx: number; readonly heightPx: number;
+};
+
+export type Measure = (clipId: string) => Box | undefined;
+
+export type Overlay = {
+  readonly element: SVGSVGElement;
+  /** The topmost clip drawn at a point, in client coordinates. */
+  hitTest(clientX: number, clientY: number): Clip | undefined;
+  /** Redraw against the picture, once it has mounted or moved. */
+  refresh(): void;
+};
+
+function rect(box: Box, className: string): SVGRectElement {
+  const node = document.createElementNS(SVG_NS, "rect");
+  node.setAttribute("x", String(box.xPx));
+  node.setAttribute("y", String(box.yPx));
+  node.setAttribute("width", String(box.widthPx));
+  node.setAttribute("height", String(box.heightPx));
+  node.setAttribute("rx", "6");
+  // preserveAspectRatio is none, so a scaled stroke would be drawn unevenly.
+  node.setAttribute("vector-effect", "non-scaling-stroke");
+  node.setAttribute("class", className);
+  return node;
+}
+
+function inside(box: Box, x: number, y: number): boolean {
+  return x >= box.xPx && x <= box.xPx + box.widthPx
+    && y >= box.yPx && y <= box.yPx + box.heightPx;
+}
+
+/**
+ * The box drawn over the picture for the selected clip.
+ *
+ * The overlay shares the composition's own coordinate system through its
+ * viewBox, so a box is written in canvas pixels with no transform to keep in
+ * step with the iframe's scale.
+ *
+ * The box itself is measured from the rendered picture rather than recomputed
+ * from the Placement Frame. Two things move a clip away from that Frame:
+ * lifecycle motion displaces it for the length of its enter and exit, and the
+ * frame paint fills the Frame while padded material does not. Measuring what
+ * was drawn is exact for both, and stays exact for whatever the renderer does
+ * next. The interpreted Frame is only the fallback for the moment before the
+ * picture has mounted.
+ */
+export function createOverlay(store: Store, measure: Measure): Overlay {
+  const element = document.createElementNS(SVG_NS, "svg");
+  element.setAttribute("class", "stage-overlay");
+  element.setAttribute("preserveAspectRatio", "none");
+
+  let live: readonly Clip[] = [];
+  let canvas = { width: 1, height: 1 };
+  let last: { snapshot: StudioSnapshot; selected: Clip | undefined; frame: number } | undefined;
+
+  /**
+   * The box is measured off the picture rather than restated from the Source:
+   * motion moves an element across its span, so only what was drawn knows where
+   * it ended up. A Present that has not mounted has no box to draw.
+   */
+  const drawnBox = (clip: Clip): Box | undefined => {
+    const ids = clip.presentId === undefined ? clip.renderIds : [clip.presentId];
+    const boxes = ids.flatMap((id) => {
+      const found = measure(id);
+      return found === undefined ? [] : [found];
+    });
+    if (boxes.length === 0) return undefined;
+    const left = Math.min(...boxes.map((box) => box.xPx));
+    const top = Math.min(...boxes.map((box) => box.yPx));
+    const right = Math.max(...boxes.map((box) => box.xPx + box.widthPx));
+    const bottom = Math.max(...boxes.map((box) => box.yPx + box.heightPx));
+    return { xPx: left, yPx: top, widthPx: right - left, heightPx: bottom - top };
+  };
+
+  const draw = (): void => {
+    if (last === undefined) return;
+    const { snapshot, selected, frame } = last;
+        canvas = { width: snapshot.space.canvasWidth, height: snapshot.space.canvasHeight };
+    element.setAttribute("viewBox", `0 0 ${canvas.width} ${canvas.height}`);
+    element.replaceChildren();
+    // Only clips actually on screen can be picked out of the picture.
+    live = store.clipsAt(frame);
+
+    if (selected === undefined) return;
+    const tones = intentTones(snapshot);
+    const tone = tones.get(selected.authoredId);
+    const onScreen = frame >= selected.startFrame && frame < selected.endFrameExclusive;
+    const box = drawnBox(selected);
+    // A Present that is not on screen at this frame was never drawn, so there
+    // is nothing to put a box around.
+    if (box === undefined) return;
+    const group = document.createElementNS(SVG_NS, "g");
+    group.setAttribute("class", `box${onScreen ? "" : " box-elsewhere"}${tone === undefined ? "" : ` tone-${tone}`}`);
+
+    group.append(rect(box, "box-content"));
+
+    const label = document.createElementNS(SVG_NS, "text");
+    label.setAttribute("x", String(box.xPx + 10));
+    label.setAttribute("y", String(Math.max(28, box.yPx - 12)));
+    label.setAttribute("class", "box-label");
+    label.textContent = onScreen ? selected.display.title : `${selected.display.title} (not at this frame)`;
+    group.append(label);
+    element.append(group);
+  };
+
+  store.subscribe(({ snapshot, selection, playhead }) => {
+    last = {
+      snapshot,
+      selected: selection.kind === "clip" ? store.clip(selection.clipId) : undefined,
+      frame: playhead.frame,
+    };
+    draw();
+  });
+
+  return {
+    element,
+    refresh: draw,
+    hitTest(clientX, clientY) {
+      const box = element.getBoundingClientRect();
+      if (box.width === 0 || box.height === 0) return undefined;
+      const x = (clientX - box.left) / box.width * canvas.width;
+      const y = (clientY - box.top) / box.height * canvas.height;
+      // clipsAt already returns topmost first, so the first hit is the one an
+      // author would say they clicked on.
+      return live.find((clip) => {
+        const box = drawnBox(clip);
+        return box !== undefined && inside(box, x, y);
+      });
+    },
+  };
+}

package/packages/studio/src/ui/resize.ts

@@ -0,0 +1,74 @@
+/**
+ * Panels the reader can size.
+ *
+ * How much room the Source needs against how much the picture needs is not a
+ * decision this tool can make: it depends on the Source, and on which of the
+ * two the reader is working on right now. So both edges are draggable, and
+ * where they were put is remembered.
+ */
+
+type Axis = "column" | "row";
+
+/**
+ * A grab handle on the edge between two panels.
+ *
+ * `apply` receives the size in pixels of the panel being sized; clamping is the
+ * caller's, because only it knows what its own panel needs to stay usable.
+ */
+export function createHandle(input: {
+  readonly axis: Axis;
+  readonly initial: number;
+  readonly minimum: number;
+  readonly maximum: () => number;
+  /** Which way the pointer moves to make the panel larger. */
+  readonly invert?: boolean;
+  readonly apply: (size: number) => void;
+  /** Where to remember it, so a reload keeps the layout. */
+  readonly remember: string;
+}): HTMLElement {
+  const element = document.createElement("div");
+  element.className = `handle handle-${input.axis}`;
+  element.setAttribute("role", "separator");
+  element.setAttribute("aria-orientation", input.axis === "column" ? "vertical" : "horizontal");
+  element.tabIndex = 0;
+
+  const held = Number(localStorage.getItem(input.remember));
+  let size = Number.isFinite(held) && held > 0 ? held : input.initial;
+  const settle = (value: number): void => {
+    size = Math.max(input.minimum, Math.min(input.maximum(), value));
+    input.apply(size);
+    localStorage.setItem(input.remember, String(size));
+  };
+  settle(size);
+
+  element.addEventListener("pointerdown", (event) => {
+    event.preventDefault();
+    element.setPointerCapture(event.pointerId);
+    const from = input.axis === "column" ? event.clientX : event.clientY;
+    const began = size;
+    const move = (moved: PointerEvent): void => {
+      const at = input.axis === "column" ? moved.clientX : moved.clientY;
+      const delta = (at - from) * (input.invert === true ? -1 : 1);
+      settle(began + delta);
+    };
+    const done = (): void => {
+      element.removeEventListener("pointermove", move);
+      element.removeEventListener("pointerup", done);
+      element.releasePointerCapture(event.pointerId);
+    };
+    element.addEventListener("pointermove", move);
+    element.addEventListener("pointerup", done);
+  });
+
+  // A handle nobody can reach with a keyboard is a handle half the readers do
+  // not have.
+  element.addEventListener("keydown", (event) => {
+    const step = event.shiftKey ? 48 : 12;
+    if (event.key === "ArrowLeft" || event.key === "ArrowUp") settle(size - step);
+    else if (event.key === "ArrowRight" || event.key === "ArrowDown") settle(size + step);
+    else return;
+    event.preventDefault();
+  });
+
+  return element;
+}