@unoverse-platform/studio 0.1.90 → 0.1.91

package/vendor/base/src/manifests/runtime/docstore/sectionizer.ts

@@ -0,0 +1,184 @@
+/**
+ * TRANSCRIBED VERBATIM from the retired smart-document package (service/sectionizer.ts),
+ * import paths aside. The parsing rules are load-bearing: a doc stored under the old node
+ * must parse identically under this one, or every section hash changes and every agent's
+ * next edit fails STALE_SECTION at once.
+ */
+import { createHash, randomBytes } from "crypto";
+import type { Section, SectionLevel, Doc } from "./types.js";
+
+/**
+ * Parse raw markdown into a flat section list.
+ * Only H1/H2 (configurable) become sections; H3+ stays in body.
+ * ATX headings only (# / ##). Setext (=== / ---) and fenced-code-lines-that-look-like-headings
+ * are intentionally ignored for simplicity.
+ */
+export function parseMarkdown(source: string, sectionizeAt: SectionLevel = 2): Section[] {
+  const lines = source.split("\n");
+  const sections: Section[] = [];
+  let current: { level: SectionLevel; heading: string; body: string[] } | null = null;
+  let inFence = false;
+  let fenceMarker = "";
+
+  const flush = () => {
+    if (current === null) return;
+    // Strip trailing empty lines from body
+    const body = current.body.join("\n").replace(/\s+$/g, "");
+    sections.push(makeSection(current.level, current.heading, body, null, sections.length));
+    current = null;
+  };
+
+  for (const line of lines) {
+    const fenceMatch = line.match(/^(```+|~~~+)/);
+    if (fenceMatch) {
+      if (!inFence) {
+        inFence = true;
+        fenceMarker = fenceMatch[1];
+      } else if (line.startsWith(fenceMarker)) {
+        inFence = false;
+        fenceMarker = "";
+      }
+    }
+
+    if (!inFence) {
+      const headingMatch = line.match(/^(#{1,6})\s+(.*?)\s*#*\s*$/);
+      if (headingMatch) {
+        const level = headingMatch[1].length;
+        const heading = headingMatch[2].trim();
+        if (level <= sectionizeAt) {
+          flush();
+          current = { level: level as SectionLevel, heading, body: [] };
+          continue;
+        }
+      }
+    }
+
+    if (current === null) {
+      // Content before the first qualifying heading — if non-empty, stash it.
+      if (line.trim().length > 0) {
+        current = { level: 1, heading: "", body: [line] };
+      }
+    } else {
+      current.body.push(line);
+    }
+  }
+
+  flush();
+
+  // Build parentId relationships: H2s nest under the most recent H1.
+  let lastH1Id: string | null = null;
+  for (const s of sections) {
+    if (s.level === 1) {
+      lastH1Id = s.id;
+      s.parentId = null;
+    } else if (s.level === 2) {
+      s.parentId = lastH1Id;
+    }
+  }
+
+  return sections;
+}
+
+/**
+ * Serialize a section list back to markdown.
+ * Order is the section array order (which is kept in render order by the caller).
+ */
+export function renderMarkdown(sections: Section[]): string {
+  const parts: string[] = [];
+  for (const s of sections) {
+    if (s.heading.length > 0) {
+      parts.push(`${"#".repeat(s.level)} ${s.heading}`);
+    }
+    if (s.body.length > 0) {
+      parts.push(s.body);
+    }
+  }
+  return parts.join("\n\n") + "\n";
+}
+
+/**
+ * Reject bodies that contain H1/H2 headings (would corrupt the outline).
+ * Returns the offending heading text, or null if clean.
+ */
+export function findDisallowedHeading(body: string, sectionizeAt: SectionLevel = 2): string | null {
+  const lines = body.split("\n");
+  let inFence = false;
+  let fenceMarker = "";
+  for (const line of lines) {
+    const fenceMatch = line.match(/^(```+|~~~+)/);
+    if (fenceMatch) {
+      if (!inFence) {
+        inFence = true;
+        fenceMarker = fenceMatch[1];
+      } else if (line.startsWith(fenceMarker)) {
+        inFence = false;
+        fenceMarker = "";
+      }
+      continue;
+    }
+    if (inFence) continue;
+    const m = line.match(/^(#{1,6})\s+(.*?)\s*#*\s*$/);
+    if (m && m[1].length <= sectionizeAt) {
+      return line;
+    }
+  }
+  return null;
+}
+
+export function makeSection(
+  level: SectionLevel,
+  heading: string,
+  body: string,
+  parentId: string | null,
+  order: number,
+): Section {
+  return {
+    id: generateId(),
+    level,
+    heading,
+    body,
+    parentId,
+    order,
+    hash: hashSection(heading, body),
+  };
+}
+
+export function generateId(): string {
+  return "sec_" + randomBytes(6).toString("base64url").replace(/[^a-z0-9]/gi, "").slice(0, 8).toLowerCase();
+}
+
+export function hashSection(heading: string, body: string): string {
+  return createHash("sha256").update(heading + "\n\n" + body).digest("hex").slice(0, 12);
+}
+
+/**
+ * Reconcile section order / parentId consistency after a mutation.
+ * H2 sections automatically re-parent to the most recent preceding H1.
+ * Rewrites `order` so siblings have contiguous ascending indices.
+ */
+export function reconcile(doc: Doc): void {
+  // 1. Re-parent H2s to most recent H1 based on array order.
+  let lastH1Id: string | null = null;
+  for (const s of doc.sections) {
+    if (s.level === 1) {
+      lastH1Id = s.id;
+      s.parentId = null;
+    } else if (s.level === 2) {
+      s.parentId = lastH1Id;
+    }
+  }
+
+  // 2. Rewrite sibling order indices.
+  const groups = new Map<string | null, Section[]>();
+  for (const s of doc.sections) {
+    const key = s.parentId;
+    const arr = groups.get(key) ?? [];
+    arr.push(s);
+    groups.set(key, arr);
+  }
+  for (const arr of groups.values()) {
+    arr.forEach((s, i) => {
+      s.order = i;
+    });
+  }
+}

package/vendor/base/src/manifests/runtime/docstore/store.ts

@@ -0,0 +1,172 @@
+/**
+ * THE DOC'S HOME IN REDIS, transcribed from the retired smart-document package
+ * (service/markdownStore.ts) with ONE change: functions take the RAW REDIS CLIENT instead
+ * of the plugin `api`, because this now lives in the manifest runtime where the executor
+ * already holds the client (the same inversion `makeStateStore` uses).
+ *
+ * WATCH/MULTI optimistic locking is the reason this is an executor capability and not a
+ * sequence of `state` calls: read-check-write must be one transaction, retried on conflict,
+ * and no manifest call list can express "and if someone wrote in between, start over".
+ *
+ * KEYS ARE DELIBERATELY NOT NAMESPACED (`md:<user>:<workflow>:<conversation>:<node>`),
+ * exactly as the retired node wrote them — a live doc mid-conversation must stay readable
+ * across this migration, and hashing or prefixing the key would orphan every one of them.
+ */
+import type { Doc, Section, SectionLevel } from "./types.js";
+import { parseMarkdown, reconcile } from "./sectionizer.js";
+
+const TTL_SECONDS = 60 * 60 * 6;
+const MAX_RETRIES = 5;
+
+export function keyFor(
+  userId: string,
+  workflowId: string,
+  conversationId: string,
+  nodeId: string,
+): string {
+  return `md:${userId}:${workflowId}:${conversationId}:${nodeId}`;
+}
+
+/**
+ * Read the doc. Lazy-migrates the legacy `{ content, version, updatedAt }` shape to the
+ * `{ sections, version, updatedAt }` shape on first access.
+ */
+export async function getDoc(
+  redis: any,
+  key: string,
+  sectionizeAt: SectionLevel = 2,
+): Promise<Doc | null> {
+  const raw = await redis.get(key);
+  if (!raw) return null;
+  try {
+    const parsed = JSON.parse(raw);
+    if (Array.isArray(parsed?.sections)) {
+      return parsed as Doc;
+    }
+    if (typeof parsed?.content === "string") {
+      const migrated = migrateLegacy(parsed.content, parsed.version ?? 1, sectionizeAt);
+      await redis.set(key, JSON.stringify(migrated), "EX", TTL_SECONDS);
+      return migrated;
+    }
+    return null;
+  } catch {
+    return null;
+  }
+}
+
+function migrateLegacy(content: string, version: number, sectionizeAt: SectionLevel): Doc {
+  const sections = parseMarkdown(content, sectionizeAt);
+  const doc: Doc = {
+    sections,
+    version: version + 1,
+    updatedAt: new Date().toISOString(),
+  };
+  reconcile(doc);
+  return doc;
+}
+
+/** Initialise the doc from `initialMarkdown` if it doesn't exist. No-op when present. */
+export async function initDoc(
+  redis: any,
+  key: string,
+  initialMarkdown: string,
+  sectionizeAt: SectionLevel = 2,
+): Promise<Doc> {
+  const existing = await getDoc(redis, key, sectionizeAt);
+  if (existing) return existing;
+  const sections = parseMarkdown(initialMarkdown, sectionizeAt);
+  const doc: Doc = {
+    sections,
+    version: 1,
+    updatedAt: new Date().toISOString(),
+  };
+  reconcile(doc);
+  await redis.set(key, JSON.stringify(doc), "EX", TTL_SECONDS);
+  return doc;
+}
+
+/** Force a fresh doc from initialMarkdown, discarding existing state. Fresh IDs. */
+export async function resetDoc(
+  redis: any,
+  key: string,
+  initialMarkdown: string,
+  sectionizeAt: SectionLevel = 2,
+): Promise<Doc> {
+  const sections = parseMarkdown(initialMarkdown, sectionizeAt);
+  const doc: Doc = {
+    sections,
+    version: 1,
+    updatedAt: new Date().toISOString(),
+  };
+  reconcile(doc);
+  await redis.set(key, JSON.stringify(doc), "EX", TTL_SECONDS);
+  return doc;
+}
+
+/**
+ * Optimistic-locking mutation with Redis WATCH/MULTI. `mutate` receives the current doc and
+ * may either mutate it in place + return ok, or return an error object. Retries up to
+ * MAX_RETRIES on WATCH conflict.
+ */
+export async function mutateDoc(
+  redis: any,
+  key: string,
+  mutate: (doc: Doc) =>
+    | { ok: true }
+    | { ok: false; error: string; extra?: Record<string, any> },
+): Promise<
+  | { ok: true; doc: Doc }
+  | { ok: false; error: string; extra?: Record<string, any> }
+> {
+  for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
+    await redis.watch(key);
+    const raw = await redis.get(key);
+    if (!raw) {
+      await redis.unwatch();
+      return { ok: false, error: "NOT_INITIALISED" };
+    }
+
+    let doc: Doc;
+    try {
+      const parsed = JSON.parse(raw);
+      if (!Array.isArray(parsed?.sections)) {
+        await redis.unwatch();
+        return { ok: false, error: "NOT_INITIALISED" };
+      }
+      doc = parsed as Doc;
+    } catch {
+      await redis.unwatch();
+      return { ok: false, error: "NOT_INITIALISED" };
+    }
+
+    const result = mutate(doc);
+    if (!result.ok) {
+      await redis.unwatch();
+      return result;
+    }
+
+    doc.version = doc.version + 1;
+    doc.updatedAt = new Date().toISOString();
+    reconcile(doc);
+
+    const execResult = await redis
+      .multi()
+      .set(key, JSON.stringify(doc), "EX", TTL_SECONDS)
+      .exec();
+
+    if (execResult !== null) {
+      return { ok: true, doc };
+    }
+    // else retry
+  }
+
+  return { ok: false, error: "CONCURRENT_UPDATE" };
+}
+
+export function findSection(doc: Doc, id: string): Section | undefined {
+  return doc.sections.find((s) => s.id === id);
+}
+
+export function findSectionIndex(doc: Doc, id: string): number {
+  return doc.sections.findIndex((s) => s.id === id);
+}

package/vendor/base/src/manifests/runtime/docstore/types.ts

@@ -0,0 +1,60 @@
+/**
+ * THE DOC STORE'S SHAPES, transcribed from the retired smart-document package
+ * (util/types.ts). The design rationale lives in _legacy/nodes/smart-document/CLAUDE.md and
+ * is worth reading whole: every field here exists to defeat one of the two failure modes of
+ * LLM-edited long documents — hallucinated addressing and stale addressing.
+ */
+
+export type SectionLevel = 1 | 2;
+
+export interface Section {
+  /** Random, server-generated, never positional and never reused. */
+  id: string;
+  level: SectionLevel;
+  /** No leading #. */
+  heading: string;
+  /** Raw markdown. May contain H3+ freely; must never contain a heading at or above the sectionize level. */
+  body: string;
+  parentId: string | null;
+  /** Sibling position. Server-managed, never an agent input. */
+  order: number;
+  /** sha256 of heading + "\n\n" + body, first 12 hex chars — the staleness check on every edit. */
+  hash: string;
+}
+
+export interface Doc {
+  sections: Section[];
+  /** Bumped on every mutation; agents detect drift by comparing across calls. */
+  version: number;
+  updatedAt: string;
+}
+
+export type ErrorCode =
+  | "STALE_SECTION"
+  | "STALE_DOC"
+  | "CONCURRENT_UPDATE"
+  | "NOT_FOUND"
+  | "NOT_UNIQUE"
+  | "INVALID_STRUCTURE"
+  | "INVALID_PLACEMENT"
+  | "INVALID_PARAMS"
+  | "NOT_INITIALISED"
+  | "UNKNOWN_METHOD";
+
+export interface ServiceError {
+  ok: false;
+  error: ErrorCode;
+  hint?: string;
+  currentHash?: string;
+  currentBody?: string;
+  currentVersion?: number;
+  matches?: number;
+}
+
+export interface ServiceSuccess {
+  ok: true;
+  version: number;
+  [key: string]: any;
+}
+
+export type ServiceResult = ServiceSuccess | ServiceError;

package/vendor/base/src/manifests/runtime/duplex/audioLane.ts

@@ -0,0 +1,165 @@
+/**
+ * THE AUDIO LANE — the platform's socket to the browser, and everything about bytes.
+ *
+ * Split from `duplex.ts` because there are TWO SOCKETS in a voice call and conflating them is the
+ * mistake this whole area invites:
+ *
+ *   duplex.ts      the VENDOR socket. The conversation with OpenAI or xAI: handshake, inbound
+ *                  event routing, tool calls, teardown. Described by `api/run.yaml`.
+ *   this file      the AUDIO LANE. The platform's own socket to the browser, bound by
+ *                  `api/audio.yaml`, and it exists for ONE reason: MCP cannot carry binary audio.
+ *
+ * Everything here is COMPUTATION OVER BYTES OR TIME, which is why it is executor code rather than
+ * anything a manifest could say: resampling is interpolation across samples, coalescing is a byte
+ * budget and a timer, barge-in is discarding a buffer rather than flushing it. All identical for
+ * every vendor, which is the test for belonging here (DECLARATIVE_NODES.md §2).
+ *
+ * These are also the parts that FAIL SILENTLY, so they are unit-tested in
+ * `tests/manifest-duplex.test.ts`: a wrong resample rate is not an error anywhere, the vendor just
+ * transcribes at the wrong speed and it reads as poor accuracy.
+ */
+/**
+ * The platform's audio lane, INJECTED so this file touches no platform global — the same
+ * arrangement `state.ts` uses for Redis. `null` when the platform has no lane, which is not an
+ * error: a voice node with nowhere to send audio still holds a valid conversation, and the
+ * transcripts still reach the workflow.
+ */
+export interface AudioLane {
+  sendAudio(conversationId: string, bytes: Buffer): void;
+  sendControl(conversationId: string, message: unknown): void;
+  setAudioDataHandler?(fn: (sessionId: string, data: ArrayBuffer) => void | Promise<void>): void;
+  setControlMessageHandler?(fn: (sessionId: string, message: any) => void | Promise<void>): void;
+  startAudioSession?(sessionId: string, conversationId: string): void;
+}
+
+
+/** Bytes per sample. PCM16 throughout, which is what every vendor here speaks. */
+const BYTES_PER_SAMPLE = 2;
+const DEFAULT_COALESCE_BYTES = 32768;
+const DEFAULT_COALESCE_MS = 50;
+/** What the browser captures at. Not configurable, because the capture side is ours. */
+export const CLIENT_RATE = 16000;
+
+/**
+ * Resample PCM16 by linear interpolation.
+ *
+ * Interpolating rather than dropping or repeating samples: nearest-neighbour at these ratios
+ * is audibly grainy, and speech recognition is measurably worse on it. Transcribed from the
+ * retired node unchanged, because it was already correct and a rewrite here would be a
+ * behaviour change disguised as a refactor.
+ */
+export function resamplePcm16(input: Buffer, fromRate: number, toRate: number): Buffer {
+  if (fromRate === toRate) return input;
+  const srcSamples = Math.floor(input.length / BYTES_PER_SAMPLE);
+  if (srcSamples === 0) return input;
+
+  const dstSamples = Math.round((srcSamples * toRate) / fromRate);
+  const output = Buffer.alloc(dstSamples * BYTES_PER_SAMPLE);
+  const ratio = srcSamples / dstSamples;
+
+  for (let i = 0; i < dstSamples; i++) {
+    const srcPos = i * ratio;
+    const srcIndex = Math.floor(srcPos);
+    const frac = srcPos - srcIndex;
+    const s0 = input.readInt16LE(srcIndex * BYTES_PER_SAMPLE);
+    const s1 = srcIndex + 1 < srcSamples ? input.readInt16LE((srcIndex + 1) * BYTES_PER_SAMPLE) : s0;
+    const sample = Math.round(s0 + frac * (s1 - s0));
+    output.writeInt16LE(Math.max(-32768, Math.min(32767, sample)), i * BYTES_PER_SAMPLE);
+  }
+  return output;
+}
+
+/**
+ * Coalesce vendor audio deltas into fewer, larger frames.
+ *
+ * `discard` is not `flush` and the difference is the whole point: on barge-in the buffered
+ * audio must be thrown away, because the user is already talking over the sentence it belongs
+ * to. Transcribed from the retired WebSocketAudioPublisher.
+ */
+export function makeAudioBuffer(
+  lane: AudioLane,
+  conversationId: string,
+  targetBytes = DEFAULT_COALESCE_BYTES,
+  maxDelayMs = DEFAULT_COALESCE_MS,
+) {
+  let chunks: Buffer[] = [];
+  let total = 0;
+  let timer: ReturnType<typeof setTimeout> | undefined;
+
+  const clear = () => {
+    if (timer) clearTimeout(timer);
+    timer = undefined;
+  };
+
+  /**
+   * SAMPLE-ALIGNED, and it must be. PCM16 is TWO BYTES PER SAMPLE, and the client decodes a frame
+   * with `new Int16Array(buffer)`, which throws outright on an odd byte length:
+   *
+   *     RangeError: byte length of Int16Array should be a multiple of 2
+   *
+   * The vendor's audio deltas are chunks of a byte stream and are NOT guaranteed to end on a sample
+   * boundary, so concatenating them freely produces odd-length frames. Any trailing odd byte is
+   * therefore held back and becomes the first byte of the next frame — which is also the only
+   * correct thing to do with it, since it is half of a real sample whose other half has not arrived.
+   * Dropping it instead would inject a click and shift the phase of everything after it.
+   */
+  const flush = () => {
+    clear();
+    if (!chunks.length) return;
+    const combined = Buffer.concat(chunks);
+
+    /**
+     * SEND A WHOLE BUFFER, NEVER A VIEW. This is what the retired node did and the drift from it
+     * was audible: nothing but a light crackle.
+     *
+     * `Buffer.concat` can return a POOLED buffer, and `subarray` returns a view whose byteLength is
+     * smaller than the ArrayBuffer behind it. A transport that takes the underlying ArrayBuffer
+     * rather than honouring the view's offset and length then puts whatever surrounds the audio on
+     * the wire, and the client decodes that as sound. The retired publisher passed
+     * `Buffer.concat(chunks)` straight through, so it never had a view to mishandle.
+     *
+     * The even case is therefore byte-for-byte what the legacy sent. Only an odd trailing byte
+     * needs slicing, and that path COPIES so the frame is still a standalone, exactly-sized buffer.
+     */
+    if (combined.length % 2 === 0) {
+      chunks = [];
+      total = 0;
+      lane.sendAudio(conversationId, combined);
+      return;
+    }
+
+    /**
+     * An odd length means the vendor's delta boundary split a sample. PCM16 is two bytes per sample
+     * and the client decodes with `new Int16Array(buffer)`, which throws on an odd length, so the
+     * trailing byte is held back to become the first byte of the next frame — it is half of a real
+     * sample whose other half has not arrived. Dropping it would click and shift the phase of
+     * everything after it.
+     */
+    const aligned = combined.length - 1;
+    if (aligned === 0) {
+      chunks = [combined];
+      total = combined.length;
+      return;
+    }
+    chunks = [Buffer.from(combined.subarray(aligned))];
+    total = 1;
+    lane.sendAudio(conversationId, Buffer.from(combined.subarray(0, aligned)));
+  };
+
+  return {
+    add(chunk: Buffer) {
+      chunks.push(chunk);
+      total += chunk.length;
+      clear();
+      if (total >= targetBytes) flush();
+      else timer = setTimeout(flush, maxDelayMs);
+    },
+    flush,
+    /** Barge-in. Drop everything held, send nothing. */
+    discard() {
+      clear();
+      chunks = [];
+      total = 0;
+    },
+  };
+}