@huynle/hub 0.8.1 → 0.8.3

package/src/central/recording/export.ts

@@ -0,0 +1,215 @@
+// REC-01 export (contract: List, read, export, delete → Export; docs/voice-recording.md). One recording: at the
+// zip's root `manifest.json`, `events.jsonl` (both sources merged, sorted by ts, then src (hub first), then
+// seq/id), `audio/<n 6 digits>.wav` (exactly as sent to STT). All: a root manifest.json of every recording plus one
+// folder `<id>/` per recording with the same three entries. Streamed: each recording's pages are read when its
+// entries are reached (the root manifest of "all" reads them once more beforehand).
+import { RECORDING_AUDIO_TYPE, RECORDING_FORMAT, RECORDING_FORMAT_VERSION, type RecordingEvent } from "../../protocol"
+import { clockAlignment, type ClockSample, type ClockSummary } from "./clock"
+import { type AudioEntry, blobTenant, type RecordingRecord, type RecordingStore } from "./store"
+import type { ZipEntry } from "./zip"
+
+export interface ExportOptions {
+  readonly version: string
+  readonly now: () => number
+}
+
+export interface SegmentManifest {
+  readonly n: number
+  readonly file?: string
+  readonly bytes?: number
+  readonly sha256?: string
+  readonly durationMs?: number
+  readonly speechMs?: number
+  readonly forced?: boolean
+  readonly startTs?: number
+  readonly endTs?: number
+  readonly text?: string
+  readonly audio: string
+  readonly redacted?: true
+}
+
+const decoder = new TextDecoder()
+const encoder = new TextEncoder()
+const audioFile = (n: number) => `audio/${String(n).padStart(6, "0")}.wav`
+
+/** A hub event ID "<podTag>.<n>" as a sort key. */
+const idKey = (id: unknown): [string, number] => {
+  const text = typeof id === "string" ? id : ""
+  const dot = text.lastIndexOf(".")
+  return [text.slice(0, dot), Number(text.slice(dot + 1)) || 0]
+}
+
+function compare(a: RecordingEvent, b: RecordingEvent): number {
+  if (a.ts! !== b.ts!) return a.ts! - b.ts!
+  if (a.src !== b.src) return a.src === "hub" ? -1 : 1
+  if (a.src === "console") return (a.seq ?? 0) - (b.seq ?? 0)
+  const [ta, na] = idKey(a.id)
+  const [tb, nb] = idKey(b.id)
+  return ta < tb ? -1 : ta > tb ? 1 : na - nb
+}
+
+interface Loaded {
+  readonly events: RecordingEvent[]
+  readonly clock: ClockSummary
+  readonly audio: Map<number, AudioEntry>
+}
+
+/** Every stored event of a recording (pages that are missing or unreadable are skipped), aligned and sorted. */
+async function load(store: RecordingStore, userID: string, record: RecordingRecord): Promise<Loaded> {
+  const index = await store.index<Record<string, unknown>>(userID, record.id)
+  const raw: RecordingEvent[] = []
+  const audio = new Map<number, AudioEntry>()
+  for (const entry of index) {
+    if (entry.key.startsWith("a")) {
+      audio.set(Number(entry.key.slice(1)), entry.value as unknown as AudioEntry)
+      continue
+    }
+    const hash = entry.value.hash
+    if (typeof hash !== "string") continue
+    const blob = await store.blobs.get(blobTenant(userID), hash).catch(() => undefined)
+    if (!blob) continue
+    const text = decoder.decode(new Uint8Array(await new Response(blob.body).arrayBuffer()))
+    for (const line of text.split("\n")) {
+      if (!line) continue
+      try {
+        raw.push(JSON.parse(line) as RecordingEvent)
+      } catch {
+        // a damaged line is skipped
+      }
+    }
+  }
+  const samples: ClockSample[] = raw.filter((e) => e.src === "console" && e.kind === "clock.sync").map((e) => ({ t0: e.t0 as number, t1: e.t1 as number, hubAt: e.hubAt as number }))
+  const clock = clockAlignment(samples, record.anchor)
+  const events = raw.map((e) => ({ ...e, ts: e.src === "console" ? clock.ts(e.t ?? 0) : (e.at ?? 0) }) as RecordingEvent)
+  events.sort(compare)
+  return { events, clock: clock.summary, audio }
+}
+
+const num = (v: unknown) => (typeof v === "number" && Number.isFinite(v) ? v : undefined)
+
+/** segments[]: the index joined with segment.closed, speech.end (aligned window), stt.result / stt.received. */
+function segments(loaded: Loaded): SegmentManifest[] {
+  const out = new Map<number, Record<string, unknown>>()
+  const at = (n: unknown) => {
+    if (typeof n !== "number" || !Number.isSafeInteger(n)) return undefined
+    let s = out.get(n)
+    if (!s) out.set(n, (s = { n }))
+    return s
+  }
+  for (const e of loaded.events) {
+    if (e.kind === "segment.closed" && e.src === "console") {
+      const s = at(e.segment)
+      if (!s) continue
+      s.durationMs ??= num(e.durationMs)
+      s.speechMs ??= num(e.speechMs)
+      if (typeof e.forced === "boolean") s.forced ??= e.forced
+      s.closedBytes ??= num(e.bytes)
+    } else if (e.kind === "speech.end" && e.src === "console") {
+      const s = at(e.segment)
+      if (!s) continue
+      s.endTs = e.ts
+      s.windowMs = num(e.durationMs)
+      s.durationMs ??= num(e.durationMs)
+      s.speechMs ??= num(e.speechMs)
+      if (typeof e.forced === "boolean") s.forced ??= e.forced
+    } else if (e.kind === "stt.result" && e.src === "hub") {
+      const s = at(e.segment)
+      if (!s) continue
+      if (typeof e.text === "string") s.text = e.text
+      if (typeof e.audio === "string") s.result = e.audio
+      if (e.redacted?.length) s.redacted = true
+    } else if (e.kind === "stt.received" && e.src === "console") {
+      const s = at(e.segment)
+      if (!s) continue
+      if (typeof e.text === "string") s.text ??= e.text
+      if (typeof e.audio === "string") s.received ??= e.audio
+    }
+  }
+  for (const n of loaded.audio.keys()) at(n)
+  return [...out.values()]
+    .sort((a, b) => (a.n as number) - (b.n as number))
+    .map((s) => {
+      const n = s.n as number
+      const stored = loaded.audio.get(n)
+      const result = (s.result ?? s.received) as string | undefined
+      const audio = stored ? "stored" : result === "stored" || result === undefined ? (result === undefined ? "off" : "failed") : result
+      const window = (s.windowMs ?? s.durationMs) as number | undefined
+      const endTs = s.endTs as number | undefined
+      return {
+        n,
+        ...(stored ? { file: audioFile(n), bytes: stored.bytes, sha256: stored.hash.replace(/^sha256:/, "") } : {}),
+        ...(s.durationMs !== undefined ? { durationMs: s.durationMs as number } : stored?.durationMs !== undefined ? { durationMs: stored.durationMs } : {}),
+        ...(s.speechMs !== undefined ? { speechMs: s.speechMs as number } : {}),
+        ...(s.forced !== undefined ? { forced: s.forced as boolean } : {}),
+        ...(endTs !== undefined && window !== undefined ? { startTs: Math.round((endTs - window) * 10) / 10 } : {}),
+        ...(endTs !== undefined ? { endTs } : {}),
+        ...(typeof s.text === "string" ? { text: s.text } : {}),
+        audio,
+        ...(s.redacted || stored?.redacted ? { redacted: true as const } : {}),
+      }
+    })
+}
+
+function manifestEntry(record: RecordingRecord, loaded: Loaded): Record<string, unknown> {
+  const list = segments(loaded)
+  const skipped = { cap: 0, failed: 0 }
+  let redacted = 0
+  for (const s of list) {
+    if (s.audio === "skipped-cap") skipped.cap++
+    else if (s.audio === "failed") skipped.failed++
+    if (s.redacted) redacted++
+  }
+  return {
+    id: record.id,
+    kind: record.kind,
+    sessionID: record.sessionID,
+    ...(record.sessionTitle !== undefined ? { sessionTitle: record.sessionTitle } : {}),
+    device: { id: record.deviceID, ...(record.deviceName !== undefined ? { name: record.deviceName } : {}) },
+    state: record.state,
+    startedAt: record.startedAt,
+    ...(record.endedAt !== undefined ? { endedAt: record.endedAt } : {}),
+    ...(record.endReason !== undefined ? { endReason: record.endReason } : {}),
+    expiresAt: record.expiresAt,
+    client: record.client,
+    ...(record.stt ? { stt: record.stt } : {}),
+    ...(record.tts ? { tts: record.tts } : {}),
+    clock: loaded.clock,
+    audio: { format: RECORDING_AUDIO_TYPE, sampleRate: 16000, channels: 1, bitsPerSample: 16 },
+    segments: list,
+    counts: { events: loaded.events.length, consoleGaps: record.consoleGaps, redactions: record.redactions, segmentsRedacted: redacted, segmentsSkipped: skipped },
+  }
+}
+
+const manifest = (entries: Record<string, unknown>[], records: readonly RecordingRecord[], options: ExportOptions) =>
+  encoder.encode(
+    `${JSON.stringify(
+      { format: RECORDING_FORMAT, version: RECORDING_FORMAT_VERSION, exportedAt: options.now(), hub: { version: options.version }, partial: records.some((r) => r.state === "active"), recordings: entries },
+      null,
+      2,
+    )}\n`,
+  )
+
+/** The three entries of one recording, under `prefix` ("" or "<id>/"). */
+async function* recordingEntries(store: RecordingStore, userID: string, record: RecordingRecord, prefix: string, options: ExportOptions): AsyncGenerator<ZipEntry> {
+  const loaded = await load(store, userID, record)
+  const entry = manifestEntry(record, loaded)
+  yield { name: `${prefix}manifest.json`, body: async () => manifest([entry], [record], options) }
+  yield { name: `${prefix}events.jsonl`, body: async () => encoder.encode(loaded.events.map((e) => JSON.stringify(e)).join("\n") + (loaded.events.length ? "\n" : "")) }
+  for (const [n, audio] of [...loaded.audio].sort((a, b) => a[0] - b[0])) {
+    const blob = await store.blobs.get(blobTenant(userID), audio.hash).catch(() => undefined)
+    if (!blob) continue
+    yield { name: `${prefix}${audioFile(n)}`, body: async () => blob.body }
+  }
+}
+
+/** The zip entries of one recording (at the root) or of several (root manifest + one folder each). */
+export async function* exportEntries(store: RecordingStore, userID: string, records: readonly RecordingRecord[], single: boolean, options: ExportOptions): AsyncGenerator<ZipEntry> {
+  if (single) {
+    yield* recordingEntries(store, userID, records[0]!, "", options)
+    return
+  }
+  const entries: Record<string, unknown>[] = []
+  for (const record of records) entries.push(manifestEntry(record, await load(store, userID, record)))
+  yield { name: "manifest.json", body: async () => manifest(entries, records, options) }
+  for (const record of records) yield* recordingEntries(store, userID, record, `${record.id}/`, options)
+}