@gmickel/gno 1.27.1 → 1.29.0

package/src/converters/adapters/shared/utf8-lines.ts

@@ -0,0 +1,141 @@
+export type Utf8LineResult =
+  | {
+      ok: true;
+      lineNumber: number;
+      text: string;
+      terminated: boolean;
+    }
+  | {
+      ok: false;
+      lineNumber: number;
+      reason: "invalid_utf8" | "line_too_large";
+      terminated: boolean;
+    };
+
+const BYTE_LF = 0x0a;
+const BYTE_CR = 0x0d;
+const UTF8_BOM = new Uint8Array([0xef, 0xbb, 0xbf]);
+
+const concatenate = (
+  parts: readonly Uint8Array[],
+  byteLength: number
+): Uint8Array => {
+  if (parts.length === 1) return parts[0] ?? new Uint8Array();
+  const bytes = new Uint8Array(byteLength);
+  let offset = 0;
+  for (const part of parts) {
+    bytes.set(part, offset);
+    offset += part.byteLength;
+  }
+  return bytes;
+};
+
+const stripLineEnvelope = (
+  bytes: Uint8Array,
+  lineNumber: number
+): Uint8Array => {
+  let start = 0;
+  let end = bytes.byteLength;
+  if (
+    lineNumber === 1 &&
+    end >= UTF8_BOM.byteLength &&
+    UTF8_BOM.every((byte, index) => bytes[index] === byte)
+  ) {
+    start = UTF8_BOM.byteLength;
+  }
+  if (end > start && bytes[end - 1] === BYTE_CR) end -= 1;
+  return bytes.subarray(start, end);
+};
+
+const decodeLine = (
+  parts: readonly Uint8Array[],
+  byteLength: number,
+  lineNumber: number,
+  terminated: boolean
+): Utf8LineResult => {
+  const bytes = stripLineEnvelope(concatenate(parts, byteLength), lineNumber);
+  try {
+    return {
+      ok: true,
+      lineNumber,
+      text: new TextDecoder("utf-8", { fatal: true }).decode(bytes),
+      terminated,
+    };
+  } catch {
+    return {
+      ok: false,
+      lineNumber,
+      reason: "invalid_utf8",
+      terminated,
+    };
+  }
+};
+
+/**
+ * Split an async byte source without retaining an unbounded physical line.
+ * Invalid UTF-8 and oversized lines are isolated so later siblings can proceed.
+ */
+export async function* readBoundedUtf8Lines(
+  source: AsyncIterable<Uint8Array>,
+  maxLineBytes: number
+): AsyncGenerator<Utf8LineResult> {
+  const safeMaxLineBytes = Math.max(1, Math.floor(maxLineBytes));
+  let parts: Uint8Array[] = [];
+  let lineBytes = 0;
+  let lineNumber = 1;
+  let oversized = false;
+
+  const reset = (): void => {
+    parts = [];
+    lineBytes = 0;
+    oversized = false;
+    lineNumber += 1;
+  };
+
+  for await (const chunk of source) {
+    let segmentStart = 0;
+    for (let index = 0; index < chunk.byteLength; index += 1) {
+      if (chunk[index] !== BYTE_LF) continue;
+      const segment = chunk.subarray(segmentStart, index);
+      if (!oversized && lineBytes + segment.byteLength <= safeMaxLineBytes) {
+        if (segment.byteLength > 0) parts.push(segment.slice());
+        lineBytes += segment.byteLength;
+      } else {
+        oversized = true;
+      }
+      if (oversized) {
+        yield {
+          ok: false,
+          lineNumber,
+          reason: "line_too_large",
+          terminated: true,
+        };
+      } else {
+        yield decodeLine(parts, lineBytes, lineNumber, true);
+      }
+      reset();
+      segmentStart = index + 1;
+    }
+
+    const remainder = chunk.subarray(segmentStart);
+    if (!oversized && lineBytes + remainder.byteLength <= safeMaxLineBytes) {
+      if (remainder.byteLength > 0) parts.push(remainder.slice());
+      lineBytes += remainder.byteLength;
+    } else if (remainder.byteLength > 0) {
+      oversized = true;
+      parts = [];
+      lineBytes = 0;
+    }
+  }
+
+  if (oversized) {
+    yield {
+      ok: false,
+      lineNumber,
+      reason: "line_too_large",
+      terminated: false,
+    };
+  } else if (lineBytes > 0) {
+    yield decodeLine(parts, lineBytes, lineNumber, false);
+  }
+}

package/src/converters/adapters/transcript/adapter.ts

@@ -0,0 +1,295 @@
+import type {
+  RecordAdapter,
+  RecordAdapterEvent,
+  RecordAdapterInput,
+  RecordAdapterRecord,
+} from "../../types";
+import type {
+  TranscriptAdapterOptions,
+  TranscriptFormat,
+  TranscriptParseEvent,
+  TranscriptSegment,
+} from "./model";
+
+import {
+  adapterLineByteLimit,
+  canonicalJson,
+  hashRecordValue,
+  safeInlineText,
+  sourceNamespace,
+} from "../shared/record-utils";
+import {
+  readBoundedUtf8Lines,
+  type Utf8LineResult,
+} from "../shared/utf8-lines";
+import { parseJsonTranscript } from "./json";
+import { parseTranscriptAdapterOptions } from "./model";
+import { parseTextTranscript } from "./text";
+import { parseTimedTranscript } from "./timed";
+
+const ADAPTER_ID = "adapter/transcript";
+const ADAPTER_VERSION = "1.0.0";
+const MAX_JSON_TRANSCRIPT_CHARS = 16 * 1024 * 1024;
+
+const resolvedFormat = (
+  input: Pick<RecordAdapterInput, "ext" | "mime">,
+  requested: TranscriptFormat
+): Exclude<TranscriptFormat, "auto"> | undefined => {
+  if (requested !== "auto") return requested;
+  if (input.mime === "text/vtt" || input.ext === ".vtt") return "vtt";
+  if (
+    input.mime === "application/x-subrip" ||
+    input.mime === "text/srt" ||
+    input.ext === ".srt"
+  ) {
+    return "srt";
+  }
+  return undefined;
+};
+
+const failureEvent = (
+  event: Extract<TranscriptParseEvent, { ok: false }>
+): RecordAdapterEvent => ({
+  type: "failure",
+  failure: {
+    code: event.tooLarge ? "RECORD_TOO_LARGE" : "MALFORMED_RECORD",
+    message: "Transcript record could not be converted.",
+    retryable: event.retryable,
+    sourceLocator: event.sourceLocator,
+  },
+});
+
+const fileTitle = (relativePath: string): string => {
+  const name = relativePath.replaceAll("\\", "/").split("/").at(-1);
+  return (name?.replace(/\.[^.]+$/u, "") || "Transcript").normalize("NFC");
+};
+
+const localIdentity = (
+  segment: TranscriptSegment,
+  occurrences: Map<string, number>
+): string => {
+  if (segment.externalId) {
+    return `external:${hashRecordValue(
+      "gno-transcript-external-id-v1",
+      segment.externalId
+    )}`;
+  }
+  if (segment.start) {
+    const key = canonicalJson({
+      end: segment.end,
+      speaker: segment.speaker,
+      start: segment.start,
+    });
+    const count = (occurrences.get(key) ?? 0) + 1;
+    occurrences.set(key, count);
+    return `time:${hashRecordValue("gno-transcript-time-id-v1", key)}:${count}`;
+  }
+  if (segment.anchorKind === "record") {
+    return `record:${hashRecordValue(
+      "gno-transcript-record-id-v1",
+      segment.anchorValue
+    )}`;
+  }
+  return `content:${hashRecordValue(
+    "gno-transcript-content-id-v1",
+    canonicalJson({
+      speaker: segment.speaker,
+      text: segment.text,
+    })
+  )}`;
+};
+
+const segmentRecord = (
+  segment: TranscriptSegment,
+  input: RecordAdapterInput,
+  occurrences: Map<string, number>
+): RecordAdapterRecord => {
+  const sessionTitle = segment.sessionTitle ?? fileTitle(input.relativePath);
+  const label = segment.speaker
+    ? `${sessionTitle} — ${segment.speaker}`
+    : sessionTitle;
+  const details: string[] = [];
+  if (segment.speaker)
+    details.push(`**Speaker:** ${safeInlineText(segment.speaker)}`);
+  if (segment.start) {
+    details.push(
+      `**Time:** ${segment.start}${segment.end ? ` → ${segment.end}` : ""}`
+    );
+  }
+  const markdown = [
+    `# ${safeInlineText(label)}`,
+    ...details,
+    segment.text,
+  ].join("\n\n");
+  const participants = [
+    ...(segment.participants ?? []),
+    ...(segment.speaker ? [segment.speaker] : []),
+  ];
+  const uniqueParticipants = [
+    ...new Set(participants.map((value) => value.normalize("NFC").trim())),
+  ].filter(Boolean);
+  const anchors: RecordAdapterRecord["anchors"] = [
+    {
+      kind: segment.anchorKind,
+      value: segment.anchorValue,
+      endValue: segment.endAnchorValue,
+    },
+  ];
+  if (segment.start) {
+    anchors.push({
+      kind: "timestamp",
+      value: segment.start,
+      endValue: segment.end,
+    });
+  }
+  return {
+    stableId: `transcript:${sourceNamespace(input)}:${localIdentity(
+      segment,
+      occurrences
+    )}`,
+    sourceLocator: segment.sourceLocator,
+    sourceHash: hashRecordValue(
+      "gno-transcript-segment-source-v1",
+      canonicalJson(segment)
+    ),
+    title: label,
+    markdown,
+    metadata: {
+      author: segment.speaker,
+      participants:
+        uniqueParticipants.length > 0 ? uniqueParticipants : undefined,
+      categories: ["transcript"],
+      dateFields: segment.dateFields,
+      sessionId: segment.sessionId,
+    },
+    anchors,
+  };
+};
+
+const jsonEvents = async function* (
+  input: RecordAdapterInput
+): AsyncGenerator<TranscriptParseEvent> {
+  const parts: string[] = [];
+  let characters = 0;
+  let lastLine = 0;
+  const maxCharacters = Math.min(
+    MAX_JSON_TRANSCRIPT_CHARS,
+    input.limits.maxTotalChars
+  );
+  for await (const line of readBoundedUtf8Lines(
+    input.open(),
+    adapterLineByteLimit(input)
+  )) {
+    lastLine = line.lineNumber;
+    if (!line.ok) {
+      yield {
+        ok: false,
+        sourceLocator: `line:${line.lineNumber}`,
+        retryable: !line.terminated,
+        tooLarge: line.reason === "line_too_large",
+      };
+      return;
+    }
+    characters += line.text.length + 1;
+    if (characters > maxCharacters) {
+      yield {
+        ok: false,
+        sourceLocator: `lines:1-${line.lineNumber}`,
+        retryable: false,
+        tooLarge: true,
+      };
+      return;
+    }
+    parts.push(line.text);
+  }
+  if (parts.length === 0) {
+    yield {
+      ok: false,
+      sourceLocator: lastLine > 0 ? `lines:1-${lastLine}` : "record:root",
+      retryable: false,
+    };
+    return;
+  }
+  yield* parseJsonTranscript(parts.join("\n"));
+};
+
+const parseEvents = (
+  input: RecordAdapterInput,
+  format: Exclude<TranscriptFormat, "auto">
+): AsyncIterable<TranscriptParseEvent> => {
+  if (format === "json") return jsonEvents(input);
+  const lines: AsyncIterable<Utf8LineResult> = readBoundedUtf8Lines(
+    input.open(),
+    adapterLineByteLimit(input)
+  );
+  if (format === "text") return parseTextTranscript(lines);
+  return parseTimedTranscript(lines, format);
+};
+
+export const createTranscriptAdapter = (
+  options: TranscriptAdapterOptions = {}
+): RecordAdapter => {
+  const { format: requested } = parseTranscriptAdapterOptions(options);
+  return {
+    id: ADAPTER_ID,
+    version: ADAPTER_VERSION,
+    configurationFingerprint: hashRecordValue(
+      "gno-transcript-config-v1",
+      requested
+    ),
+    canHandle: (mime, ext) => {
+      if (requested === "json")
+        return mime === "application/json" || ext === ".json";
+      if (requested === "text") return mime === "text/plain" || ext === ".txt";
+      if (requested === "vtt") return mime === "text/vtt" || ext === ".vtt";
+      if (requested === "srt")
+        return (
+          mime === "application/x-subrip" ||
+          mime === "text/srt" ||
+          ext === ".srt"
+        );
+      return Boolean(resolvedFormat({ mime, ext }, requested));
+    },
+    records: async function* (
+      input: RecordAdapterInput
+    ): AsyncGenerator<RecordAdapterEvent> {
+      const format = resolvedFormat(input, requested);
+      if (!format) {
+        yield failureEvent({ ok: false, retryable: false });
+        yield { type: "snapshot", state: "partial" };
+        return;
+      }
+      const occurrences = new Map<string, number>();
+      let hadFailure = false;
+      try {
+        for await (const event of parseEvents(input, format)) {
+          if (!event.ok) {
+            hadFailure = true;
+            yield failureEvent(event);
+            continue;
+          }
+          yield {
+            type: "record",
+            record: segmentRecord(event.segment, input, occurrences),
+          };
+        }
+      } catch {
+        hadFailure = true;
+        yield {
+          type: "failure",
+          failure: {
+            code: "ADAPTER_FAILURE",
+            message: "Transcript source could not be read completely.",
+            retryable: true,
+          },
+        };
+      }
+      yield {
+        type: "snapshot",
+        state: hadFailure ? "partial" : "complete",
+      };
+    },
+  };
+};
+
+export const transcriptAdapter = createTranscriptAdapter();

package/src/converters/adapters/transcript/json.ts

@@ -0,0 +1,184 @@
+import { scalarList, scalarText } from "../shared/record-utils";
+import {
+  cleanTranscriptText,
+  parseTranscriptTimestamp,
+  type TranscriptParseEvent,
+  type TranscriptSegment,
+} from "./model";
+
+const ownValue = (
+  record: Record<string, unknown>,
+  names: readonly string[]
+): unknown => {
+  for (const name of names) {
+    if (Object.hasOwn(record, name)) return record[name];
+  }
+  return undefined;
+};
+
+const objectValue = (value: unknown): Record<string, unknown> | undefined =>
+  value && typeof value === "object" && !Array.isArray(value)
+    ? (value as Record<string, unknown>)
+    : undefined;
+
+const personText = (value: unknown): string | undefined => {
+  const scalar = scalarText(value);
+  if (scalar) return scalar;
+  const object = objectValue(value);
+  return object
+    ? scalarText(ownValue(object, ["name", "displayName", "label", "id"]))
+    : undefined;
+};
+
+const peopleList = (value: unknown): string[] | undefined => {
+  if (!Array.isArray(value)) return scalarList(value);
+  const people = value
+    .map(personText)
+    .filter((person): person is string => Boolean(person));
+  return people.length > 0 ? people : undefined;
+};
+
+const segmentArray = (
+  root: unknown
+):
+  | {
+      segments: unknown[];
+      pointerName: string;
+      session: Record<string, unknown>;
+    }
+  | undefined => {
+  if (Array.isArray(root)) {
+    return { segments: root, pointerName: "", session: {} };
+  }
+  const rootObject = objectValue(root);
+  if (!rootObject) return undefined;
+  const nested = objectValue(ownValue(rootObject, ["transcript"]));
+  const session = nested ?? rootObject;
+  for (const name of ["segments", "utterances", "items"] as const) {
+    const value = ownValue(session, [name]);
+    if (Array.isArray(value)) {
+      return {
+        segments: value,
+        pointerName: nested ? `transcript/${name}` : name,
+        session,
+      };
+    }
+  }
+  return undefined;
+};
+
+const sessionDateFields = (
+  session: Record<string, unknown>
+): Record<string, string> | undefined => {
+  const fields: Record<string, string> = {};
+  for (const [name, aliases] of [
+    ["recorded", ["recordedAt", "recorded_at", "date"]],
+    ["created", ["createdAt", "created_at"]],
+  ] as const) {
+    const value = scalarText(ownValue(session, aliases));
+    if (value) fields[name] = value;
+  }
+  return Object.keys(fields).length > 0 ? fields : undefined;
+};
+
+export function* parseJsonTranscript(
+  text: string
+): Generator<TranscriptParseEvent> {
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(text);
+  } catch {
+    const finalCharacter = text.trimEnd().at(-1);
+    yield {
+      ok: false,
+      retryable: finalCharacter !== "}" && finalCharacter !== "]",
+    };
+    return;
+  }
+  const container = segmentArray(parsed);
+  if (!container) {
+    yield { ok: false, retryable: false };
+    return;
+  }
+
+  const sessionId = scalarText(
+    ownValue(container.session, ["sessionId", "session_id", "id"])
+  );
+  const sessionTitle = scalarText(
+    ownValue(container.session, ["title", "name"])
+  );
+  const participants = peopleList(
+    ownValue(container.session, ["participants", "speakers"])
+  );
+  const dateFields = sessionDateFields(container.session);
+
+  for (const [index, candidate] of container.segments.entries()) {
+    const pointer = `/${container.pointerName ? `${container.pointerName}/` : ""}${index}`;
+    const sourceLocator = `record:${pointer.slice(1) || index}`;
+    const record = objectValue(candidate);
+    if (!record) {
+      yield { ok: false, sourceLocator, retryable: false };
+      continue;
+    }
+    const rawText = scalarText(
+      ownValue(record, ["text", "content", "transcript"])
+    );
+    if (!rawText) {
+      yield { ok: false, sourceLocator, retryable: false };
+      continue;
+    }
+    const cleaned = cleanTranscriptText(rawText);
+    if (!cleaned.text) {
+      yield { ok: false, sourceLocator, retryable: false };
+      continue;
+    }
+
+    const rawStart = ownValue(record, [
+      "start",
+      "startTime",
+      "start_time",
+      "startSeconds",
+    ]);
+    const rawEnd = ownValue(record, [
+      "end",
+      "endTime",
+      "end_time",
+      "endSeconds",
+    ]);
+    const start =
+      rawStart === undefined ? undefined : parseTranscriptTimestamp(rawStart);
+    const end =
+      rawEnd === undefined ? undefined : parseTranscriptTimestamp(rawEnd);
+    if (
+      (rawStart !== undefined && !start) ||
+      (rawEnd !== undefined && !end) ||
+      (start && end && end.milliseconds < start.milliseconds)
+    ) {
+      yield { ok: false, sourceLocator, retryable: false };
+      continue;
+    }
+
+    const speaker =
+      personText(
+        ownValue(record, ["speaker", "speakerName", "speaker_name", "author"])
+      ) ?? cleaned.speaker;
+    const externalId = scalarText(
+      ownValue(record, ["id", "segmentId", "segment_id", "utteranceId"])
+    );
+    const segment: TranscriptSegment = {
+      externalId,
+      text: cleaned.text,
+      speaker,
+      start: start?.text,
+      end: end?.text,
+      sourceLocator,
+      anchorKind: "record",
+      anchorValue: pointer,
+      sessionId,
+      sessionTitle,
+      participants,
+      dateFields,
+    };
+    yield { ok: true, segment };
+  }
+}