@gmickel/gno 1.27.0 → 1.28.0

package/src/converters/adapters/transcript/model.ts

@@ -0,0 +1,171 @@
+import {
+  decodeHtmlEntitiesOnce,
+  htmlFragmentToText,
+} from "../shared/html-text";
+import {
+  safeInlineText,
+  safeMarkdownText,
+  scalarText,
+} from "../shared/record-utils";
+
+export type TranscriptFormat = "auto" | "json" | "srt" | "text" | "vtt";
+
+export interface TranscriptAdapterOptions {
+  /** Generic JSON and text sources require explicit selection at registration. */
+  format?: TranscriptFormat;
+}
+
+export interface TranscriptSegment {
+  externalId?: string;
+  text: string;
+  speaker?: string;
+  start?: string;
+  end?: string;
+  sourceLocator: string;
+  anchorKind: "cue" | "line" | "record";
+  anchorValue: string;
+  endAnchorValue?: string;
+  sessionId?: string;
+  sessionTitle?: string;
+  participants?: string[];
+  dateFields?: Record<string, string>;
+}
+
+export type TranscriptParseEvent =
+  | { ok: true; segment: TranscriptSegment }
+  | {
+      ok: false;
+      sourceLocator?: string;
+      retryable: boolean;
+      tooLarge?: boolean;
+    };
+
+const TIMESTAMP_PATTERN = /^(?:(\d{1,}):)?([0-5]?\d):([0-5]\d)[,.](\d{3})$/;
+
+export const parseTranscriptTimestamp = (
+  value: unknown
+): { text: string; milliseconds: number } | undefined => {
+  if (typeof value === "number" && Number.isFinite(value) && value >= 0) {
+    const milliseconds = Math.round(value * 1_000);
+    if (Number.isSafeInteger(milliseconds)) {
+      return { text: formatTranscriptTimestamp(milliseconds), milliseconds };
+    }
+    return undefined;
+  }
+  const scalar = scalarText(value);
+  if (!scalar) return undefined;
+  if (/^\d+(?:\.\d+)?$/.test(scalar)) {
+    const milliseconds = Math.round(Number(scalar) * 1_000);
+    if (Number.isSafeInteger(milliseconds)) {
+      return { text: formatTranscriptTimestamp(milliseconds), milliseconds };
+    }
+  }
+  const match = TIMESTAMP_PATTERN.exec(scalar);
+  if (!match) return undefined;
+  const hours = Number(match[1] ?? "0");
+  const minutes = Number(match[2]);
+  const seconds = Number(match[3]);
+  const millisecondsPart = Number(match[4]);
+  const milliseconds =
+    (hours * 60 * 60 + minutes * 60 + seconds) * 1_000 + millisecondsPart;
+  if (!Number.isSafeInteger(milliseconds)) return undefined;
+  return { text: formatTranscriptTimestamp(milliseconds), milliseconds };
+};
+
+export const formatTranscriptTimestamp = (milliseconds: number): string => {
+  const totalSeconds = Math.floor(milliseconds / 1_000);
+  const hours = Math.floor(totalSeconds / 3_600);
+  const minutes = Math.floor((totalSeconds % 3_600) / 60);
+  const seconds = totalSeconds % 60;
+  const millis = milliseconds % 1_000;
+  return `${String(hours).padStart(2, "0")}:${String(minutes).padStart(2, "0")}:${String(seconds).padStart(2, "0")}.${String(millis).padStart(3, "0")}`;
+};
+
+const VALID_TRANSCRIPT_FORMATS = new Set<TranscriptFormat>([
+  "auto",
+  "json",
+  "srt",
+  "text",
+  "vtt",
+]);
+
+export const parseTranscriptAdapterOptions = (
+  options: TranscriptAdapterOptions
+): Required<TranscriptAdapterOptions> => {
+  const format = options.format ?? "auto";
+  if (!VALID_TRANSCRIPT_FORMATS.has(format)) {
+    throw new Error("Unsupported transcript adapter format.");
+  }
+  return { format };
+};
+
+const vttVoiceSpeaker = (value: string): string | undefined => {
+  const source = value.trimStart();
+  if (
+    source.length < 3 ||
+    source[0] !== "<" ||
+    source[1]?.toLowerCase() !== "v"
+  )
+    return undefined;
+  const end = source.indexOf(">", 2);
+  if (end < 0) return undefined;
+  let cursor = 2;
+  if (source[cursor] === ".") {
+    while (
+      cursor < end &&
+      source[cursor] !== " " &&
+      source[cursor] !== "\t" &&
+      source[cursor] !== "\n" &&
+      source[cursor] !== "\r"
+    )
+      cursor += 1;
+  }
+  if (
+    source[cursor] !== " " &&
+    source[cursor] !== "\t" &&
+    source[cursor] !== "\n" &&
+    source[cursor] !== "\r"
+  )
+    return undefined;
+  while (
+    cursor < end &&
+    (source[cursor] === " " ||
+      source[cursor] === "\t" ||
+      source[cursor] === "\n" ||
+      source[cursor] === "\r")
+  )
+    cursor += 1;
+  const speaker = source.slice(cursor, end).trim();
+  if (!speaker || speaker.length > 80) return undefined;
+  return safeInlineText(decodeHtmlEntitiesOnce(speaker));
+};
+
+const speakerPrefix = (
+  value: string
+): { speaker: string; text: string } | undefined => {
+  const separator = value.indexOf(":");
+  if (separator <= 0 || separator > 80) return undefined;
+  const speaker = value.slice(0, separator);
+  if (speaker.includes("\n")) return undefined;
+  const remainder = value.slice(separator + 1);
+  if (
+    remainder.length === 0 ||
+    (remainder[0] !== " " &&
+      remainder[0] !== "\t" &&
+      remainder[0] !== "\n" &&
+      remainder[0] !== "\r")
+  )
+    return undefined;
+  const text = remainder.trim();
+  if (!text) return undefined;
+  return { speaker: safeInlineText(speaker), text };
+};
+
+export const cleanTranscriptText = (
+  value: string
+): { text: string; speaker?: string } => {
+  const speaker = vttVoiceSpeaker(value);
+  const cleaned = safeMarkdownText(htmlFragmentToText(value)).trim();
+  if (speaker) return { text: cleaned, speaker };
+  return speakerPrefix(cleaned) ?? { text: cleaned };
+};

package/src/converters/adapters/transcript/text.ts

@@ -0,0 +1,58 @@
+import type { Utf8LineResult } from "../shared/utf8-lines";
+
+import {
+  cleanTranscriptText,
+  parseTranscriptTimestamp,
+  type TranscriptParseEvent,
+  type TranscriptSegment,
+} from "./model";
+
+const TIMESTAMP_PREFIX = /^\[([^\]]{1,32})\]\s*(.*)$/su;
+
+export async function* parseTextTranscript(
+  source: AsyncIterable<Utf8LineResult>
+): AsyncGenerator<TranscriptParseEvent> {
+  for await (const line of source) {
+    if (!line.ok) {
+      yield {
+        ok: false,
+        sourceLocator: `line:${line.lineNumber}`,
+        retryable: !line.terminated,
+        tooLarge: line.reason === "line_too_large",
+      };
+      continue;
+    }
+    const raw = line.text.trim();
+    if (!raw) continue;
+    const timestampPrefix = TIMESTAMP_PREFIX.exec(raw);
+    const timestamp = timestampPrefix
+      ? parseTranscriptTimestamp(timestampPrefix[1])
+      : undefined;
+    if (timestampPrefix && !timestamp) {
+      yield {
+        ok: false,
+        sourceLocator: `line:${line.lineNumber}`,
+        retryable: false,
+      };
+      continue;
+    }
+    const cleaned = cleanTranscriptText(timestampPrefix?.[2] ?? raw);
+    if (!cleaned.text) {
+      yield {
+        ok: false,
+        sourceLocator: `line:${line.lineNumber}`,
+        retryable: false,
+      };
+      continue;
+    }
+    const segment: TranscriptSegment = {
+      text: cleaned.text,
+      speaker: cleaned.speaker,
+      start: timestamp?.text,
+      sourceLocator: `line:${line.lineNumber}`,
+      anchorKind: "line",
+      anchorValue: String(line.lineNumber),
+    };
+    yield { ok: true, segment };
+  }
+}

package/src/converters/adapters/transcript/timed.ts

@@ -0,0 +1,152 @@
+import type { Utf8LineResult } from "../shared/utf8-lines";
+
+import {
+  cleanTranscriptText,
+  parseTranscriptTimestamp,
+  type TranscriptParseEvent,
+  type TranscriptSegment,
+} from "./model";
+
+interface SourceLine {
+  lineNumber: number;
+  text: string;
+}
+
+const VTT_TIMING_PATTERN =
+  /^(\d{1,}:\d{2}:\d{2}\.\d{3}|\d{1,2}:\d{2}\.\d{3})\s+-->\s+(\d{1,}:\d{2}:\d{2}\.\d{3}|\d{1,2}:\d{2}\.\d{3})(?:\s+.*)?$/u;
+const SRT_TIMING_PATTERN =
+  /^(\d{1,}:\d{2}:\d{2}[,.]\d{3})\s+-->\s+(\d{1,}:\d{2}:\d{2}[,.]\d{3})(?:\s+.*)?$/u;
+
+const locator = (lines: readonly SourceLine[]): string => {
+  const first = lines[0]?.lineNumber ?? 1;
+  const last = lines.at(-1)?.lineNumber ?? first;
+  return `lines:${first}-${last}`;
+};
+
+const malformed = (
+  lines: readonly SourceLine[],
+  retryable = false
+): TranscriptParseEvent => ({
+  ok: false,
+  sourceLocator: locator(lines),
+  retryable,
+});
+
+const parseCueBlock = (
+  lines: readonly SourceLine[],
+  format: "srt" | "vtt"
+): TranscriptParseEvent | undefined => {
+  if (lines.length === 0) return undefined;
+  const first = lines[0]?.text.trim() ?? "";
+  if (format === "vtt" && /^(?:NOTE|STYLE|REGION)(?:\s|$)/u.test(first)) {
+    return undefined;
+  }
+
+  const timingPattern =
+    format === "vtt" ? VTT_TIMING_PATTERN : SRT_TIMING_PATTERN;
+  let timingIndex = 0;
+  let externalId: string | undefined;
+  if (!timingPattern.test(first)) {
+    if (!lines[1] || !timingPattern.test(lines[1].text.trim())) {
+      return malformed(lines);
+    }
+    externalId = first || undefined;
+    timingIndex = 1;
+  }
+  const timing = timingPattern.exec(lines[timingIndex]?.text.trim() ?? "");
+  if (!(timing?.[1] && timing[2])) return malformed(lines);
+  const start = parseTranscriptTimestamp(timing[1]);
+  const end = parseTranscriptTimestamp(timing[2]);
+  if (!(start && end) || end.milliseconds < start.milliseconds) {
+    return malformed(lines);
+  }
+  const payload = lines
+    .slice(timingIndex + 1)
+    .map((line) => line.text)
+    .join("\n");
+  const cleaned = cleanTranscriptText(payload);
+  if (!cleaned.text) return malformed(lines);
+  const firstLine = lines[0]?.lineNumber ?? 1;
+  const lastLine = lines.at(-1)?.lineNumber ?? firstLine;
+  const segment: TranscriptSegment = {
+    externalId,
+    text: cleaned.text,
+    speaker: cleaned.speaker,
+    start: start.text,
+    end: end.text,
+    sourceLocator: `lines:${firstLine}-${lastLine}`,
+    anchorKind: "cue",
+    anchorValue: externalId ?? start.text,
+    endAnchorValue: end.text,
+  };
+  return { ok: true, segment };
+};
+
+export async function* parseTimedTranscript(
+  source: AsyncIterable<Utf8LineResult>,
+  format: "srt" | "vtt"
+): AsyncGenerator<TranscriptParseEvent> {
+  let block: SourceLine[] = [];
+  let vttHeaderChecked = format !== "vtt";
+  let vttHeaderEnded = format !== "vtt";
+  let headerInvalid = false;
+  let lastLineNumber = 0;
+
+  const flush = (): TranscriptParseEvent | undefined => {
+    const event = parseCueBlock(block, format);
+    block = [];
+    return event;
+  };
+
+  for await (const line of source) {
+    lastLineNumber = line.lineNumber;
+    if (!line.ok) {
+      const pending = flush();
+      if (pending) yield pending;
+      yield {
+        ok: false,
+        sourceLocator: `line:${line.lineNumber}`,
+        retryable: !line.terminated,
+        tooLarge: line.reason === "line_too_large",
+      };
+      continue;
+    }
+
+    if (!vttHeaderChecked) {
+      vttHeaderChecked = true;
+      if (!line.text.startsWith("WEBVTT")) {
+        headerInvalid = true;
+        yield {
+          ok: false,
+          sourceLocator: `line:${line.lineNumber}`,
+          retryable: false,
+        };
+      }
+      continue;
+    }
+    if (headerInvalid) continue;
+    if (!vttHeaderEnded) {
+      if (line.text.trim() === "") vttHeaderEnded = true;
+      continue;
+    }
+
+    if (line.text.trim() === "") {
+      const event = flush();
+      if (event) yield event;
+      continue;
+    }
+    block.push({ lineNumber: line.lineNumber, text: line.text });
+  }
+  if (!headerInvalid && !vttHeaderEnded) {
+    yield {
+      ok: false,
+      sourceLocator: `lines:1-${Math.max(1, lastLineNumber)}`,
+      retryable: true,
+    };
+    return;
+  }
+  if (!headerInvalid) {
+    const event = flush();
+    if (event) yield event;
+  }
+}

package/src/converters/index.ts

@@ -47,5 +47,15 @@ export type {
   ConvertResult,
   ConvertWarning,
   PipelineResult,
+  RecordAdapter,
+  RecordAdapterEvent,
+  RecordAdapterFailure,
+  RecordAdapterFailureCode,
+  RecordAdapterInput,
+  RecordAdapterLimits,
+  RecordAdapterRecord,
+  RecordAnchor,
+  RecordAttachmentInventoryItem,
+  RecordMetadata,
 } from "./types";
-export { DEFAULT_LIMITS } from "./types";
+export { DEFAULT_LIMITS, DEFAULT_RECORD_ADAPTER_LIMITS } from "./types";

package/src/converters/mime.ts

@@ -35,6 +35,14 @@ const EXTENSION_MAP: Record<string, string> = {
   ".pptx":
     "application/vnd.openxmlformats-officedocument.presentationml.presentation",
   ".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+  ".jsonl": "application/x-ndjson",
+  ".ndjson": "application/x-ndjson",
+  ".eml": "message/rfc822",
+  ".mbox": "application/mbox",
+  ".ics": "text/calendar",
+  ".vtt": "text/vtt",
+  ".srt": "application/x-subrip",
+  ".browser-export": "application/x-gno-browser-export+json",
 };
 
 /** OOXML extension to MIME mapping */

package/src/converters/pipeline.ts

@@ -13,7 +13,12 @@
  * CRITICAL: Canonicalization is ONLY done here, not in individual converters.
  */
 
-import type { ConversionArtifact, ConvertInput, PipelineResult } from "./types";
+import type {
+  ConversionArtifact,
+  ConvertInput,
+  PipelineResult,
+  RecordAdapter,
+} from "./types";
 
 import { canonicalize, mirrorHash } from "./canonicalize";
 import { internalError, outputTooLargeError } from "./errors";
@@ -135,6 +140,15 @@ export class ConversionPipeline {
     }
   }
 
+  /** Select a streaming multi-record adapter without reading the full file. */
+  async selectRecordAdapter(
+    mime: string,
+    ext: string
+  ): Promise<RecordAdapter | undefined> {
+    const registry = await this.ensureRegistry();
+    return registry.selectRecordAdapter(mime, ext);
+  }
+
   /**
    * List available converters.
    */

package/src/converters/registry.ts

@@ -3,12 +3,18 @@
  * PRD §8.6 - Converter registry
  */
 
-import type { Converter, ConvertInput, ConvertResult } from "./types";
+import type {
+  Converter,
+  ConvertInput,
+  ConvertResult,
+  RecordAdapter,
+} from "./types";
 
 import { unsupportedError } from "./errors";
 
 export class ConverterRegistry {
   private readonly converters: Converter[] = [];
+  private readonly recordAdapters: RecordAdapter[] = [];
 
   /**
    * Register a converter. Order matters - first match wins.
@@ -34,6 +40,25 @@ export class ConverterRegistry {
     return this.converters.map((c) => c.id);
   }
 
+  /** Register a streaming container adapter without changing converter routing. */
+  registerRecordAdapter(adapter: RecordAdapter): void {
+    this.recordAdapters.push(adapter);
+  }
+
+  /** Select the first streaming adapter that handles a MIME/extension pair. */
+  selectRecordAdapter(mime: string, ext: string): RecordAdapter | undefined {
+    const normalizedMime = mime.toLowerCase();
+    const normalizedExt = ext.toLowerCase();
+    return this.recordAdapters.find((adapter) =>
+      adapter.canHandle(normalizedMime, normalizedExt)
+    );
+  }
+
+  /** List streaming adapters independently of byte-oriented converters. */
+  listRecordAdapters(): string[] {
+    return this.recordAdapters.map((adapter) => adapter.id);
+  }
+
   /**
    * Convert a file using the appropriate converter.
    */
@@ -63,12 +88,23 @@ export async function createDefaultRegistry(): Promise<ConverterRegistry> {
   const { markitdownAdapter } = await import("./adapters/markitdownTs/adapter");
   const { officeparserAdapter } =
     await import("./adapters/officeparser/adapter");
+  const { jsonlAdapter } = await import("./adapters/jsonl/adapter");
+  const { transcriptAdapter } = await import("./adapters/transcript/adapter");
+  const { emailRecordAdapter } = await import("./adapters/email/adapter");
+  const { icalAdapter } = await import("./adapters/ical/adapter");
+  const { browserExportAdapter } =
+    await import("./adapters/browser-export/adapter");
 
   // Register in priority order
   registry.register(markdownConverter);
   registry.register(plaintextConverter);
   registry.register(markitdownAdapter);
   registry.register(officeparserAdapter);
+  registry.registerRecordAdapter(jsonlAdapter);
+  registry.registerRecordAdapter(transcriptAdapter);
+  registry.registerRecordAdapter(emailRecordAdapter);
+  registry.registerRecordAdapter(icalAdapter);
+  registry.registerRecordAdapter(browserExportAdapter);
 
   return registry;
 }

package/src/converters/types.ts

@@ -94,6 +94,132 @@ export interface Converter {
   convert(input: ConvertInput): Promise<ConvertResult>;
 }
 
+/** Declarative limits applied to every multi-record container adapter. */
+export interface RecordAdapterLimits {
+  /** End-to-end adapter deadline; production callers always set it. */
+  timeoutMs?: number;
+  /** Maximum bytes an adapter may read from the container source. */
+  maxSourceBytes: number;
+  /** Maximum canonical characters accepted for one logical record. */
+  maxRecordChars: number;
+  /** Maximum canonical metadata characters accepted for one logical record. */
+  maxMetadataChars: number;
+  /** Maximum canonical characters retained across the whole snapshot. */
+  maxTotalChars: number;
+  /** Maximum logical records retained from one snapshot. */
+  maxRecords: number;
+  /** Maximum isolated failures retained before consumption stops. */
+  maxFailures: number;
+}
+
+/** Input for a streaming adapter. The opener is wrapped with the source cap. */
+export interface RecordAdapterInput {
+  sourcePath: string;
+  relativePath: string;
+  collection: string;
+  mime: string;
+  ext: string;
+  open: (signal?: AbortSignal) => AsyncIterable<Uint8Array>;
+  signal?: AbortSignal;
+  limits: RecordAdapterLimits;
+}
+
+export interface RecordAnchor {
+  kind: "line" | "cue" | "timestamp" | "message" | "event" | "record";
+  value: string;
+  endValue?: string;
+}
+
+export interface RecordAttachmentInventoryItem {
+  name: string;
+  mime?: string;
+  bytes?: number;
+  disposition?: "inline" | "attachment";
+  sha256?: string;
+}
+
+/** Closed per-field limits shared by ingestion and versioned output schemas. */
+export const RECORD_METADATA_LIMITS = {
+  maxItems: 256,
+  maxAdapterIdChars: 128,
+  maxAdapterVersionChars: 64,
+  maxTitleChars: 2048,
+  maxPersonChars: 2048,
+  maxCategoryChars: 512,
+  maxIdentifierChars: 2048,
+  maxDateFieldKeyChars: 128,
+  maxDateFieldValueChars: 256,
+  maxAttachmentNameChars: 512,
+  maxAttachmentMimeChars: 256,
+  maxAnchorChars: 512,
+  maxLanguageHintChars: 64,
+} as const;
+
+/** Metadata shared by export adapters and later search/get projections. */
+export interface RecordMetadata {
+  author?: string;
+  participants?: string[];
+  categories?: string[];
+  dateFields?: Record<string, string>;
+  messageId?: string;
+  inReplyTo?: string;
+  references?: string[];
+  threadId?: string;
+  eventId?: string;
+  sessionId?: string;
+  attachments?: RecordAttachmentInventoryItem[];
+}
+
+/** One raw logical record yielded by a container adapter. */
+export interface RecordAdapterRecord {
+  stableId: string;
+  sourceLocator: string;
+  markdown: string;
+  /** Optional exact source-item hash. Derived deterministically when omitted. */
+  sourceHash?: string;
+  title?: string;
+  languageHint?: string;
+  metadata?: RecordMetadata;
+  anchors?: RecordAnchor[];
+}
+
+export type RecordAdapterFailureCode =
+  | "MALFORMED_RECORD"
+  | "MISSING_ID"
+  | "DUPLICATE_ID"
+  | "RECORD_TOO_LARGE"
+  | "SOURCE_TOO_LARGE"
+  | "TIMEOUT"
+  | "RECORD_LIMIT"
+  | "FAILURE_LIMIT"
+  | "INVALID_LOCATOR"
+  | "INVALID_SOURCE_HASH"
+  | "ADAPTER_FAILURE"
+  | "INVALID_SNAPSHOT";
+
+export interface RecordAdapterFailure {
+  code: RecordAdapterFailureCode;
+  message: string;
+  retryable: boolean;
+  stableId?: string;
+  sourceLocator?: string;
+}
+
+export type RecordAdapterEvent =
+  | { type: "record"; record: RecordAdapterRecord }
+  | { type: "failure"; failure: RecordAdapterFailure }
+  | { type: "snapshot"; state: "complete" | "partial" };
+
+/** Separate lane from one-file converters so byte-oriented behavior is stable. */
+export interface RecordAdapter {
+  readonly id: string;
+  readonly version: string;
+  /** Stable fingerprint of declarative adapter configuration, when present. */
+  readonly configurationFingerprint?: string;
+  canHandle(mime: string, ext: string): boolean;
+  records(input: RecordAdapterInput): AsyncIterable<RecordAdapterEvent>;
+}
+
 /**
  * Pipeline output after canonicalization and hash computation.
  * This is what consumers receive from the conversion pipeline.
@@ -121,3 +247,13 @@ export const DEFAULT_LIMITS = {
   timeoutMs: 60_000, // 60 seconds
   maxOutputChars: 50_000_000, // 50M chars (zip bomb protection)
 } as const;
+
+export const DEFAULT_RECORD_ADAPTER_LIMITS: RecordAdapterLimits = {
+  timeoutMs: 60_000,
+  maxSourceBytes: 100 * 1024 * 1024,
+  maxRecordChars: 2_000_000,
+  maxMetadataChars: 100_000,
+  maxTotalChars: 50_000_000,
+  maxRecords: 100_000,
+  maxFailures: 1_000,
+};