@gmickel/gno 0.3.0

package/src/converters/canonicalize.ts

@@ -0,0 +1,89 @@
+/**
+ * Markdown canonicalization for deterministic output.
+ * PRD §8.4 - Canonical Markdown conventions
+ *
+ * CRITICAL: These rules are a compatibility contract.
+ * Changing them invalidates all existing mirrorHash values.
+ */
+
+/**
+ * Control character pattern built dynamically to avoid lint issues with literal control chars.
+ * Matches U+0000-U+0008, U+000B-U+000C, U+000E-U+001F, U+007F (excludes \n and \t)
+ */
+const CONTROL_CHAR_PATTERN = new RegExp(
+  `[${String.fromCharCode(0)}-${String.fromCharCode(8)}${String.fromCharCode(11)}${String.fromCharCode(12)}${String.fromCharCode(14)}-${String.fromCharCode(31)}${String.fromCharCode(127)}]`,
+  'g'
+);
+
+/**
+ * Canonicalize markdown to ensure deterministic output.
+ *
+ * Rules (PRD §8.4):
+ * 0. Strip BOM (U+FEFF) if present
+ * 1. Normalize to \n newlines (no \r)
+ * 2. Apply NFC Unicode normalization (cross-platform hash stability)
+ * 3. Strip control chars U+0000-U+001F and U+007F except \n (U+000A) and \t (U+0009)
+ * 4. Trim trailing whitespace per line
+ * 5. Treat whitespace-only lines as blank (trim first, then count)
+ * 6. Collapse 2+ consecutive blank lines to exactly 1 (content\n\ncontent)
+ * 7. Ensure exactly one final \n
+ */
+export function canonicalize(markdown: string): string {
+  if (!markdown) {
+    return '\n';
+  }
+
+  // 0. Strip BOM if present (U+FEFF) - ensures deterministic hashing
+  let result = markdown.startsWith('\uFEFF') ? markdown.slice(1) : markdown;
+
+  // 1. Normalize line endings: \r\n → \n, lone \r → \n
+  result = result.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
+
+  // 2. Apply NFC Unicode normalization
+  result = result.normalize('NFC');
+
+  // 3. Strip control characters except \n (U+000A) and \t (U+0009)
+  // Range: U+0000-U+0008, U+000B-U+000C, U+000E-U+001F, U+007F
+  result = result.replace(CONTROL_CHAR_PATTERN, '');
+
+  // 4. Trim trailing whitespace per line and
+  // 5. Treat whitespace-only lines as blank
+  const lines = result.split('\n').map((line) => line.trimEnd());
+
+  // 6. Collapse multiple blank lines to exactly 1
+  // (i.e., content\n\ncontent between paragraphs)
+  const collapsed: string[] = [];
+  let blankCount = 0;
+
+  for (const line of lines) {
+    if (line === '') {
+      blankCount += 1;
+      // Only keep one blank line between content
+      if (blankCount === 1) {
+        collapsed.push(line);
+      }
+    } else {
+      blankCount = 0;
+      collapsed.push(line);
+    }
+  }
+
+  // 7. Ensure exactly one final \n
+  // Remove trailing blank lines first
+  while (collapsed.length > 0 && collapsed.at(-1) === '') {
+    collapsed.pop();
+  }
+
+  // Join and add single final newline
+  return `${collapsed.join('\n')}\n`;
+}
+
+/**
+ * Compute SHA-256 hash of canonical markdown.
+ * Returns lowercase hex string (64 chars).
+ */
+export function mirrorHash(canonical: string): string {
+  const hasher = new Bun.CryptoHasher('sha256');
+  hasher.update(canonical);
+  return hasher.digest('hex');
+}

package/src/converters/errors.ts

@@ -0,0 +1,218 @@
+/**
+ * Converter error types and helpers.
+ * PRD §8.3 - Error model
+ */
+
+import type { ConvertError, ConvertErrorCode, ConvertInput } from './types';
+
+type ConvertErrorOpts = Omit<ConvertError, 'code'>;
+
+/** Max length for error messages/causes to prevent bloat */
+const MAX_CAUSE_LENGTH = 1000;
+
+/**
+ * Normalize a cause to a safe, serializable format.
+ * Extracts essential info from Error objects, limits length.
+ */
+function normalizeCause(
+  cause: unknown
+): { name: string; message: string } | string | undefined {
+  if (cause === undefined || cause === null) {
+    return;
+  }
+
+  if (cause instanceof Error) {
+    const message =
+      cause.message.length > MAX_CAUSE_LENGTH
+        ? `${cause.message.slice(0, MAX_CAUSE_LENGTH)}...`
+        : cause.message;
+    return { name: cause.name, message };
+  }
+
+  if (typeof cause === 'string') {
+    return cause.length > MAX_CAUSE_LENGTH
+      ? `${cause.slice(0, MAX_CAUSE_LENGTH)}...`
+      : cause;
+  }
+
+  // For other types, try to stringify safely
+  try {
+    const str = String(cause);
+    return str.length > MAX_CAUSE_LENGTH
+      ? `${str.slice(0, MAX_CAUSE_LENGTH)}...`
+      : str;
+  } catch {
+    return '[unserializable cause]';
+  }
+}
+
+/**
+ * Create a ConvertError with the given code and options.
+ * Normalizes cause to prevent bloat and serialization issues.
+ */
+export function convertError(
+  code: ConvertErrorCode,
+  opts: ConvertErrorOpts
+): ConvertError {
+  return {
+    code,
+    ...opts,
+    cause: normalizeCause(opts.cause),
+  };
+}
+
+/**
+ * Check if an error code indicates a retryable failure.
+ */
+export function isRetryable(code: ConvertErrorCode): boolean {
+  return ['TIMEOUT', 'IO', 'ADAPTER_FAILURE'].includes(code);
+}
+
+/**
+ * Create a standard error result for unsupported file types.
+ */
+export function unsupportedError(
+  input: Pick<ConvertInput, 'sourcePath' | 'mime' | 'ext'>,
+  converterId = 'registry'
+): ConvertError {
+  return convertError('UNSUPPORTED', {
+    message: `No converter for ${input.mime} (${input.ext})`,
+    retryable: false,
+    fatal: false,
+    converterId,
+    sourcePath: input.sourcePath,
+    mime: input.mime,
+    ext: input.ext,
+  });
+}
+
+/**
+ * Create an error for files exceeding size limits.
+ */
+export function tooLargeError(
+  input: Pick<ConvertInput, 'sourcePath' | 'mime' | 'ext' | 'bytes' | 'limits'>,
+  converterId: string
+): ConvertError {
+  return convertError('TOO_LARGE', {
+    message: `File size ${input.bytes.length} exceeds limit ${input.limits.maxBytes}`,
+    retryable: false,
+    fatal: false,
+    converterId,
+    sourcePath: input.sourcePath,
+    mime: input.mime,
+    ext: input.ext,
+    details: {
+      size: input.bytes.length,
+      limit: input.limits.maxBytes,
+    },
+  });
+}
+
+/**
+ * Create an error for conversion output exceeding size limits.
+ * Distinct from tooLargeError (input) - this is for output (zip bomb protection).
+ */
+export function outputTooLargeError(
+  input: Pick<ConvertInput, 'sourcePath' | 'mime' | 'ext'>,
+  converterId: string,
+  opts: { outputChars: number; limitChars: number; stage: 'raw' | 'canonical' }
+): ConvertError {
+  return convertError('TOO_LARGE', {
+    message: `Conversion output (${opts.outputChars} chars at ${opts.stage}) exceeds limit ${opts.limitChars}`,
+    retryable: false,
+    fatal: false,
+    converterId,
+    sourcePath: input.sourcePath,
+    mime: input.mime,
+    ext: input.ext,
+    details: {
+      outputChars: opts.outputChars,
+      limitChars: opts.limitChars,
+      stage: opts.stage,
+    },
+  });
+}
+
+/**
+ * Create an error for conversion timeouts.
+ */
+export function timeoutError(
+  input: Pick<ConvertInput, 'sourcePath' | 'mime' | 'ext' | 'limits'>,
+  converterId: string
+): ConvertError {
+  return convertError('TIMEOUT', {
+    message: `Conversion timed out after ${input.limits.timeoutMs}ms`,
+    retryable: true,
+    fatal: false,
+    converterId,
+    sourcePath: input.sourcePath,
+    mime: input.mime,
+    ext: input.ext,
+    details: {
+      timeoutMs: input.limits.timeoutMs,
+    },
+  });
+}
+
+/**
+ * Create an error for corrupt or invalid files.
+ */
+export function corruptError(
+  input: Pick<ConvertInput, 'sourcePath' | 'mime' | 'ext'>,
+  converterId: string,
+  message: string,
+  cause?: unknown
+): ConvertError {
+  return convertError('CORRUPT', {
+    message,
+    retryable: false,
+    fatal: false,
+    converterId,
+    sourcePath: input.sourcePath,
+    mime: input.mime,
+    ext: input.ext,
+    cause,
+  });
+}
+
+/**
+ * Create an error for adapter-level failures.
+ */
+export function adapterError(
+  input: Pick<ConvertInput, 'sourcePath' | 'mime' | 'ext'>,
+  converterId: string,
+  message: string,
+  cause?: unknown
+): ConvertError {
+  return convertError('ADAPTER_FAILURE', {
+    message,
+    retryable: true,
+    fatal: false,
+    converterId,
+    sourcePath: input.sourcePath,
+    mime: input.mime,
+    ext: input.ext,
+    cause,
+  });
+}
+
+/**
+ * Create an error for internal pipeline failures.
+ */
+export function internalError(
+  input: Pick<ConvertInput, 'sourcePath' | 'mime' | 'ext'>,
+  converterId: string,
+  message: string,
+  cause?: unknown
+): ConvertError {
+  return convertError('INTERNAL', {
+    message,
+    retryable: false,
+    fatal: true,
+    converterId,
+    sourcePath: input.sourcePath,
+    mime: input.mime,
+    ext: input.ext,
+    cause,
+  });
+}

package/src/converters/index.ts

@@ -0,0 +1,51 @@
+/**
+ * Converter subsystem public API.
+ *
+ * Usage:
+ *   import { getDefaultPipeline } from './converters';
+ *   const pipeline = getDefaultPipeline();
+ *   const result = await pipeline.convert(input);
+ */
+
+// Canonicalization
+export { canonicalize, mirrorHash } from './canonicalize';
+// Errors
+export {
+  adapterError,
+  convertError,
+  corruptError,
+  isRetryable,
+  timeoutError,
+  tooLargeError,
+  unsupportedError,
+} from './errors';
+// MIME detection
+export type { MimeDetection, MimeDetector } from './mime';
+export {
+  DefaultMimeDetector,
+  getDefaultMimeDetector,
+  isSupportedExtension,
+  SUPPORTED_EXTENSIONS,
+} from './mime';
+// Pipeline (main entry point)
+export {
+  ConversionPipeline,
+  getDefaultPipeline,
+  resetDefaultPipeline,
+} from './pipeline';
+// Registry
+export { ConverterRegistry, createDefaultRegistry } from './registry';
+// Types
+export type {
+  ConversionArtifact,
+  ConvertError,
+  ConvertErrorCode,
+  Converter,
+  ConverterId,
+  ConvertInput,
+  ConvertOutput,
+  ConvertResult,
+  ConvertWarning,
+  PipelineResult,
+} from './types';
+export { DEFAULT_LIMITS } from './types';

package/src/converters/mime.ts

@@ -0,0 +1,163 @@
+/**
+ * MIME type detection with magic byte sniffing and extension mapping.
+ * PRD §8.5 - MIME detection strategy
+ */
+
+import { extname } from './path';
+
+export interface MimeDetection {
+  mime: string;
+  ext: string;
+  confidence: 'high' | 'medium' | 'low';
+  via: 'sniff' | 'sniff+ext' | 'ext' | 'fallback';
+}
+
+export interface MimeDetector {
+  detect(path: string, bytes: Uint8Array): MimeDetection;
+}
+
+/** Extension to MIME type mapping (PRD §8.5) */
+const EXTENSION_MAP: Record<string, string> = {
+  '.md': 'text/markdown',
+  '.txt': 'text/plain',
+  '.pdf': 'application/pdf',
+  '.docx':
+    'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+  '.pptx':
+    'application/vnd.openxmlformats-officedocument.presentationml.presentation',
+  '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+};
+
+/** OOXML extension to MIME mapping */
+const OOXML_MAP: Record<string, string> = {
+  '.docx':
+    'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+  '.pptx':
+    'application/vnd.openxmlformats-officedocument.presentationml.presentation',
+  '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+};
+
+/** PDF magic bytes: %PDF- */
+const PDF_MAGIC = new Uint8Array([0x25, 0x50, 0x44, 0x46, 0x2d]);
+
+/** ZIP/OOXML magic bytes: PK\x03\x04 */
+const ZIP_MAGIC = new Uint8Array([0x50, 0x4b, 0x03, 0x04]);
+
+/**
+ * Check if bytes start with the given prefix.
+ */
+function startsWith(bytes: Uint8Array, prefix: Uint8Array): boolean {
+  if (bytes.length < prefix.length) {
+    return false;
+  }
+  for (let i = 0; i < prefix.length; i++) {
+    if (bytes[i] !== prefix[i]) {
+      return false;
+    }
+  }
+  return true;
+}
+
+interface SniffResult {
+  mime: string;
+  /** True if sniff alone is sufficient (e.g., PDF); false if ext-assisted (OOXML) */
+  pureSniff: boolean;
+}
+
+/**
+ * Sniff MIME type from magic bytes.
+ * Returns detected MIME or undefined if no match.
+ */
+function sniffMagicBytes(
+  bytes: Uint8Array,
+  ext: string
+): SniffResult | undefined {
+  // PDF detection - pure sniff, no extension needed
+  if (startsWith(bytes, PDF_MAGIC)) {
+    return { mime: 'application/pdf', pureSniff: true };
+  }
+
+  // ZIP/OOXML detection - requires extension to distinguish OOXML from generic ZIP
+  if (startsWith(bytes, ZIP_MAGIC)) {
+    const ooxmlMime = Object.hasOwn(OOXML_MAP, ext)
+      ? OOXML_MAP[ext]
+      : undefined;
+    if (ooxmlMime) {
+      // ZIP magic + OOXML extension = extension-assisted sniff
+      return { mime: ooxmlMime, pureSniff: false };
+    }
+    // Generic ZIP (not OOXML)
+    return { mime: 'application/zip', pureSniff: true };
+  }
+
+  return;
+}
+
+/**
+ * Default MIME detector implementation.
+ * Detection priority:
+ * 1. Magic bytes (sniff) → high confidence for pure sniff
+ * 2. Magic bytes + extension → medium confidence (OOXML via ZIP+ext)
+ * 3. Extension map → medium confidence
+ * 4. Fallback application/octet-stream → low confidence
+ */
+export class DefaultMimeDetector implements MimeDetector {
+  detect(path: string, bytes: Uint8Array): MimeDetection {
+    const ext = extname(path);
+
+    // 1. Try magic byte sniffing (first 512 bytes sufficient)
+    // Use subarray for zero-copy view (no allocation)
+    const sniffBytes = bytes.subarray(0, 512);
+    const sniffed = sniffMagicBytes(sniffBytes, ext);
+    if (sniffed) {
+      return {
+        mime: sniffed.mime,
+        ext,
+        // Pure sniff (e.g., PDF) is high confidence
+        // Extension-assisted sniff (OOXML) is medium confidence
+        confidence: sniffed.pureSniff ? 'high' : 'medium',
+        via: sniffed.pureSniff ? 'sniff' : 'sniff+ext',
+      };
+    }
+
+    // 2. Try extension mapping
+    const extMime = Object.hasOwn(EXTENSION_MAP, ext)
+      ? EXTENSION_MAP[ext]
+      : undefined;
+    if (extMime) {
+      return {
+        mime: extMime,
+        ext,
+        confidence: 'medium',
+        via: 'ext',
+      };
+    }
+
+    // 3. Fallback
+    return {
+      mime: 'application/octet-stream',
+      ext,
+      confidence: 'low',
+      via: 'fallback',
+    };
+  }
+}
+
+/** Singleton default detector */
+let defaultDetector: MimeDetector | null = null;
+
+export function getDefaultMimeDetector(): MimeDetector {
+  if (!defaultDetector) {
+    defaultDetector = new DefaultMimeDetector();
+  }
+  return defaultDetector;
+}
+
+/** Supported extensions for conversion */
+export const SUPPORTED_EXTENSIONS = Object.keys(EXTENSION_MAP);
+
+/** Check if extension is supported for conversion (prototype-safe) */
+export function isSupportedExtension(ext: string): boolean {
+  const normalized = ext.toLowerCase();
+  return Object.hasOwn(EXTENSION_MAP, normalized);
+}

package/src/converters/native/markdown.ts

@@ -0,0 +1,115 @@
+/**
+ * Native Markdown converter (passthrough).
+ * Simply reads .md files and extracts title from first heading.
+ */
+
+import type { Converter, ConvertInput, ConvertResult } from '../types';
+import { NATIVE_VERSIONS } from '../versions';
+
+const CONVERTER_ID = 'native/markdown' as const;
+const CONVERTER_VERSION = NATIVE_VERSIONS.markdown;
+
+/** UTF-8 BOM character */
+const BOM = '\uFEFF';
+
+/** Regex to match # heading at line start */
+const HEADING_PATTERN = /^\s*#\s+(.+)/;
+
+/** Regex to detect code fence start (captures the fence chars and optional info string) */
+const CODE_FENCE_START = /^(`{3,}|~{3,})/;
+
+/**
+ * Check if a line closes a code fence.
+ * Closing fence must be same char type and at least as long as opening.
+ */
+function isClosingFence(
+  line: string,
+  fenceChar: string,
+  fenceLen: number
+): boolean {
+  const trimmed = line.trim();
+  // Must be only fence chars (no info string on close)
+  if (trimmed.length < fenceLen) {
+    return false;
+  }
+  // All chars must be the fence char
+  for (const char of trimmed) {
+    if (char !== fenceChar) {
+      return false;
+    }
+  }
+  return true;
+}
+
+/**
+ * Extract title from first # heading in markdown, skipping code blocks.
+ * Returns undefined if no heading found.
+ */
+function extractFirstHeading(markdown: string): string | undefined {
+  const lines = markdown.split('\n');
+  let fenceChar = '';
+  let fenceLen = 0;
+
+  for (const line of lines) {
+    // If inside a fence, check for closing
+    if (fenceLen > 0) {
+      if (isClosingFence(line, fenceChar, fenceLen)) {
+        fenceChar = '';
+        fenceLen = 0;
+      }
+      continue;
+    }
+
+    // Check for fence opening
+    const fenceMatch = line.match(CODE_FENCE_START);
+    if (fenceMatch?.[1]) {
+      fenceChar = fenceMatch[1].charAt(0);
+      fenceLen = fenceMatch[1].length;
+      continue;
+    }
+
+    // Check for heading (not inside fence)
+    const headingMatch = line.match(HEADING_PATTERN);
+    if (headingMatch?.[1]) {
+      return headingMatch[1].trim();
+    }
+  }
+
+  return;
+}
+
+export const markdownConverter: Converter = {
+  id: CONVERTER_ID,
+  version: CONVERTER_VERSION,
+
+  canHandle(mime: string, ext: string): boolean {
+    return mime === 'text/markdown' || ext === '.md';
+  },
+
+  convert(input: ConvertInput): Promise<ConvertResult> {
+    // Decode bytes to string (assumes UTF-8)
+    let text = new TextDecoder('utf-8', { fatal: false }).decode(input.bytes);
+
+    // Strip BOM if present (ensures consistent hashes)
+    if (text.startsWith(BOM)) {
+      text = text.slice(1);
+    }
+
+    // Extract title from first heading
+    const title = extractFirstHeading(text);
+
+    // NOTE: Do NOT canonicalize here - pipeline.ts handles all normalization
+    return Promise.resolve({
+      ok: true,
+      value: {
+        markdown: text,
+        title,
+        meta: {
+          converterId: CONVERTER_ID,
+          converterVersion: CONVERTER_VERSION,
+          sourceMime: input.mime,
+        },
+      },
+    });
+  },
+};

package/src/converters/native/plaintext.ts

@@ -0,0 +1,56 @@
+/**
+ * Native plaintext converter.
+ * Converts .txt files to markdown (passthrough as paragraphs).
+ */
+
+import { basenameWithoutExt } from '../path';
+import type { Converter, ConvertInput, ConvertResult } from '../types';
+import { NATIVE_VERSIONS } from '../versions';
+
+const CONVERTER_ID = 'native/plaintext' as const;
+const CONVERTER_VERSION = NATIVE_VERSIONS.plaintext;
+
+/** UTF-8 BOM character */
+const BOM = '\uFEFF';
+
+export const plaintextConverter: Converter = {
+  id: CONVERTER_ID,
+  version: CONVERTER_VERSION,
+
+  canHandle(mime: string, ext: string): boolean {
+    return mime === 'text/plain' || ext === '.txt';
+  },
+
+  convert(input: ConvertInput): Promise<ConvertResult> {
+    // Decode as UTF-8 with replacement for invalid bytes (deterministic)
+    const decoder = new TextDecoder('utf-8', {
+      fatal: false, // Don't throw on invalid bytes
+      ignoreBOM: false, // We'll strip manually for determinism
+    });
+
+    let text = decoder.decode(input.bytes);
+
+    // Strip BOM if present (ensures consistent hashes)
+    if (text.startsWith(BOM)) {
+      text = text.slice(1);
+    }
+
+    // Derive title from filename (cross-platform safe)
+    const title = basenameWithoutExt(input.relativePath);
+
+    // Pass through as paragraphs (no code fence wrapping - better for search)
+    // NOTE: Do NOT canonicalize here - pipeline.ts handles all normalization
+    return Promise.resolve({
+      ok: true,
+      value: {
+        markdown: text,
+        title,
+        meta: {
+          converterId: CONVERTER_ID,
+          converterVersion: CONVERTER_VERSION,
+          sourceMime: input.mime,
+        },
+      },
+    });
+  },
+};