@gmickel/gno 0.3.0

package/src/converters/path.ts

@@ -0,0 +1,48 @@
+/**
+ * Cross-platform path utilities for deterministic behavior.
+ * Uses POSIX path operations to ensure identical results across platforms.
+ *
+ * CRITICAL: All converter path operations MUST use these utilities.
+ * Using node:path directly will produce different results on Windows vs POSIX.
+ */
+
+// OK: no Bun path utils, must use node:path/posix for cross-platform determinism
+import {
+  basename as posixBasename,
+  extname as posixExtname,
+  normalize as posixNormalize,
+} from 'node:path/posix';
+
+/**
+ * Normalize a relative path to POSIX format.
+ * Converts Windows separators, normalizes ../ and ./, strips trailing slashes.
+ */
+export function normalizePath(p: string): string {
+  // Convert Windows separators to POSIX
+  const posixPath = p.replace(/\\/g, '/');
+  // Normalize (resolve .., ., double slashes)
+  return posixNormalize(posixPath);
+}
+
+/**
+ * Get the base filename from a path (POSIX-safe).
+ */
+export function basename(p: string): string {
+  return posixBasename(normalizePath(p));
+}
+
+/**
+ * Get the file extension from a path (POSIX-safe, lowercase).
+ */
+export function extname(p: string): string {
+  return posixExtname(normalizePath(p)).toLowerCase();
+}
+
+/**
+ * Get filename without extension (for title derivation).
+ */
+export function basenameWithoutExt(p: string): string {
+  const name = basename(p);
+  const ext = posixExtname(name);
+  return ext ? name.slice(0, -ext.length) : name;
+}

package/src/converters/pipeline.ts

@@ -0,0 +1,159 @@
+/**
+ * Conversion pipeline - single entry point for all document conversions.
+ * PRD §8.4 - Canonical Markdown conventions
+ *
+ * The pipeline:
+ * 1. Delegates to the registry to find and invoke the appropriate converter
+ * 2. Enforces pre-canonicalization output size limit (early bailout)
+ * 3. Canonicalizes the raw markdown output (centralized, not per-converter)
+ * 4. Enforces post-canonicalization output size limit (zip bomb protection)
+ * 5. Computes mirrorHash from canonical markdown
+ * 6. Returns ConversionArtifact (not ConvertOutput)
+ *
+ * CRITICAL: Canonicalization is ONLY done here, not in individual converters.
+ */
+
+import { canonicalize, mirrorHash } from './canonicalize';
+import { internalError, outputTooLargeError } from './errors';
+import { type ConverterRegistry, createDefaultRegistry } from './registry';
+import type { ConversionArtifact, ConvertInput, PipelineResult } from './types';
+
+export class ConversionPipeline {
+  private registry: ConverterRegistry | null = null;
+  private initPromise: Promise<void> | null = null;
+
+  /**
+   * Create a pipeline with default registry.
+   * Registry is lazily initialized on first use.
+   */
+  constructor(registry?: ConverterRegistry) {
+    if (registry) {
+      this.registry = registry;
+    }
+  }
+
+  /**
+   * Ensure registry is initialized.
+   * Resets initPromise on failure to allow retry.
+   */
+  private async ensureRegistry(): Promise<ConverterRegistry> {
+    if (this.registry) {
+      return this.registry;
+    }
+
+    if (!this.initPromise) {
+      this.initPromise = createDefaultRegistry().then((r) => {
+        this.registry = r;
+      });
+    }
+
+    try {
+      await this.initPromise;
+    } catch (err) {
+      // Reset to allow retry on next call
+      this.initPromise = null;
+      throw err;
+    }
+
+    // Safe: after await, this.registry is always set
+    return this.registry as unknown as ConverterRegistry;
+  }
+
+  /**
+   * Convert a file through the pipeline.
+   * Returns ConversionArtifact with canonical markdown and mirrorHash.
+   *
+   * All exceptions are caught and mapped to INTERNAL errors.
+   */
+  async convert(input: ConvertInput): Promise<PipelineResult> {
+    try {
+      // 0. Initialize registry
+      const registry = await this.ensureRegistry();
+
+      // 1. Delegate to registry (finds converter + invokes)
+      const result = await registry.convert(input);
+
+      if (!result.ok) {
+        return result; // Pass through error
+      }
+
+      const maxChars = input.limits.maxOutputChars ?? 0;
+      const rawMarkdown = result.value.markdown;
+
+      // 2. Pre-canonicalization size check (early bailout to avoid expensive canonicalization)
+      if (maxChars > 0 && rawMarkdown.length > maxChars) {
+        return {
+          ok: false,
+          error: outputTooLargeError(input, 'pipeline', {
+            outputChars: rawMarkdown.length,
+            limitChars: maxChars,
+            stage: 'raw',
+          }),
+        };
+      }
+
+      // 3. Canonicalize the raw markdown output
+      const canonical = canonicalize(rawMarkdown);
+
+      // 4. Post-canonicalization size check (canonicalization may expand slightly)
+      if (maxChars > 0 && canonical.length > maxChars) {
+        return {
+          ok: false,
+          error: outputTooLargeError(input, 'pipeline', {
+            outputChars: canonical.length,
+            limitChars: maxChars,
+            stage: 'canonical',
+          }),
+        };
+      }
+
+      // 5. Compute content-addressed hash
+      const hash = mirrorHash(canonical);
+
+      // 6. Return artifact with all pipeline-computed fields
+      const artifact: ConversionArtifact = {
+        markdown: canonical,
+        mirrorHash: hash,
+        title: result.value.title,
+        languageHint: result.value.languageHint,
+        meta: result.value.meta,
+      };
+
+      return { ok: true, value: artifact };
+    } catch (cause) {
+      // Catch any unhandled exceptions and map to INTERNAL error
+      return {
+        ok: false,
+        error: internalError(
+          input,
+          'pipeline',
+          cause instanceof Error ? cause.message : 'Unknown pipeline error',
+          cause
+        ),
+      };
+    }
+  }
+
+  /**
+   * List available converters.
+   */
+  async listConverters(): Promise<string[]> {
+    const registry = await this.ensureRegistry();
+    return registry.listConverters();
+  }
+}
+
+/** Singleton for simple usage */
+let defaultPipeline: ConversionPipeline | null = null;
+
+export function getDefaultPipeline(): ConversionPipeline {
+  if (!defaultPipeline) {
+    defaultPipeline = new ConversionPipeline();
+  }
+  return defaultPipeline;
+}
+
+/** Reset singleton (for testing) */
+export function resetDefaultPipeline(): void {
+  defaultPipeline = null;
+}

package/src/converters/registry.ts

@@ -0,0 +1,74 @@
+/**
+ * Converter registry for routing files to appropriate converters.
+ * PRD §8.6 - Converter registry
+ */
+
+import { unsupportedError } from './errors';
+import type { Converter, ConvertInput, ConvertResult } from './types';
+
+export class ConverterRegistry {
+  private readonly converters: Converter[] = [];
+
+  /**
+   * Register a converter. Order matters - first match wins.
+   */
+  register(converter: Converter): void {
+    this.converters.push(converter);
+  }
+
+  /**
+   * Select the first converter that can handle the given MIME/ext.
+   * Normalizes to lowercase for consistent matching.
+   */
+  select(mime: string, ext: string): Converter | undefined {
+    const m = mime.toLowerCase();
+    const e = ext.toLowerCase();
+    return this.converters.find((c) => c.canHandle(m, e));
+  }
+
+  /**
+   * List all registered converter IDs.
+   */
+  listConverters(): string[] {
+    return this.converters.map((c) => c.id);
+  }
+
+  /**
+   * Convert a file using the appropriate converter.
+   */
+  convert(input: ConvertInput): Promise<ConvertResult> {
+    const converter = this.select(input.mime, input.ext);
+    if (!converter) {
+      return Promise.resolve({ ok: false, error: unsupportedError(input) });
+    }
+    return converter.convert(input);
+  }
+}
+
+/**
+ * Create the default registry with all MVP converters.
+ * Priority order per PRD §8.6:
+ * 1. native/markdown - handles .md
+ * 2. native/plaintext - handles .txt
+ * 3. adapter/markitdown-ts - handles .pdf, .docx, .xlsx
+ * 4. adapter/officeparser - handles .pptx
+ */
+export async function createDefaultRegistry(): Promise<ConverterRegistry> {
+  const registry = new ConverterRegistry();
+
+  // Import converters dynamically to avoid circular deps
+  const { markdownConverter } = await import('./native/markdown');
+  const { plaintextConverter } = await import('./native/plaintext');
+  const { markitdownAdapter } = await import('./adapters/markitdownTs/adapter');
+  const { officeparserAdapter } = await import(
+    './adapters/officeparser/adapter'
+  );
+
+  // Register in priority order
+  registry.register(markdownConverter);
+  registry.register(plaintextConverter);
+  registry.register(markitdownAdapter);
+  registry.register(officeparserAdapter);
+
+  return registry;
+}

package/src/converters/types.ts

@@ -0,0 +1,123 @@
+/**
+ * Converter subsystem types.
+ * PRD §8.2 - Converter interfaces
+ */
+
+export type ConverterId = string;
+
+export interface ConvertInput {
+  /** Absolute path to source file */
+  sourcePath: string;
+  /** Relative path within collection */
+  relativePath: string;
+  /** Collection name */
+  collection: string;
+  /** File contents */
+  bytes: Uint8Array;
+  /** Detected MIME type */
+  mime: string;
+  /** File extension (e.g., ".pdf") */
+  ext: string;
+  /** Conversion limits */
+  limits: {
+    /** Max file size in bytes (default: 100MB) */
+    maxBytes: number;
+    /** Conversion timeout in ms (default: 60000) */
+    timeoutMs: number;
+    /** Max output chars after conversion (zip bomb protection, default: 50M) */
+    maxOutputChars?: number;
+  };
+}
+
+export interface ConvertWarning {
+  code:
+    | 'LOSSY'
+    | 'TRUNCATED'
+    | 'PARTIAL'
+    | 'UNSUPPORTED_FEATURE'
+    | 'LOW_CONFIDENCE';
+  message: string;
+  details?: Record<string, unknown>;
+}
+
+/**
+ * Raw output from individual converters.
+ * Note: markdown is NOT canonical - pipeline.ts handles normalization.
+ */
+export interface ConvertOutput {
+  /** Raw markdown (pipeline canonicalizes) */
+  markdown: string;
+  /** Extracted or derived title */
+  title?: string;
+  /** BCP-47 language hint or "und" */
+  languageHint?: string;
+  /** Conversion metadata */
+  meta: {
+    converterId: ConverterId;
+    converterVersion: string;
+    sourceMime: string;
+    warnings?: ConvertWarning[];
+  };
+}
+
+export type ConvertErrorCode =
+  | 'UNSUPPORTED'
+  | 'TOO_LARGE'
+  | 'TIMEOUT'
+  | 'CORRUPT'
+  | 'PERMISSION'
+  | 'IO'
+  | 'ADAPTER_FAILURE'
+  | 'INTERNAL';
+
+export interface ConvertError {
+  code: ConvertErrorCode;
+  message: string;
+  retryable: boolean;
+  fatal: boolean;
+  converterId: string;
+  sourcePath: string;
+  mime: string;
+  ext: string;
+  cause?: unknown;
+  details?: Record<string, unknown>;
+}
+
+export type ConvertResult =
+  | { ok: true; value: ConvertOutput }
+  | { ok: false; error: ConvertError };
+
+export interface Converter {
+  readonly id: ConverterId;
+  readonly version: string;
+  canHandle(mime: string, ext: string): boolean;
+  convert(input: ConvertInput): Promise<ConvertResult>;
+}
+
+/**
+ * Pipeline output after canonicalization and hash computation.
+ * This is what consumers receive from the conversion pipeline.
+ */
+export interface ConversionArtifact {
+  /** Canonical markdown after pipeline normalization */
+  markdown: string;
+  /** SHA-256 hex of canonical markdown - content-addressed key */
+  mirrorHash: string;
+  /** Title from conversion (or derived from filename) */
+  title?: string;
+  /** Language hint from conversion */
+  languageHint?: string;
+  /** Conversion metadata */
+  meta: ConvertOutput['meta'];
+}
+
+export type PipelineResult =
+  | { ok: true; value: ConversionArtifact }
+  | { ok: false; error: ConvertError };
+
+/** Default conversion limits */
+export const DEFAULT_LIMITS = {
+  maxBytes: 100 * 1024 * 1024, // 100MB
+  timeoutMs: 60_000, // 60 seconds
+  maxOutputChars: 50_000_000, // 50M chars (zip bomb protection)
+} as const;

package/src/converters/versions.ts

@@ -0,0 +1,24 @@
+/**
+ * Centralized converter version tracking.
+ *
+ * Native converters use our own versioning.
+ * Adapter versions MUST match the wrapped npm package version.
+ *
+ * When updating npm dependencies, update these versions too.
+ * Run `bun pm ls markitdown-ts officeparser` to check current versions.
+ */
+
+/** Native converter versions (our own) */
+export const NATIVE_VERSIONS = {
+  markdown: '1.0.0',
+  plaintext: '1.0.0',
+} as const;
+
+/**
+ * Adapter versions - MUST match npm package versions.
+ * Update these when running `bun update`.
+ */
+export const ADAPTER_VERSIONS = {
+  'markitdown-ts': '0.0.8',
+  officeparser: '5.2.0',
+} as const;

package/src/index.ts

@@ -0,0 +1,27 @@
+#!/usr/bin/env bun
+/**
+ * GNO CLI entry point.
+ * Thin bootstrap that delegates to CLI runner.
+ *
+ * @module src/index
+ */
+
+import { runCli } from './cli/run';
+
+// SIGINT handler for graceful shutdown
+process.on('SIGINT', () => {
+  process.stderr.write('\nInterrupted\n');
+  process.exit(130);
+});
+
+// Run CLI and exit
+runCli(process.argv)
+  .then((code) => {
+    process.exit(code);
+  })
+  .catch((err) => {
+    process.stderr.write(
+      `Fatal error: ${err instanceof Error ? err.message : String(err)}\n`
+    );
+    process.exit(1);
+  });

package/src/ingestion/chunker.ts

@@ -0,0 +1,238 @@
+/**
+ * Markdown chunker implementation.
+ * Char-based chunking with line tracking.
+ *
+ * @module src/ingestion/chunker
+ */
+
+import { defaultLanguageDetector } from './language';
+import type { ChunkerPort, ChunkOutput, ChunkParams } from './types';
+import { DEFAULT_CHUNK_PARAMS } from './types';
+
+/** Approximate chars per token (conservative estimate) */
+const CHARS_PER_TOKEN = 4;
+
+/** Minimum valid maxTokens to prevent degenerate behavior */
+const MIN_MAX_TOKENS = 10;
+
+/** Maximum valid overlap percentage */
+const MAX_OVERLAP_PERCENT = 0.5;
+
+/** Regex for sentence ending followed by whitespace and capital letter (global) */
+const SENTENCE_END_REGEX = /[.!?](\s+)[A-Z]/g;
+
+/**
+ * Line index for O(1) line number lookups.
+ * Stores positions of all newline characters.
+ */
+interface LineIndex {
+  /** Positions of '\n' characters in the text */
+  newlines: number[];
+}
+
+/**
+ * Build a line index from text (O(n) once).
+ */
+function buildLineIndex(text: string): LineIndex {
+  const newlines: number[] = [];
+  for (let i = 0; i < text.length; i += 1) {
+    if (text[i] === '\n') {
+      newlines.push(i);
+    }
+  }
+  return { newlines };
+}
+
+/**
+ * Find line number at character position using binary search (O(log n)).
+ * Returns 1-based line number.
+ */
+function lineAtPosition(index: LineIndex, pos: number): number {
+  const { newlines } = index;
+
+  // Binary search for the number of newlines before pos
+  let low = 0;
+  let high = newlines.length;
+
+  while (low < high) {
+    const mid = Math.floor((low + high) / 2);
+    const newlinePos = newlines[mid];
+    if (newlinePos !== undefined && newlinePos < pos) {
+      low = mid + 1;
+    } else {
+      high = mid;
+    }
+  }
+
+  return low + 1; // 1-based line number
+}
+
+/**
+ * Normalize chunk parameters to valid ranges.
+ * Prevents degenerate behavior from invalid inputs.
+ */
+function normalizeChunkParams(params?: ChunkParams): Required<ChunkParams> {
+  const maxTokens = Math.max(
+    MIN_MAX_TOKENS,
+    Math.floor(params?.maxTokens ?? DEFAULT_CHUNK_PARAMS.maxTokens)
+  );
+  const overlapPercent = Math.max(
+    0,
+    Math.min(
+      MAX_OVERLAP_PERCENT,
+      params?.overlapPercent ?? DEFAULT_CHUNK_PARAMS.overlapPercent
+    )
+  );
+  return { maxTokens, overlapPercent };
+}
+
+/**
+ * Find a good break point near target position.
+ * Prefers paragraph breaks, then sentence endings, then word boundaries.
+ */
+function findBreakPoint(
+  text: string,
+  target: number,
+  windowSize: number
+): number {
+  const start = Math.max(0, target - windowSize);
+  const end = Math.min(text.length, target + windowSize);
+  const windowText = text.slice(start, end);
+
+  // Look for paragraph break (double newline) - prefer last one
+  const paraBreak = windowText.lastIndexOf('\n\n');
+  if (paraBreak !== -1) {
+    return start + paraBreak + 2;
+  }
+
+  // Look for sentence ending - find the LAST match before target
+  // Reset regex state for fresh search
+  SENTENCE_END_REGEX.lastIndex = 0;
+  let lastSentenceMatch: RegExpExecArray | null = null;
+  let match: RegExpExecArray | null = null;
+
+  while (true) {
+    match = SENTENCE_END_REGEX.exec(windowText);
+    if (!match) {
+      break;
+    }
+
+    // Only consider matches that would give us a break point before or near target
+    const whitespace = match[1] ?? '';
+    const breakPos = start + match.index + 1 + whitespace.length;
+    if (breakPos <= target + windowSize) {
+      lastSentenceMatch = match;
+    }
+  }
+
+  if (lastSentenceMatch) {
+    // Break after the punctuation and whitespace, before the capital
+    const whitespace = lastSentenceMatch[1] ?? '';
+    return start + lastSentenceMatch.index + 1 + whitespace.length;
+  }
+
+  // Look for single newline
+  const lineBreak = windowText.lastIndexOf('\n');
+  if (lineBreak !== -1) {
+    return start + lineBreak + 1;
+  }
+
+  // Look for word boundary
+  const wordBoundary = windowText.lastIndexOf(' ');
+  if (wordBoundary !== -1) {
+    return start + wordBoundary + 1;
+  }
+
+  // Fall back to target
+  return target;
+}
+
+/**
+ * Markdown chunker implementation.
+ * Uses character-based chunking with semantic break points.
+ *
+ * Note: Chunk text is preserved exactly as-is (no trimming) to maintain
+ * accurate pos/line mappings and preserve Markdown semantics like
+ * indented code blocks.
+ */
+export class MarkdownChunker implements ChunkerPort {
+  chunk(
+    markdown: string,
+    params?: ChunkParams,
+    documentLanguageHint?: string
+  ): ChunkOutput[] {
+    if (!markdown || markdown.trim().length === 0) {
+      return [];
+    }
+
+    // Normalize params to prevent degenerate behavior
+    const { maxTokens, overlapPercent } = normalizeChunkParams(params);
+
+    const maxChars = maxTokens * CHARS_PER_TOKEN;
+    const overlapChars = Math.floor(maxChars * overlapPercent);
+    const windowSize = Math.floor(maxChars * 0.1); // 10% window for break search
+
+    // Build line index once for O(log n) lookups
+    const lineIndex = buildLineIndex(markdown);
+
+    const chunks: ChunkOutput[] = [];
+    let pos = 0;
+    let seq = 0;
+
+    while (pos < markdown.length) {
+      // Calculate target end position
+      const targetEnd = pos + maxChars;
+
+      let endPos: number;
+      if (targetEnd >= markdown.length) {
+        // Last chunk - take rest
+        endPos = markdown.length;
+      } else {
+        // Find a good break point
+        endPos = findBreakPoint(markdown, targetEnd, windowSize);
+      }
+
+      // Extract chunk text - preserve exactly (no trim!)
+      // This maintains accurate pos/line mappings and Markdown semantics
+      const text = markdown.slice(pos, endPos);
+
+      // Only skip truly empty chunks (all whitespace after full content consumed)
+      if (text.trim().length > 0) {
+        const startLine = lineAtPosition(lineIndex, pos);
+        const endLine = lineAtPosition(lineIndex, endPos - 1);
+
+        // Detect language for this chunk
+        const language =
+          documentLanguageHint ?? defaultLanguageDetector.detect(text);
+
+        chunks.push({
+          seq,
+          pos,
+          text,
+          startLine,
+          endLine,
+          language,
+          tokenCount: null, // Char-based, no exact token count
+        });
+
+        seq += 1;
+      }
+
+      // Move position, accounting for overlap
+      if (endPos >= markdown.length) {
+        break;
+      }
+
+      // Calculate next position with overlap
+      const nextPos = endPos - overlapChars;
+      pos = Math.max(pos + 1, nextPos); // Ensure we always advance
+    }
+
+    return chunks;
+  }
+}
+
+/**
+ * Default chunker instance.
+ */
+export const defaultChunker = new MarkdownChunker();