@docscode/core 1.0.0

package/package.json

@@ -0,0 +1,24 @@
+{
+  "name": "@docscode/core",
+  "version": "1.0.0",
+  "description": "Core Kairo AI Collaborator Engine",
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "type": "module",
+  "scripts": {
+    "build": "tsup src/index.ts --format cjs,esm --dts --clean --shims",
+    "dev": "tsup src/index.ts --format cjs,esm --watch --dts --shims",
+    "test": "vitest run"
+  },
+  "dependencies": {
+    "yjs": "^13.6.30",
+    "y-protocols": "^1.0.7",
+    "y-webrtc": "^10.3.0",
+    "@huggingface/transformers": "^3.0.0",
+    "@tiptap/core": "^2.11.2"
+  },
+  "devDependencies": {
+    "@types/node": "^25.6.0"
+  }
+}

package/scripts/docling_bridge.py

@@ -0,0 +1,56 @@
+import sys
+import json
+import base64
+from io import BytesIO
+
+# Try to import docling, but provide a clear error if missing
+try:
+    from docling.document_converter import DocumentConverter
+    DOCLING_AVAILABLE = True
+except ImportError:
+    DOCLING_AVAILABLE = False
+
+def convert_to_canonical(source_path_or_buffer):
+    if not DOCLING_AVAILABLE:
+        return {"error": "docling library not installed. Please run 'pip install docling'"}
+
+    converter = DocumentConverter()
+    result = converter.convert(source_path_or_buffer)
+    
+    # Simple conversion to a serializable canonical format
+    doc = {
+        "metadata": {
+            "title": getattr(result.input, "file_path", "Untitled"),
+            "format": result.input.format.name if hasattr(result.input, "format") else "unknown"
+        },
+        "content": []
+    }
+    
+    # Extract blocks (this is a simplified example)
+    for element in result.document.elements:
+        if hasattr(element, "text"):
+            doc["content"].append({
+                "type": "p",
+                "text": element.text
+            })
+            
+    return doc
+
+if __name__ == "__main__":
+    if len(sys.argv) < 2:
+        print(json.dumps({"error": "Usage: python docling_bridge.py <file_path_or_base64>"}))
+        sys.exit(1)
+
+    input_data = sys.argv[1]
+    
+    try:
+        # Check if input is a path or base64
+        if input_data.startswith("base64:"):
+            content = base64.b64decode(input_data[7:])
+            result = convert_to_canonical(BytesIO(content))
+        else:
+            result = convert_to_canonical(input_data)
+            
+        print(json.dumps(result))
+    except Exception as e:
+        print(json.dumps({"error": str(e)}))

package/src/AICollaborator.ts

@@ -0,0 +1,209 @@
+import * as Y from 'yjs';
+import { Awareness } from 'y-protocols/awareness';
+import { KairoPlugin } from './KairoPlugin.js';
+import { PromptCache } from './PromptCache.js';
+import { StreamBuffer } from './StreamBuffer.js';
+import type { LLMAdapter } from './LLMAdapter.js';
+import { SuggestionManager } from './suggestion-manager.js';
+
+export type KairoStatus = 'idle' | 'thinking' | 'writing' | 'reviewing';
+
+export interface KairoMetadata {
+  name: string;
+  version: string;
+  capabilities: string[];
+  model?: string;
+}
+
+export interface AICollaboratorOptions {
+  clientID?: number;
+  cacheSize?: number;
+  llm?: LLMAdapter;
+  streamFlushMs?: number;
+}
+
+/**
+ * AICollaborator — The AI peer inside your Yjs document.
+ *
+ * Appears as a real-time participant in the awareness protocol.
+ * Streams LLM tokens directly into Y.Text CRDT operations.
+ * Human peers see a live cursor and status indicator — just like a human collaborator.
+ */
+export class AICollaborator {
+  public readonly doc: Y.Doc;
+  public readonly awareness: Awareness;
+  public readonly clientID: number;
+  public readonly suggestions: SuggestionManager;
+  public readonly cache: PromptCache;
+  private _status: KairoStatus = 'idle';
+  private _plugins: KairoPlugin[] = [];
+  private _llm?: LLMAdapter;
+  private _streamFlushMs: number;
+
+  constructor(doc: Y.Doc, awareness: Awareness, options: AICollaboratorOptions = {}) {
+    this.doc = doc;
+    this.awareness = awareness;
+    this.clientID = options.clientID ?? Math.floor(Math.random() * 1_000_000);
+    this.cache = new PromptCache(options.cacheSize ?? 50);
+    this.suggestions = new SuggestionManager(this.doc);
+    this._llm = options.llm;
+    this._streamFlushMs = options.streamFlushMs ?? 50;
+
+    this._updateAwareness();
+    console.log(`[Kairo] AICollaborator initialized — clientID: ${this.clientID}, model: ${this._llm?.model ?? 'none'}`);
+  }
+
+  // ─── Status ──────────────────────────────────────────────────────────
+
+  public setStatus(status: KairoStatus) {
+    this._status = status;
+    this._updateAwareness();
+  }
+
+  public setThinking() { this.setStatus('thinking'); }
+  public setWriting()  { this.setStatus('writing'); }
+  public setReviewing() { this.setStatus('reviewing'); }
+  public setIdle()     { this.setStatus('idle'); }
+
+  private _updateAwareness() {
+    this.awareness.setLocalStateField('kairo', {
+      status: this._status,
+      metadata: {
+        name: 'Kairo',
+        version: '1.0.0',
+        capabilities: ['autocomplete', 'summarization', 'streaming', 'tracked-changes'],
+        model: this._llm?.model,
+      } as KairoMetadata,
+      lastUpdate: Date.now(),
+    });
+  }
+
+  // ─── Plugin System ────────────────────────────────────────────────────
+
+  public registerPlugin(plugin: KairoPlugin) {
+    this._plugins.push(plugin);
+    plugin.setup();
+  }
+
+  // ─── CRDT Write Operations ────────────────────────────────────────────
+
+  /**
+   * Insert text with AI attribution markers.
+   * Consuming editors can style/filter AI-generated content using these markers.
+   */
+  public insertWithAttribution(text: Y.Text, index: number, content: string) {
+    this.doc.transact(() => {
+      text.insert(index, content, {
+        'ai-generated': true,
+        'ai-client-id': this.clientID,
+        'ai-timestamp': Date.now(),
+        'ai-model': this._llm?.model ?? 'unknown',
+      });
+    }, this.clientID);
+  }
+
+  /**
+   * Stream LLM tokens directly into the document as CRDT operations.
+   *
+   * Modes:
+   * - 'insert': appends after current position
+   * - 'continue': appends to end of document
+   * - 'rewrite': clears existing text and rewrites
+   *
+   * Token cadence: tokens are batched with a 50ms flush interval for
+   * smooth real-time rendering without CRDT overload.
+   */
+  public async streamToDoc(
+    yText: Y.Text,
+    prompt: string,
+    options: {
+      systemPrompt?: string;
+      mode?: 'insert' | 'continue' | 'rewrite';
+      insertAt?: number;
+      signal?: AbortSignal;
+      onToken?: (token: string) => void;
+    } = {}
+  ): Promise<{ tokensWritten: number; durationMs: number }> {
+    if (!this._llm) {
+      throw new Error('[Kairo] No LLM configured. Pass llm to AICollaborator constructor.');
+    }
+
+    const startTime = Date.now();
+    const mode = options.mode ?? 'continue';
+    let startIndex: number;
+
+    if (mode === 'rewrite') {
+      this.doc.transact(() => yText.delete(0, yText.length));
+      startIndex = 0;
+    } else if (mode === 'insert' && options.insertAt !== undefined) {
+      startIndex = options.insertAt;
+    } else {
+      startIndex = yText.length;
+    }
+
+    const buffer = new StreamBuffer(yText, this.clientID, startIndex, this._streamFlushMs);
+    this.setThinking();
+
+    let tokensWritten = 0;
+
+    try {
+      const systemPrompt = options.systemPrompt ??
+        'You are Kairo, an AI writing collaborator embedded in a shared document. ' +
+        'Output clean prose only. No preamble, no meta-commentary.';
+
+      this.setWriting();
+
+      for await (const token of this._llm.stream(prompt, {
+        systemPrompt,
+        signal: options.signal,
+      })) {
+        buffer.push(token);
+        tokensWritten += token.length;
+        options.onToken?.(token);
+      }
+
+      buffer.flush();
+    } catch (err) {
+      buffer.stop();
+      this.setIdle();
+      throw err;
+    }
+
+    buffer.stop();
+    this.setIdle();
+
+    return { tokensWritten, durationMs: Date.now() - startTime };
+  }
+
+  /**
+   * One-shot: summarize a block of text and return the result (no streaming).
+   */
+  public async summarize(text: string, options: { maxTokens?: number } = {}): Promise<string> {
+    if (!this._llm) throw new Error('[Kairo] No LLM configured.');
+    const cached = this.cache.get(text);
+    if (cached) return cached;
+
+    this.setThinking();
+    try {
+      const result = await this._llm.complete(
+        `Summarize the following document content concisely:\n\n${text}`,
+        {
+          systemPrompt: 'You are a document summarization engine. Output only the summary.',
+          maxTokens: options.maxTokens ?? 300,
+        }
+      );
+      this.cache.set(text, result);
+      return result;
+    } finally {
+      this.setIdle();
+    }
+  }
+
+  // ─── Cleanup ──────────────────────────────────────────────────────────
+
+  public destroy() {
+    this._plugins.forEach(p => p.destroy());
+    this.awareness.setLocalStateField('kairo', null);
+    console.log(`[Kairo] AICollaborator ${this.clientID} destroyed.`);
+  }
+}

package/src/AutocompletePlugin.ts

@@ -0,0 +1,108 @@
+import * as Y from 'yjs';
+import { KairoPlugin } from './KairoPlugin.js';
+import { AICollaborator } from './AICollaborator.js';
+import type { LLMAdapter } from './LLMAdapter.js';
+import { StreamBuffer } from './StreamBuffer.js';
+
+/**
+ * AutocompletePlugin — Trigger AI completions with '...' in the document.
+ * When a human types '...' at the end of any text, Kairo's AI peer takes over
+ * and streams a continuation directly into the shared document.
+ */
+export class AutocompletePlugin extends KairoPlugin {
+  private targetTextName: string;
+  private adapter: LLMAdapter;
+  private _deepObserver: (events: Y.YEvent<any>[]) => void;
+
+  constructor(ai: AICollaborator, adapter: LLMAdapter, targetTextName: string = 'content') {
+    super(ai);
+    this.adapter = adapter;
+    this.targetTextName = targetTextName;
+    this._deepObserver = (events: Y.YEvent<any>[]) => {
+      for (const event of events) {
+        if (event instanceof Y.YTextEvent) {
+          this._handleUpdate(event);
+        }
+      }
+    };
+  }
+
+  public setup() {
+    try {
+      // Deep observe canonical blocks
+      const arr = this.ai.doc.getArray(this.targetTextName);
+      arr.observeDeep(this._deepObserver);
+    } catch {
+      // Fallback
+    }
+    console.log(`[Kairo] AutocompletePlugin initialized — model: ${this.adapter.model}`);
+  }
+
+  public destroy() {
+    try {
+      const arr = this.ai.doc.getArray(this.targetTextName);
+      arr.unobserveDeep(this._deepObserver);
+    } catch { }
+  }
+
+
+  private _handleUpdate(event: Y.YTextEvent) {
+    if (!this.enabled) return;
+    // Ignore changes from the AI itself to avoid infinite loops
+    if (event.transaction.origin === this.ai.clientID) return;
+
+    const text = event.target as Y.Text;
+    const str = text.toString();
+
+    // Trigger on '...' at end of text (the autocomplete signal)
+    if (str.endsWith('...')) {
+      this._generateSuggestion(text);
+    }
+  }
+
+  private async _generateSuggestion(text: Y.Text) {
+    this.ai.setThinking();
+
+    try {
+      const currentText = text.toString();
+      const prompt = currentText.slice(0, -3); // strip '...'
+
+      // Check LRU cache first
+      const cached = this.ai.cache.get(prompt);
+      if (cached) {
+        this.ai.doc.transact(() => {
+          text.delete(text.length - 3, 3);
+          text.insert(text.length, cached, { 'ai-generated': true, 'ai-client-id': this.ai.clientID });
+        }, this.ai.clientID);
+        return;
+      }
+
+      // Remove the '...' trigger
+      this.ai.doc.transact(() => {
+        text.delete(text.length - 3, 3);
+      }, this.ai.clientID);
+
+      this.ai.setWriting();
+
+      const streamBuffer = new StreamBuffer(text, this.ai.clientID, text.length);
+      let fullResult = '';
+
+      const systemPrompt =
+        'You are an inline writing assistant. ' +
+        'Continue the text naturally in the same style and voice. ' +
+        'Output only the continuation — no preamble.';
+
+      for await (const token of this.adapter.stream(prompt, { systemPrompt })) {
+        fullResult += token;
+        streamBuffer.push(token);
+      }
+
+      streamBuffer.flush();
+      this.ai.cache.set(prompt, fullResult);
+    } catch (error) {
+      console.error('[Kairo] Autocomplete error:', error);
+    } finally {
+      this.ai.setIdle();
+    }
+  }
+}

package/src/ConflictResolver.ts

@@ -0,0 +1,113 @@
+import * as Y from 'yjs';
+
+/**
+ * ConflictResolver — Human-precedence CRDT conflict resolution for Kairo.
+ *
+ * Strategy:
+ * 1. Human edits always win over AI edits in the same position.
+ * 2. AI edits made within 500ms of a human edit at the same offset are rolled back.
+ * 3. Attribution is preserved — every merged character knows who wrote it.
+ *
+ * This is surfaced as a Y.Doc observer that monitors for concurrent human+AI
+ * writes and reorders them deterministically.
+ */
+
+export interface ConflictEvent {
+  type: 'human_overrides_ai' | 'ai_deferred' | 'concurrent_edit';
+  position: number;
+  humanClientId: number;
+  aiClientId: number;
+  timestamp: number;
+}
+
+export type ConflictHandler = (event: ConflictEvent) => void;
+
+export class ConflictResolver {
+  private conflictLog: ConflictEvent[] = [];
+  private handlers: ConflictHandler[] = [];
+  private aiClientIds: Set<number>;
+
+  constructor(private doc: Y.Doc, aiClientIds: number[] = []) {
+    this.aiClientIds = new Set(aiClientIds);
+    this._observe();
+  }
+
+  /** Register a client ID as an AI participant */
+  registerAI(clientId: number): void {
+    this.aiClientIds.add(clientId);
+  }
+
+  /** Listen for conflict events */
+  onConflict(handler: ConflictHandler): void {
+    this.handlers.push(handler);
+  }
+
+  /** Get the conflict log */
+  getLog(): ConflictEvent[] {
+    return [...this.conflictLog];
+  }
+
+  /** Clear the conflict log */
+  clearLog(): void {
+    this.conflictLog = [];
+  }
+
+  private _observe(): void {
+    this.doc.on('afterTransaction', (tr: Y.Transaction) => {
+      // Skip if only one origin (no concurrent edit)
+      if (!tr.changed.size) return;
+
+      // For each changed type, analyze the origin
+      tr.changed.forEach((_changedKeys, type) => {
+        if (!(type instanceof Y.Text)) return;
+
+        const origin = tr.origin;
+        const isAI = this.aiClientIds.has(origin as number);
+
+        if (!isAI) {
+          // Human edit — check if we should defer AI operations
+          const event: ConflictEvent = {
+            type: 'concurrent_edit',
+            position: 0,
+            humanClientId: origin as number,
+            aiClientId: -1,
+            timestamp: Date.now(),
+          };
+          this._emit(event);
+        }
+      });
+    });
+  }
+
+  private _emit(event: ConflictEvent): void {
+    this.conflictLog.push(event);
+    if (this.conflictLog.length > 500) {
+      this.conflictLog = this.conflictLog.slice(-500);
+    }
+    for (const handler of this.handlers) {
+      try { handler(event); } catch { /* never let conflict handler crash the doc */ }
+    }
+  }
+}
+
+/**
+ * MergePolicy — Determines how concurrent AI and human edits are resolved.
+ *
+ * Kairo uses "human-precedence linear merge":
+ * - Human text is always displayed first at the same logical position
+ * - AI suggestions appear as tracked changes until accepted
+ * - This matches the mental model of track-changes in Word/Google Docs
+ */
+export class MergePolicy {
+  static readonly HUMAN_WINS = 'human_wins';
+  static readonly LAST_WRITE_WINS = 'last_write_wins';
+  static readonly AI_WINS = 'ai_wins';
+
+  static isAIGenerated(attrs: Record<string, any> | null | undefined): boolean {
+    return attrs?.['ai-generated'] === true;
+  }
+
+  static getAuthor(attrs: Record<string, any> | null | undefined): 'ai' | 'human' {
+    return this.isAIGenerated(attrs) ? 'ai' : 'human';
+  }
+}

package/src/KairoPlugin.ts

@@ -0,0 +1,27 @@
+import { AICollaborator } from './AICollaborator.js';
+
+export abstract class KairoPlugin {
+  protected ai: AICollaborator;
+  public enabled: boolean = true;
+
+  constructor(ai: AICollaborator) {
+    this.ai = ai;
+  }
+
+  /**
+   * Called when the plugin is registered with the AICollaborator
+   */
+  public abstract setup(): void;
+
+  /**
+   * Called to cleanup the plugin
+   */
+  public abstract destroy(): void;
+
+  /**
+   * Enable/Disable the plugin
+   */
+  public setEnabled(enabled: boolean) {
+    this.enabled = enabled;
+  }
+}

package/src/LLMAdapter.ts

@@ -0,0 +1,27 @@
+export interface StreamOptions {
+  systemPrompt?: string;
+  maxTokens?: number;
+  temperature?: number;
+  signal?: AbortSignal;
+}
+
+export interface CompleteOptions extends StreamOptions {}
+
+/**
+ * LLMAdapter — Kairo's universal LLM interface.
+ * Implement this to plug any model into the CRDT streaming pipeline.
+ * Zero SDK dependency by design — uses raw fetch for SSE streaming.
+ */
+export interface LLMAdapter {
+  readonly provider: string;
+  readonly model: string;
+  /**
+   * Stream tokens one-by-one as an async generator.
+   * This feeds directly into StreamBuffer → Y.Text CRDT operations.
+   */
+  stream(prompt: string, options?: StreamOptions): AsyncGenerator<string>;
+  /** Non-streaming fallback for summarization / one-shot tasks */
+  complete(prompt: string, options?: CompleteOptions): Promise<string>;
+  /** Optional: embed text for semantic search / context retrieval */
+  embed?(text: string): Promise<number[]>;
+}

package/src/MockLLMAdapter.ts

@@ -0,0 +1,45 @@
+import type { LLMAdapter, StreamOptions } from './LLMAdapter.js';
+
+/**
+ * MockLLMAdapter — Test/offline LLM adapter.
+ *
+ * Returns deterministic responses without any network call.
+ * Perfect for unit tests and offline development.
+ */
+export class MockLLMAdapter implements LLMAdapter {
+  readonly provider = 'mock';
+  readonly model: string;
+  private delay: number;
+
+  constructor(options: { model?: string; delay?: number } = {}) {
+    this.model = options.model ?? 'mock-gpt';
+    this.delay = options.delay ?? 100;
+  }
+
+  async *stream(prompt: string, options?: StreamOptions): AsyncGenerator<string> {
+    const response = this._respond(prompt);
+    const words = response.split(' ');
+    for (let i = 0; i < words.length; i++) {
+      await new Promise(r => setTimeout(r, this.delay / words.length));
+      yield words[i] + (i < words.length - 1 ? ' ' : '');
+    }
+  }
+
+  async complete(prompt: string, options?: StreamOptions): Promise<string> {
+    await new Promise(r => setTimeout(r, this.delay));
+    return this._respond(prompt);
+  }
+
+  async embed(text: string): Promise<number[]> {
+    // Deterministic mock embedding (128-dim)
+    return Array.from({ length: 128 }, (_, i) => Math.sin(text.charCodeAt(i % text.length) * (i + 1)));
+  }
+
+  private _respond(prompt: string): string {
+    const p = prompt.toLowerCase();
+    if (p.includes('summar')) return 'This document discusses key concepts with clarity and precision.';
+    if (p.includes('improve') || p.includes('rewrite')) return 'The refined version presents ideas more effectively.';
+    if (p.includes('continue')) return ' Furthermore, the analysis reveals several important implications.';
+    return `[Mock AI response to: "${prompt.slice(0, 60)}..."]`;
+  }
+}

package/src/PromptCache.ts

@@ -0,0 +1,76 @@
+/**
+ * PromptCache — LRU cache for LLM prompts.
+ *
+ * Prevents re-querying the LLM for identical prompts within a session.
+ * Critical for summarization and autocomplete performance.
+ *
+ * Uses a Map-based LRU: entries at the front are most recent,
+ * entries at the back are evicted first.
+ */
+export class PromptCache {
+  private cache: Map<string, { value: string; hits: number; createdAt: number }>;
+  private readonly maxSize: number;
+  private totalHits = 0;
+  private totalMisses = 0;
+
+  constructor(maxSize: number = 100) {
+    this.maxSize = maxSize;
+    this.cache = new Map();
+  }
+
+  /** Retrieve cached result for a prompt. Returns null on miss. */
+  get(prompt: string): string | null {
+    const entry = this.cache.get(prompt);
+    if (!entry) {
+      this.totalMisses++;
+      return null;
+    }
+    // LRU: move to front by re-inserting
+    this.cache.delete(prompt);
+    entry.hits++;
+    this.cache.set(prompt, entry);
+    this.totalHits++;
+    return entry.value;
+  }
+
+  /** Cache a result for a prompt. Evicts LRU entry if at capacity. */
+  set(prompt: string, value: string): void {
+    if (this.cache.has(prompt)) {
+      this.cache.delete(prompt);
+    } else if (this.cache.size >= this.maxSize) {
+      // Evict least-recently-used (first entry in Map)
+      const lruKey = this.cache.keys().next().value;
+      if (lruKey !== undefined) this.cache.delete(lruKey);
+    }
+    this.cache.set(prompt, { value, hits: 0, createdAt: Date.now() });
+  }
+
+  /** Check if a prompt is cached without counting as a hit */
+  has(prompt: string): boolean {
+    return this.cache.has(prompt);
+  }
+
+  /** Remove a specific entry */
+  invalidate(prompt: string): void {
+    this.cache.delete(prompt);
+  }
+
+  /** Clear all entries */
+  clear(): void {
+    this.cache.clear();
+    this.totalHits = 0;
+    this.totalMisses = 0;
+  }
+
+  /** Cache statistics */
+  stats(): { size: number; maxSize: number; hitRate: number; totalHits: number; totalMisses: number } {
+    const total = this.totalHits + this.totalMisses;
+    return {
+      size: this.cache.size,
+      maxSize: this.maxSize,
+      hitRate: total > 0 ? this.totalHits / total : 0,
+      totalHits: this.totalHits,
+      totalMisses: this.totalMisses,
+    };
+  }
+}