kotadb 2.0.0

package/src/sync/deletion-manifest.ts

@@ -0,0 +1,210 @@
+/**
+ * Deletion manifest for tracking removed entities during sync
+ * 
+ * Problem: JSONL export only captures current state. If you delete
+ * a repository locally, other machines won't know to delete it on
+ * import because it's simply absent from the export.
+ * 
+ * Solution: .deletions.jsonl manifest tracks deletions explicitly:
+ * ```jsonl
+ * {"table":"repositories","id":"abc-123","deleted_at":"2025-12-15T10:30:00Z"}
+ * {"table":"indexed_files","id":"def-456","deleted_at":"2025-12-15T10:31:00Z"}
+ * ```
+ * 
+ * Lifecycle:
+ * 1. Export: Record deletions in manifest
+ * 2. Git: Manifest syncs like any other file
+ * 3. Import: Apply deletions before importing new data
+ * 4. Cleanup: Clear manifest after successful import
+ * 
+ * @module @sync/deletion-manifest
+ */
+
+import { existsSync, appendFileSync } from "node:fs";
+import { join } from "node:path";
+import { createLogger } from "@logging/logger.js";
+import type { KotaDatabase } from "@db/sqlite/sqlite-client.js";
+import { getDefaultExportDir } from "@db/sqlite/jsonl-exporter.js";
+
+const logger = createLogger({ module: "deletion-manifest" });
+
+/**
+ * Deletion entry in manifest
+ */
+export interface DeletionEntry {
+  table: string;
+  id: string;
+  deleted_at: string;
+}
+
+/**
+ * Record a deletion in the manifest
+ */
+export async function recordDeletion(
+  table: string,
+  id: string,
+  exportDir: string = getDefaultExportDir()
+): Promise<void> {
+  const manifestPath = join(exportDir, ".deletions.jsonl");
+
+  const entry: DeletionEntry = {
+    table,
+    id,
+    deleted_at: new Date().toISOString()
+  };
+
+  const line = JSON.stringify(entry) + "\n";
+
+  // Append to manifest (create if doesn't exist)
+  appendFileSync(manifestPath, line, "utf-8");
+
+  logger.debug("Deletion recorded", { table, id });
+}
+
+/**
+ * Load all deletions from manifest
+ */
+export async function loadDeletionManifest(
+  manifestPath: string
+): Promise<DeletionEntry[]> {
+  if (!existsSync(manifestPath)) {
+    return [];
+  }
+
+  const content = await Bun.file(manifestPath).text();
+  const lines = content.trim().split("\n").filter(Boolean);
+  const entries: DeletionEntry[] = [];
+
+  for (let idx = 0; idx < lines.length; idx++) {
+    const line = lines[idx];
+    try {
+      const entry = JSON.parse(line as string) as DeletionEntry;
+      entries.push(entry);
+    } catch (error) {
+      logger.warn("Invalid deletion entry, skipping", {
+        line_number: idx + 1,
+        error: error instanceof Error ? error.message : String(error)
+      });
+    }
+  }
+
+  return entries;
+}
+
+/**
+ * Apply deletions from manifest to database
+ */
+export async function applyDeletionManifest(
+  db: KotaDatabase,
+  manifestPath: string
+): Promise<{ deletedCount: number; errors: string[] }> {
+  const entries = await loadDeletionManifest(manifestPath);
+
+  if (entries.length === 0) {
+    logger.debug("No deletions to apply");
+    return { deletedCount: 0, errors: [] };
+  }
+
+  logger.info("Applying deletion manifest", { entry_count: entries.length });
+
+  const errors: string[] = [];
+  let deletedCount = 0;
+
+  // Group by table for batch deletions
+  const byTable = new Map<string, string[]>();
+  for (const entry of entries) {
+    if (!byTable.has(entry.table)) {
+      byTable.set(entry.table, []);
+    }
+    byTable.get(entry.table)!.push(entry.id);
+  }
+
+  // Apply deletions in transaction
+  try {
+    db.immediateTransaction(() => {
+      for (const [table, ids] of byTable) {
+        if (!db.tableExists(table)) {
+          logger.warn("Deletion target table not found, skipping", { table });
+          errors.push(`Table not found: ${table}`);
+          continue;
+        }
+
+        // DELETE FROM table WHERE id IN (?, ?, ...)
+        const placeholders = ids.map(() => "?").join(", ");
+        const sql = `DELETE FROM ${table} WHERE id IN (${placeholders})`;
+
+        try {
+          db.run(sql, ids as (string | number | bigint | boolean | null | Uint8Array)[]);
+          // Note: db.run() doesn't return changes count, so we count the IDs
+          deletedCount += ids.length;
+
+          logger.debug("Deleted entries from table", {
+            table,
+            deleted_count: ids.length
+          });
+        } catch (error) {
+          const errorMsg = error instanceof Error ? error.message : String(error);
+          logger.error(`Failed to delete from ${table}`, new Error(errorMsg));
+          errors.push(`${table}: ${errorMsg}`);
+        }
+      }
+    });
+  } catch (txError) {
+    const errorMsg = txError instanceof Error ? txError.message : String(txError);
+    logger.error("Deletion transaction failed", new Error(errorMsg));
+    return { deletedCount: 0, errors: [errorMsg] };
+  }
+
+  logger.info("Deletion manifest applied", {
+    deleted_count: deletedCount,
+    error_count: errors.length
+  });
+
+  return { deletedCount, errors };
+}
+
+/**
+ * Clear deletion manifest after successful import
+ */
+export async function clearDeletionManifest(
+  exportDir: string = getDefaultExportDir()
+): Promise<void> {
+  const manifestPath = join(exportDir, ".deletions.jsonl");
+
+  if (!existsSync(manifestPath)) {
+    return;
+  }
+
+  await Bun.write(manifestPath, "");
+  logger.info("Deletion manifest cleared");
+}
+
+/**
+ * Hook into database operations to track deletions
+ * 
+ * Usage:
+ * ```typescript
+ * const db = getClient();
+ * trackDeletions(db);
+ * 
+ * // Now all DELETE operations are logged to manifest
+ * db.run("DELETE FROM repositories WHERE id = ?", ["abc-123"]);
+ * ```
+ * 
+ * Note: This is a post-Phase-3 enhancement. For MVP, manual calls
+ * to recordDeletion() are sufficient.
+ */
+export function trackDeletions(
+  db: KotaDatabase,
+  exportDir: string = getDefaultExportDir()
+): void {
+  // Implementation note: This would require wrapping db.run() to intercept
+  // DELETE statements and extract table/id pairs. Complex pattern matching
+  // needed. Consider for Phase 3B or Phase 4.
+
+  logger.warn("Automatic deletion tracking not yet implemented");
+  logger.info(
+    "Use recordDeletion(table, id) manually after deletions",
+    { export_dir: exportDir }
+  );
+}

package/src/sync/index.ts

@@ -0,0 +1,16 @@
+/**
+ * Sync layer module exports
+ * 
+ * @module @sync
+ */
+
+export { SyncWatcher, createWatcher } from "./watcher.js";
+export { runMergeDriver } from "./merge-driver.js";
+export {
+  recordDeletion,
+  loadDeletionManifest,
+  applyDeletionManifest,
+  clearDeletionManifest,
+  trackDeletions,
+  type DeletionEntry
+} from "./deletion-manifest.js";

package/src/sync/merge-driver.ts

@@ -0,0 +1,172 @@
+/**
+ * Custom git merge driver for JSONL files
+ * 
+ * Resolves conflicts in .jsonl files using line-based reconciliation:
+ * - Lines with same ID: use THEIRS (assume remote is authoritative)
+ * - Lines unique to OURS: keep them
+ * - Lines unique to THEIRS: keep them
+ * 
+ * Algorithm:
+ * 1. Parse BASE, OURS, THEIRS into ID-keyed maps
+ * 2. Collect all unique IDs across versions
+ * 3. For each ID, choose THEIRS if present, else OURS
+ * 4. Sort by ID for deterministic output
+ * 5. Write merged JSONL to OURS path
+ * 
+ * Installation:
+ * ```bash
+ * # Add to .git/config or ~/.gitconfig
+ * [merge "jsonl"]
+ *   name = JSONL merge driver
+ *   driver = bun run src/sync/merge-driver.ts %O %A %B %L
+ * ```
+ * 
+ * @module @sync/merge-driver
+ */
+
+import { readFileSync, writeFileSync } from "node:fs";
+import { createLogger } from "@logging/logger.js";
+
+const logger = createLogger({ module: "merge-driver" });
+
+/**
+ * Parsed JSONL entry with ID
+ */
+interface JSONLEntry {
+  id: string;
+  line: string;
+  data: Record<string, unknown>;
+}
+
+/**
+ * Parse JSONL file into ID-keyed map
+ */
+function parseJSONL(filepath: string): Map<string, JSONLEntry> {
+  const content = readFileSync(filepath, "utf-8");
+  const lines = content.trim().split("\n").filter(Boolean);
+  const entries = new Map<string, JSONLEntry>();
+
+  for (const line of lines) {
+    try {
+      const data = JSON.parse(line) as Record<string, unknown>;
+      const id = data.id as string;
+
+      if (!id) {
+        logger.warn("JSONL entry missing ID, skipping", { line });
+        continue;
+      }
+
+      entries.set(id, { id, line, data });
+    } catch (error) {
+      logger.error(
+        "Failed to parse JSONL line",
+        error instanceof Error ? error : new Error(String(error)),
+        { line }
+      );
+    }
+  }
+
+  return entries;
+}
+
+/**
+ * Merge three JSONL versions (base, ours, theirs)
+ * 
+ * Strategy: THEIRS-preferred merge
+ * - If ID in THEIRS: use THEIRS
+ * - Else if ID in OURS: use OURS
+ * - Else: skip (was deleted in both)
+ */
+function mergeJSONL(
+  basePath: string,
+  oursPath: string,
+  theirsPath: string
+): string {
+  const base = parseJSONL(basePath);
+  const ours = parseJSONL(oursPath);
+  const theirs = parseJSONL(theirsPath);
+
+  // Collect all IDs
+  const allIds = new Set<string>([
+    ...base.keys(),
+    ...ours.keys(),
+    ...theirs.keys()
+  ]);
+
+  // Merge: prefer THEIRS, fallback to OURS
+  const merged: JSONLEntry[] = [];
+  for (const id of allIds) {
+    if (theirs.has(id)) {
+      merged.push(theirs.get(id)!);
+    } else if (ours.has(id)) {
+      merged.push(ours.get(id)!);
+    }
+    // If neither has it, ID was deleted in both - skip
+  }
+
+  // Sort by ID for deterministic output
+  merged.sort((a, b) => a.id.localeCompare(b.id));
+
+  // Format as JSONL
+  return merged.map((entry) => entry.line).join("\n") + "\n";
+}
+
+/**
+ * Main merge driver entry point
+ * 
+ * Git invokes as: merge-driver %O %A %B %L
+ * - %O: base version path
+ * - %A: ours version path (current branch)
+ * - %B: theirs version path (incoming branch)
+ * - %L: conflict marker size (unused)
+ */
+export function runMergeDriver(
+  basePath: string,
+  oursPath: string,
+  theirsPath: string,
+  _markerSize: string
+): number {
+  logger.info("JSONL merge driver invoked", {
+    base: basePath,
+    ours: oursPath,
+    theirs: theirsPath
+  });
+
+  try {
+    const merged = mergeJSONL(basePath, oursPath, theirsPath);
+
+    // Write merged result to OURS path
+    writeFileSync(oursPath, merged, "utf-8");
+
+    logger.info("JSONL merge completed successfully", {
+      output: oursPath
+    });
+
+    return 0; // Success
+  } catch (error) {
+    logger.error(
+      "JSONL merge failed",
+      error instanceof Error ? error : new Error(String(error)),
+      {
+        base: basePath,
+        ours: oursPath,
+        theirs: theirsPath
+      }
+    );
+
+    return 1; // Conflict (git will mark file as conflicted)
+  }
+}
+
+// CLI entry point (when run via `bun run merge-driver.ts`)
+if (import.meta.main) {
+  const [basePath, oursPath, theirsPath, markerSize] = process.argv.slice(2);
+
+  if (!basePath || !oursPath || !theirsPath) {
+    process.stderr.write("Usage: merge-driver.ts <base> <ours> <theirs> <marker-size>\n");
+    process.exit(1);
+  }
+
+  const exitCode = runMergeDriver(basePath, oursPath, theirsPath, markerSize || "7");
+  process.exit(exitCode);
+}

package/src/sync/watcher.ts

@@ -0,0 +1,221 @@
+/**
+ * File watcher for automatic JSONL import on git pull
+ * 
+ * Watches .kotadb/export/*.jsonl (project-local) for changes and triggers
+ * import when modifications are detected (e.g., after git pull).
+ * 
+ * Features:
+ * - Debounced import (1-second delay to batch rapid changes)
+ * - Hash-based change detection (skip unchanged files)
+ * - Graceful error handling (log failures, don't crash)
+ * 
+ * @module @sync/watcher
+ */
+
+import { watch, type FSWatcher } from "node:fs";
+import { existsSync } from "node:fs";
+import { join } from "node:path";
+import { createLogger } from "@logging/logger.js";
+import { getDefaultExportDir } from "@db/sqlite/jsonl-exporter.js";
+import { importFromJSONL } from "@db/sqlite/jsonl-importer.js";
+import type { KotaDatabase } from "@db/sqlite/sqlite-client.js";
+import { getClient } from "@db/client.js";
+import { applyDeletionManifest } from "@sync/deletion-manifest.js";
+
+const logger = createLogger({ module: "sync-watcher" });
+
+/**
+ * Watcher state for debouncing imports
+ */
+interface WatcherState {
+  timer: ReturnType<typeof setTimeout> | null;
+  changedFiles: Set<string>;
+  lastImportAt: string;
+}
+
+/**
+ * SyncWatcher - Watches JSONL export directory for changes.
+ * 
+ * Usage:
+ * ```typescript
+ * const watcher = new SyncWatcher();
+ * watcher.start();
+ * 
+ * // Later...
+ * watcher.stop();
+ * ```
+ */
+export class SyncWatcher {
+  private fsWatcher: FSWatcher | null = null;
+  private state: WatcherState;
+  private readonly exportDir: string;
+  private readonly debounceMs: number;
+
+  constructor(
+    exportDir: string = getDefaultExportDir(),
+    debounceMs: number = 1000
+  ) {
+    this.exportDir = exportDir;
+    this.debounceMs = debounceMs;
+    this.state = {
+      timer: null,
+      changedFiles: new Set(),
+      lastImportAt: new Date().toISOString()
+    };
+  }
+
+  /**
+   * Start watching the export directory
+   */
+  start(): void {
+    if (this.fsWatcher) {
+      logger.warn("Watcher already started");
+      return;
+    }
+
+    if (!existsSync(this.exportDir)) {
+      logger.error("Export directory not found", { path: this.exportDir });
+      throw new Error(`Export directory not found: ${this.exportDir}`);
+    }
+
+    this.fsWatcher = watch(
+      this.exportDir,
+      { recursive: false },
+      (eventType, filename) => {
+        if (!filename || !filename.endsWith(".jsonl")) {
+          return;
+        }
+
+        // Skip deletion manifest (handled separately)
+        if (filename === ".deletions.jsonl") {
+          return;
+        }
+
+        logger.debug("File change detected", {
+          event: eventType,
+          file: filename
+        });
+
+        this.state.changedFiles.add(filename);
+        this.scheduleImport();
+      }
+    );
+
+    logger.info("Sync watcher started", {
+      export_dir: this.exportDir,
+      debounce_ms: this.debounceMs
+    });
+  }
+
+  /**
+   * Stop watching (cleanup)
+   */
+  stop(): void {
+    if (this.state.timer) {
+      clearTimeout(this.state.timer);
+      this.state.timer = null;
+    }
+
+    if (this.fsWatcher) {
+      this.fsWatcher.close();
+      this.fsWatcher = null;
+      logger.info("Sync watcher stopped");
+    }
+  }
+
+  /**
+   * Schedule import with debouncing
+   */
+  private scheduleImport(): void {
+    if (this.state.timer) {
+      clearTimeout(this.state.timer);
+    }
+
+    this.state.timer = setTimeout(() => {
+      this.executeImport().catch((error) => {
+        logger.error(
+          "Scheduled import failed",
+          error instanceof Error ? error : new Error(String(error))
+        );
+      });
+    }, this.debounceMs);
+
+    logger.debug("Import scheduled (debounced)", {
+      changed_files: Array.from(this.state.changedFiles)
+    });
+  }
+
+  /**
+   * Execute import for all changed files
+   */
+  private async executeImport(): Promise<void> {
+    const changedFiles = Array.from(this.state.changedFiles);
+    this.state.changedFiles.clear();
+
+    logger.info("Starting automatic import", {
+      files: changedFiles,
+      count: changedFiles.length
+    });
+
+    const startTime = Date.now();
+
+    try {
+      const db = getClient() as KotaDatabase;
+
+      // Import JSONL files
+      const result = await importFromJSONL(db, this.exportDir);
+
+      // Apply deletion manifest if present
+      const deletionManifestPath = join(this.exportDir, ".deletions.jsonl");
+      if (existsSync(deletionManifestPath)) {
+        await applyDeletionManifest(db, deletionManifestPath);
+      }
+
+      const duration = Date.now() - startTime;
+
+      logger.info("Automatic import completed", {
+        tables_imported: result.tablesImported,
+        rows_imported: result.totalRowsImported,
+        duration_ms: duration,
+        errors: result.errors
+      });
+
+      this.state.lastImportAt = new Date().toISOString();
+    } catch (error) {
+      logger.error(
+        "Import failed",
+        error instanceof Error ? error : new Error(String(error)),
+        {
+          changed_files: changedFiles
+        }
+      );
+    }
+  }
+
+  /**
+   * Get current watcher state (for debugging)
+   */
+  getState(): {
+    isRunning: boolean;
+    lastImportAt: string;
+    pendingFiles: string[];
+  } {
+    return {
+      isRunning: this.fsWatcher !== null,
+      lastImportAt: this.state.lastImportAt,
+      pendingFiles: Array.from(this.state.changedFiles)
+    };
+  }
+}
+
+/**
+ * Factory function to create and start watcher
+ */
+export function createWatcher(
+  exportDir?: string,
+  debounceMs?: number
+): SyncWatcher {
+  const watcher = new SyncWatcher(exportDir, debounceMs);
+  watcher.start();
+  return watcher;
+}

package/src/types/rate-limit.ts

@@ -0,0 +1,88 @@
+/**
+ * Rate limiting types for KotaDB API.
+ *
+ * Types for rate limit enforcement and response headers.
+ * Backend-only types (moved from shared/ to resolve Docker build context issue).
+ */
+
+import type { Tier } from "@shared/types/auth";
+
+/**
+ * Rate limit enforcement result.
+ * Contains current status and metadata for response headers.
+ */
+export interface RateLimitResult {
+	/** Whether the request is allowed (within limit) */
+	allowed: boolean;
+
+	/** Requests remaining in current window */
+	remaining: number;
+
+	/** Seconds until window resets (only set when limit exceeded) */
+	retryAfter?: number;
+
+	/** Unix timestamp when window resets */
+	resetAt: number;
+
+	/** Total limit for this key's tier */
+	limit: number;
+}
+
+/**
+ * Rate limit response headers.
+ * Standard header names for rate limit metadata.
+ */
+export interface RateLimitHeaders {
+	/** Total requests allowed per hour for the tier */
+	"X-RateLimit-Limit": string;
+
+	/** Requests remaining in current window */
+	"X-RateLimit-Remaining": string;
+
+	/** Unix timestamp when the limit resets */
+	"X-RateLimit-Reset": string;
+
+	/** Seconds until retry (only present in 429 responses) */
+	"Retry-After"?: string;
+}
+
+/**
+ * Rate limit configuration by tier.
+ * Defines request quotas for each subscription level.
+ */
+export interface RateLimitConfig {
+	/** Subscription tier */
+	tier: Tier;
+
+	/** Requests allowed per hour */
+	requestsPerHour: number;
+}
+
+/**
+ * Import centralized rate limit configuration.
+ * Re-exported for backward compatibility.
+ */
+import { RATE_LIMITS as CONFIG_RATE_LIMITS } from "@config/constants";
+
+/**
+ * Standard hourly rate limit configurations by tier.
+ * Updated in #423 to support realistic development workflows.
+ * Re-exported from @config/constants for convenience.
+ */
+export const RATE_LIMITS: Record<Tier, number> = {
+	free: CONFIG_RATE_LIMITS.FREE.HOURLY,
+	solo: CONFIG_RATE_LIMITS.SOLO.HOURLY,
+	team: CONFIG_RATE_LIMITS.TEAM.HOURLY,
+};
+
+/**
+ * Daily rate limit configurations by tier.
+ * Provides cost protection while enabling burst usage patterns.
+ * Both hourly and daily limits are enforced; whichever is reached first blocks requests.
+ * Re-exported from @config/constants for convenience.
+ */
+export const DAILY_RATE_LIMITS: Record<Tier, number> = {
+	free: CONFIG_RATE_LIMITS.FREE.DAILY,
+	solo: CONFIG_RATE_LIMITS.SOLO.DAILY,
+	team: CONFIG_RATE_LIMITS.TEAM.DAILY,
+};