forge-server 0.1.0

package/src/server.ts

@@ -0,0 +1,352 @@
+// server.ts — B19
+// MCP server setup: initialises all services and registers all tools.
+//
+// Startup sequence:
+//   1. Initialize SQLite (PipelineDB) — always succeeds; creates file if needed
+//   2. Attempt Qdrant connection with graceful degradation on failure
+//   3. Attempt FalkorDB connection with graceful degradation on failure
+//   4. Create ConfidenceManager for knowledge confidence tracking
+//   5. Create file content cache
+//   6. Create service instances (engine, registry, search, context, injector)
+//   7. Collect all tool handlers (pipeline, phase, context, registration,
+//      learning, ingestion, memory, graph, collaboration)
+//   8. Register all tools with the MCP server
+//   9. Start periodic confidence decay (every 24 hours)
+//
+// Lazy loading: embedding model is NOT loaded here. It is lazy-loaded in
+// embedder.ts on first use. This keeps server startup under 2 seconds.
+
+import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+
+import { PipelineDB } from './storage/sqlite.js';
+import { QdrantVectorStore } from './storage/qdrant-store.js';
+import { FalkorDBGraphStore } from './storage/falkordb-store.js';
+import { LRUFileContentCache } from './storage/file-cache.js';
+
+import { PipelineEngine } from './pipeline/engine.js';
+import { RepoRegistry } from './knowledge/registry.js';
+import { KnowledgeYamlStore } from './knowledge/store.js';
+import { KnowledgeHydrator } from './knowledge/hydrator.js';
+import { KnowledgeSearch } from './knowledge/search.js';
+
+import { CodebaseContext } from './context/codebase.js';
+import { MemoryBridge } from './context/memory.js';
+import { ContextInjector } from './context/injector.js';
+
+import { createPipelineTools } from './tools/pipeline-tools.js';
+import { createPhaseTools } from './tools/phase-tools.js';
+import { createContextTools } from './tools/context-tools.js';
+import { createRegistrationTools } from './tools/registration-tools.js';
+import { createLearningTools } from './tools/learning-tools.js';
+import { createIngestionTools } from './tools/ingestion-tools.js';
+import { createMemoryTools } from './tools/memory-tools.js';
+import { createGraphTools } from './tools/graph-tools.js';
+import { createCollaborationTools } from './tools/collaboration-tools.js';
+
+import { ConfidenceManager } from './learning/confidence.js';
+
+import { logger } from './util/logger.js';
+
+// ---------------------------------------------------------------------------
+// Default paths
+// ---------------------------------------------------------------------------
+
+const DEFAULT_DB_PATH = join(homedir(), '.forge', 'pipeline.db');
+const QDRANT_URL = process.env['QDRANT_URL'] ?? 'http://localhost:6333';
+const FALKORDB_HOST = process.env['FALKORDB_HOST'] ?? 'localhost';
+const FALKORDB_PORT = parseInt(process.env['FALKORDB_PORT'] ?? '6380', 10);
+
+// ---------------------------------------------------------------------------
+// Server info
+// ---------------------------------------------------------------------------
+
+const SERVER_NAME = 'dk-forge';
+const SERVER_VERSION = '0.1.0';
+
+// ---------------------------------------------------------------------------
+// createForgeServer
+// ---------------------------------------------------------------------------
+
+export async function createForgeServer(): Promise<McpServer> {
+  const startTime = Date.now();
+  logger.info(`${SERVER_NAME}: initializing`, { version: SERVER_VERSION });
+
+  // -------------------------------------------------------------------------
+  // 1. SQLite — always succeeds; creates ~/.forge/pipeline.db if needed
+  // -------------------------------------------------------------------------
+  const dbPath = process.env['FORGE_DB_PATH'] ?? DEFAULT_DB_PATH;
+  const db = new PipelineDB(dbPath);
+  db.initialize();
+  logger.info('server: SQLite initialized', { path: dbPath });
+
+  // -------------------------------------------------------------------------
+  // 2. Qdrant — graceful degradation if unavailable
+  // -------------------------------------------------------------------------
+  let vectorStore: QdrantVectorStore | null = null;
+  try {
+    const qdrant = new QdrantVectorStore(QDRANT_URL);
+    const qdrantHealthy = await qdrant.isHealthy();
+    if (qdrantHealthy) {
+      // Ensure collections exist (384-dim for all-MiniLM-L6-v2)
+      await qdrant.ensureCollections(384);
+      vectorStore = qdrant;
+      logger.info('server: Qdrant connected', { url: QDRANT_URL });
+    } else {
+      logger.warn('server: Qdrant unavailable — knowledge search and codebase context will be degraded', {
+        url: QDRANT_URL,
+      });
+    }
+  } catch (err) {
+    logger.warn('server: Qdrant connection failed — running without vector search', {
+      url: QDRANT_URL,
+      error: String(err),
+    });
+  }
+
+  // -------------------------------------------------------------------------
+  // 3. FalkorDB — graceful degradation if unavailable
+  // -------------------------------------------------------------------------
+  let graphStore: FalkorDBGraphStore | null = null;
+  try {
+    const falkor = new FalkorDBGraphStore(
+      `redis://${FALKORDB_HOST}:${FALKORDB_PORT}`,
+    );
+    await falkor.connect();
+    const falkorHealthy = await falkor.isHealthy();
+    if (falkorHealthy) {
+      graphStore = falkor;
+      logger.info('server: FalkorDB connected', { host: FALKORDB_HOST, port: FALKORDB_PORT });
+    } else {
+      logger.warn('server: FalkorDB unavailable — graph-boosted search disabled, using vector-only');
+      try { await falkor.disconnect(); } catch {}
+    }
+  } catch (err) {
+    logger.warn('server: FalkorDB connection failed — running without graph store', {
+      error: String(err),
+    });
+  }
+
+  // -------------------------------------------------------------------------
+  // 4. Confidence manager — tracks knowledge item usage and decay
+  // -------------------------------------------------------------------------
+  const confidenceManager = new ConfidenceManager();
+
+  // -------------------------------------------------------------------------
+  // 5. File content cache
+  // -------------------------------------------------------------------------
+  const fileCache = new LRUFileContentCache({
+    maxSizeBytes: 128 * 1024 * 1024, // 128 MB
+    maxEntries: 2000,
+    ttlMs: 30 * 60 * 1000, // 30 minutes
+  });
+
+  // -------------------------------------------------------------------------
+  // 6. Service instances
+  // -------------------------------------------------------------------------
+
+  // Knowledge hydrator — wires YAML store + Qdrant
+  // The KnowledgeHydrator's yamlStore is overridden per-registration call inside RepoRegistry,
+  // so we pass a placeholder that uses ~/.forge as the base directory.
+  const knowledgeHydrator = vectorStore
+    ? new KnowledgeHydrator(
+        new KnowledgeYamlStore(join(homedir(), '.forge')),
+        vectorStore,
+      )
+    : undefined;
+
+  // Repo registry — passes the hydrator for knowledge sync on registration
+  const registry = new RepoRegistry(db, knowledgeHydrator);
+
+  // Pipeline engine
+  const engine = new PipelineEngine(db);
+
+  // Knowledge search — wraps Qdrant knowledge collection
+  const knowledgeSearch = vectorStore
+    ? new KnowledgeSearch(vectorStore)
+    : createDegradedKnowledgeSearch();
+
+  // Codebase context — wraps hybrid search
+  // Sharing resolver: looks up a repo's sharing mode from the registry so that
+  // cross-repo search results from private repos are filtered out.
+  const codebaseCtx = new CodebaseContext(
+    graphStore,
+    vectorStore,
+    fileCache,
+    (repoId) => registry.getRepo(repoId)?.sharing,
+  );
+
+  // Memory bridge — wraps Qdrant observations collection
+  const memoryBridge = new MemoryBridge(vectorStore);
+
+  // Context injector — orchestrates all context assembly
+  // Accepts confidenceManager + vectorStore to auto-boost knowledge confidence
+  // on consumption (Phase 2 swarm learning).
+  const contextInjector = new ContextInjector(
+    knowledgeSearch,
+    codebaseCtx,
+    memoryBridge,
+    db,
+    confidenceManager,
+    vectorStore as QdrantVectorStore | undefined,
+  );
+
+  // -------------------------------------------------------------------------
+  // 7. Collect all tool handlers
+  // -------------------------------------------------------------------------
+
+  // Existing tool factories — updated with new params for swarm learning
+  const pipelineTools = createPipelineTools(engine, registry);
+  const phaseTools = createPhaseTools(engine, contextInjector, vectorStore, graphStore);
+  const contextTools = createContextTools(
+    knowledgeSearch, codebaseCtx, engine,
+    memoryBridge, vectorStore, graphStore,
+  );
+  const registrationTools = createRegistrationTools(registry, {
+    vectorStore,
+    graphStore,
+    fileCache,
+  });
+  const learningTools = createLearningTools(
+    knowledgeSearch,
+    confidenceManager,
+    vectorStore as QdrantVectorStore | null,
+  );
+  const ingestionTools = createIngestionTools(graphStore);
+
+  // New tool factories — swarm learning, graph analysis, collaboration
+  const memoryTools = createMemoryTools(memoryBridge, vectorStore, graphStore);
+  const graphTools = createGraphTools(graphStore);
+  const collaborationTools = createCollaborationTools(vectorStore, graphStore, db);
+
+  const allTools = {
+    ...pipelineTools,
+    ...phaseTools,
+    ...contextTools,
+    ...registrationTools,
+    ...learningTools,
+    ...ingestionTools,
+    ...memoryTools,
+    ...graphTools,
+    ...collaborationTools,
+  };
+
+  // -------------------------------------------------------------------------
+  // 8. Create and configure MCP server
+  // -------------------------------------------------------------------------
+  const server = new McpServer({
+    name: SERVER_NAME,
+    version: SERVER_VERSION,
+  });
+
+  // Register each tool with the MCP server
+  for (const [name, tool] of Object.entries(allTools)) {
+    const { schema, description, handler } = tool as {
+      schema: { shape: Record<string, unknown> };
+      description: string;
+      handler: (input: Record<string, unknown>) => Promise<unknown>;
+    };
+
+    server.tool(
+      name,
+      description,
+      // MCP SDK accepts raw Zod shape objects (not the full z.object wrapper)
+      (schema as unknown as { shape: Record<string, unknown> }).shape,
+      async (input: Record<string, unknown>) => {
+        try {
+          const result = await handler(input);
+          return {
+            content: [
+              {
+                type: 'text' as const,
+                text: JSON.stringify(result, null, 2),
+              },
+            ],
+          };
+        } catch (err) {
+          const errorPayload = {
+            error: 'TOOL_EXECUTION_ERROR',
+            tool: name,
+            message: String(err),
+          };
+          return {
+            content: [
+              {
+                type: 'text' as const,
+                text: JSON.stringify(errorPayload, null, 2),
+              },
+            ],
+            isError: true,
+          };
+        }
+      },
+    );
+
+    logger.debug(`server: registered tool: ${name}`);
+  }
+
+  const elapsed = Date.now() - startTime;
+  logger.info(`${SERVER_NAME}: ready`, {
+    toolCount: Object.keys(allTools).length,
+    qdrant: vectorStore !== null,
+    falkordb: graphStore !== null,
+    startupMs: elapsed,
+  });
+
+  // -------------------------------------------------------------------------
+  // 9. Periodic confidence decay — runs every 24 hours
+  // -------------------------------------------------------------------------
+  // Knowledge items that haven't been accessed recently see their confidence
+  // decrease over time. This prevents stale knowledge from dominating search
+  // results. The decay is best-effort and never blocks the server.
+  const DECAY_INTERVAL = 24 * 60 * 60 * 1000; // 24 hours
+  setInterval(() => {
+    if (vectorStore) {
+      confidenceManager.decayAll(vectorStore as QdrantVectorStore)
+        .catch(err => logger.warn('Periodic confidence decay failed', { error: String(err) }));
+    }
+  }, DECAY_INTERVAL);
+
+  // Expose close method for graceful shutdown
+  (server as McpServer & { _closeConnections?: () => Promise<void> })._closeConnections = async () => {
+    db.close();
+    if (graphStore) {
+      try { await graphStore.disconnect(); } catch {}
+    }
+    logger.info('server: connections closed');
+  };
+
+  return server;
+}
+
+// ---------------------------------------------------------------------------
+// Degraded KnowledgeSearch stub — used when Qdrant is unavailable
+// ---------------------------------------------------------------------------
+
+function createDegradedKnowledgeSearch(): KnowledgeSearch {
+  // We need a QdrantVectorStore that always reports unhealthy.
+  // This causes KnowledgeSearch to return empty results gracefully.
+  const degradedStore = {
+    isHealthy: async () => false,
+    searchKnowledge: async () => [],
+    upsertKnowledge: async () => {},
+    deleteKnowledge: async () => {},
+    getKnowledgeCount: async () => 0,
+    searchCodeChunks: async () => [],
+    searchObservations: async () => [],
+    upsertCodeChunks: async () => {},
+    upsertObservation: async () => {},
+    deleteFileChunks: async () => {},
+    deleteRepoChunks: async () => {},
+    getCounts: async () => ({ codeChunks: 0, observations: 0 }),
+    getObservation: async () => null,
+    updateObservation: async () => {},
+    filterObservations: async () => [],
+    ensureCollections: async () => {},
+    connect: async () => {},
+    disconnect: async () => {},
+  } as unknown as QdrantVectorStore;
+
+  return new KnowledgeSearch(degradedStore);
+}