forge-server 0.1.0

package/src/context/memory.ts

@@ -0,0 +1,187 @@
+// MemoryBridge — B8
+//
+// Wraps the Qdrant vector store's observation collection to provide
+// memory search and save operations for the context injection layer.
+//
+// Uses the QdrantVectorStore directly (not the VectorStore interface) because
+// we need the searchObservations + upsertObservation methods that are Qdrant-specific.
+// When vectorStore is null or unhealthy, all methods degrade gracefully.
+
+import { randomUUID } from 'node:crypto';
+import type { QdrantVectorStore } from '../storage/qdrant-store.js';
+import type { ObservationPayload } from '../util/types.js';
+import { embedText } from '../ingestion/embedder.js';
+import { logger } from '../util/logger.js';
+
+// ---------------------------------------------------------------------------
+// Public types
+// ---------------------------------------------------------------------------
+
+export interface MemorySearchResult {
+  content: string;
+  createdAt: number;
+  relevanceScore: number;
+  sessionId?: string;
+}
+
+export interface MemorySearchOptions {
+  repoId?: string;
+  sessionId?: string;
+  limit?: number;
+  scoreThreshold?: number;
+}
+
+// ---------------------------------------------------------------------------
+// MemoryBridge
+// ---------------------------------------------------------------------------
+
+export class MemoryBridge {
+  constructor(private readonly vectorStore: QdrantVectorStore | null) {}
+
+  /**
+   * Search observations by semantic similarity.
+   * Optionally filters by repoId and/or sessionId.
+   * Returns [] gracefully when Qdrant is unavailable.
+   */
+  async searchMemories(
+    query: string,
+    options: MemorySearchOptions = {},
+  ): Promise<MemorySearchResult[]> {
+    if (!this.vectorStore) {
+      return [];
+    }
+
+    const healthy = await this.vectorStore.isHealthy();
+    if (!healthy) {
+      logger.debug('MemoryBridge: Qdrant unavailable — returning empty memories');
+      return [];
+    }
+
+    const {
+      repoId,
+      sessionId,
+      limit = 10,
+      scoreThreshold = 0.3,
+    } = options;
+
+    let queryVector: number[];
+    try {
+      queryVector = await embedText(query);
+    } catch (err) {
+      logger.warn('MemoryBridge: embedding failed', { error: String(err) });
+      return [];
+    }
+
+    // Build filter — only include conditions for fields that are specified
+    const mustConditions: Record<string, unknown>[] = [];
+    if (repoId) {
+      mustConditions.push({ key: 'repo_id', match: { value: repoId } });
+    }
+    if (sessionId) {
+      mustConditions.push({ key: 'session_id', match: { value: sessionId } });
+    }
+
+    // Exclude session markers from memory results
+    const mustNotConditions: Record<string, unknown>[] = [
+      { key: 'tags', match: { value: 'session_marker' } },
+    ];
+
+    const filter: Record<string, unknown> =
+      mustConditions.length > 0
+        ? { must: mustConditions, must_not: mustNotConditions }
+        : { must_not: mustNotConditions };
+
+    try {
+      const results = await this.vectorStore.searchObservations(queryVector, {
+        limit,
+        scoreThreshold,
+        filter,
+      });
+
+      return results
+        .map((r) => {
+          const payload = r.payload as ObservationPayload;
+          return {
+            content: payload.content,
+            createdAt: payload.created_at,
+            relevanceScore: Math.round(r.score * 1000) / 1000,
+            sessionId: payload.session_id || undefined,
+          };
+        })
+        .filter((m) => m.content && m.content.trim().length > 0);
+    } catch (err) {
+      logger.warn('MemoryBridge: search failed', { error: String(err) });
+      return [];
+    }
+  }
+
+  /**
+   * Save a memory (observation) to the vector store.
+   * Returns the UUID of the stored observation.
+   * Throws when storage fails so the caller can decide whether to propagate.
+   */
+  async saveMemory(
+    content: string,
+    repoId: string,
+    sessionId?: string,
+  ): Promise<string> {
+    if (!this.vectorStore) {
+      throw new Error('MemoryBridge: vector store not configured');
+    }
+
+    const healthy = await this.vectorStore.isHealthy();
+    if (!healthy) {
+      throw new Error('MemoryBridge: Qdrant is unavailable');
+    }
+
+    const id = randomUUID();
+    const now = Date.now();
+    const effectiveSessionId = sessionId ?? `session-${now}`;
+
+    let vector: number[];
+    try {
+      vector = await embedText(content);
+    } catch (err) {
+      logger.warn('MemoryBridge: embedding failed, using zero vector', {
+        error: String(err),
+      });
+      vector = new Array(384).fill(0);
+    }
+
+    const payload: ObservationPayload = {
+      id,
+      repo_id: repoId,
+      session_id: effectiveSessionId,
+      content,
+      type: 'semantic',
+      category: null,
+      tags: [],
+      importance: 0.5,
+      created_at: now,
+      accessed_at: now,
+      access_count: 0,
+      is_stale: false,
+      linked_symbols: [],
+      source: 'forge',
+      metadata: {},
+      key: null,
+      namespace: 'default',
+    };
+
+    await this.vectorStore.upsertObservation(id, vector, payload);
+
+    logger.debug('MemoryBridge: memory saved', {
+      id,
+      repoId,
+      sessionId: effectiveSessionId,
+      contentLength: content.length,
+    });
+
+    return id;
+  }
+
+  /** True when the vector store is configured (does not perform a health check). */
+  isAvailable(): boolean {
+    return this.vectorStore !== null;
+  }
+}

package/src/context/session-index.ts

@@ -0,0 +1,327 @@
+// session-index.ts
+//
+// Session-scoped context index — virtual memory for LLM context windows.
+// Instead of pruning old context (losing it), offloads to Qdrant for
+// semantic retrieval. The injector searches this before assembling
+// context, retrieving only what's relevant to the current task.
+//
+// Uses existing forge_observations collection with type='working' to
+// avoid creating new infrastructure.
+
+import type { QdrantVectorStore } from '../storage/qdrant-store.js';
+import type { GraphStore } from '../storage/interfaces.js';
+import { saveObservation } from '../memory/observation-store.js';
+import { embedText } from '../ingestion/embedder.js';
+import { logger } from '../util/logger.js';
+
+// No-op graph store for saveObservation calls (graph linking not needed for cache)
+const noOpGraph: GraphStore = {
+  connect: async () => {},
+  disconnect: async () => {},
+  isHealthy: async () => false,
+  query: async () => ({ nodes: [], edges: [] }),
+  upsertNode: async () => {},
+  upsertEdge: async () => {},
+  deleteFile: async () => {},
+  deleteRepo: async () => {},
+  getCounts: async () => ({ totalNodes: 0, totalEdges: 0, byLabel: {} }),
+  ensureIndexes: async () => {},
+};
+
+// Phase names for tag extraction
+const KNOWN_PHASES = [
+  'interview',
+  'requirements',
+  'architecture',
+  'design',
+  'qa_strategy',
+  'implementation',
+  'inspection',
+  'knowledge_collection',
+] as const;
+
+// Event types for tag extraction
+const KNOWN_EVENT_TYPES = [
+  'phase_output',
+  'broadcast',
+  'tool_result',
+  'agent_summary',
+] as const;
+
+export interface SessionEvent {
+  type: 'phase_output' | 'broadcast' | 'tool_result' | 'agent_summary';
+  phase: string;
+  agent: string;
+  projectId: string;
+  content: string;
+  created_at: number;
+}
+
+export interface TieredContext {
+  /** Tier 0: critical directives, contracts, blockers — always injected */
+  alwaysInject: string[];
+  /** Tier 1: top-K semantic matches for this module — smart-injected */
+  smartInjected: string[];
+  /** Tier 2: count of remaining searchable items (for the agent's awareness) */
+  searchableCount: number;
+}
+
+export class SessionContextIndex {
+  constructor(
+    private readonly vectorStore: QdrantVectorStore,
+    private readonly sessionId: string,
+  ) {}
+
+  /**
+   * Index a session event for later retrieval.
+   * Fire-and-forget — never blocks the caller.
+   */
+  async index(event: SessionEvent): Promise<void> {
+    try {
+      await saveObservation(
+        {
+          content: event.content.slice(0, 2000),
+          sessionId: this.sessionId,
+          type: 'working',
+          category: `session_${event.type}`,
+          importance: event.type === 'broadcast' ? 0.7 : 0.5,
+          tags: [
+            'session_cache',
+            this.sessionId,
+            event.projectId,
+            event.phase,
+            event.agent,
+            event.type,
+          ],
+        },
+        this.vectorStore,
+        noOpGraph,
+      );
+    } catch (err) {
+      logger.debug('SessionContextIndex.index: failed (non-fatal)', {
+        type: event.type,
+        error: String(err),
+      });
+    }
+  }
+
+  /**
+   * Semantic search scoped to this session + project.
+   * Returns the most relevant cached items for a query.
+   */
+  async retrieve(
+    query: string,
+    projectId: string,
+    limit: number = 5,
+  ): Promise<SessionEvent[]> {
+    try {
+      // Attempt semantic search first — requires embedding the query
+      let queryVector: number[] | null = null;
+      try {
+        queryVector = await embedText(query);
+      } catch {
+        // Embedding unavailable — fall back to filter-only below
+      }
+
+      if (queryVector) {
+        // Semantic search: ranked by vector similarity
+        const results = await this.vectorStore.searchObservations(queryVector, {
+          limit,
+          scoreThreshold: 0.2,
+          filter: {
+            must: [
+              { key: 'tags', match: { value: 'session_cache' } },
+              { key: 'tags', match: { value: this.sessionId } },
+              { key: 'tags', match: { value: projectId } },
+            ],
+          },
+        });
+        return results.map((r) => this.resultToSessionEvent(r.payload as unknown as Record<string, unknown>, projectId));
+      }
+
+      // Fallback: filter-only retrieval (no semantic ranking)
+      const filter = {
+        must: [
+          { key: 'tags', match: { value: 'session_cache' } },
+          { key: 'tags', match: { value: this.sessionId } },
+          { key: 'tags', match: { value: projectId } },
+        ],
+      };
+      const results = await this.vectorStore.filterObservations(filter, limit);
+      return results.map((r) => this.resultToSessionEvent(r.payload as unknown as Record<string, unknown>, projectId));
+    } catch (err) {
+      logger.debug('SessionContextIndex.retrieve: failed (non-fatal)', { error: String(err) });
+      return [];
+    }
+  }
+
+  /**
+   * Build tiered context for a module using semantic search.
+   *
+   * Tier 0: Items tagged as critical/directive (importance >= 0.8) — always injected
+   * Tier 1: Top-K semantic matches for the module description — smart-injected
+   * Tier 2: Count of remaining items (awareness only)
+   */
+  async buildTieredContext(
+    moduleDescription: string,
+    projectId: string,
+  ): Promise<TieredContext> {
+    const defaultCtx: TieredContext = {
+      alwaysInject: [],
+      smartInjected: [],
+      searchableCount: 0,
+    };
+
+    try {
+      // Tier 0: All session cache items for this project (we filter high-importance in memory)
+      const allFilter = {
+        must: [
+          { key: 'tags', match: { value: 'session_cache' } },
+          { key: 'tags', match: { value: this.sessionId } },
+          { key: 'tags', match: { value: projectId } },
+        ],
+      };
+      const allResults = await this.vectorStore.filterObservations(allFilter, 50);
+
+      // Extract high-importance items for Tier 0
+      const criticals = allResults.filter((r) => {
+        const payload = r.payload as unknown as Record<string, unknown>;
+        return (payload['importance'] as number) >= 0.8;
+      });
+      defaultCtx.alwaysInject = criticals.map((r) => {
+        const payload = r.payload as unknown as Record<string, unknown>;
+        return (payload['content'] as string) ?? '';
+      });
+
+      // Tier 1: Semantic search for this module's description
+      // Requires embedding the module description into a query vector
+      let smartResults: string[] = [];
+      if (moduleDescription) {
+        let queryVector: number[] | null = null;
+        try {
+          queryVector = await embedText(moduleDescription);
+        } catch {
+          // Embedding unavailable — skip semantic search
+        }
+
+        if (queryVector) {
+          const searchResults = await this.vectorStore.searchObservations(
+            queryVector,
+            {
+              limit: 5,
+              scoreThreshold: 0.3,
+              filter: {
+                must: [
+                  { key: 'tags', match: { value: 'session_cache' } },
+                  { key: 'tags', match: { value: this.sessionId } },
+                  { key: 'tags', match: { value: projectId } },
+                ],
+              },
+            },
+          );
+          smartResults = searchResults.map((r) => {
+            const payload = r.payload as unknown as Record<string, unknown>;
+            return (payload['content'] as string) ?? '';
+          });
+        } else {
+          // Fallback: use the non-critical filter results as Tier 1
+          const nonCriticalItems = allResults.filter((r) => {
+            const payload = r.payload as unknown as Record<string, unknown>;
+            return (payload['importance'] as number) < 0.8;
+          });
+          smartResults = nonCriticalItems.slice(0, 5).map((r) => {
+            const payload = r.payload as unknown as Record<string, unknown>;
+            return (payload['content'] as string) ?? '';
+          });
+        }
+      }
+      defaultCtx.smartInjected = smartResults;
+
+      // Tier 2: Total count minus what we already injected
+      defaultCtx.searchableCount = Math.max(
+        0,
+        allResults.length - defaultCtx.alwaysInject.length - defaultCtx.smartInjected.length,
+      );
+
+      return defaultCtx;
+    } catch (err) {
+      logger.debug('SessionContextIndex.buildTieredContext: failed (non-fatal)', { error: String(err) });
+      return defaultCtx;
+    }
+  }
+
+  /**
+   * End-of-session cleanup: promote high-value items to permanent
+   * observations, let the rest expire naturally.
+   */
+  async promoteAndExpire(): Promise<{ promoted: number; expired: number }> {
+    try {
+      const filter = {
+        must: [
+          { key: 'tags', match: { value: 'session_cache' } },
+          { key: 'tags', match: { value: this.sessionId } },
+        ],
+      };
+      const all = await this.vectorStore.filterObservations(filter, 100);
+
+      let promoted = 0;
+      for (const item of all) {
+        const payload = item.payload as unknown as Record<string, unknown>;
+        const importance = (payload['importance'] as number) ?? 0;
+        // Promote items with high importance to permanent episodic memory
+        if (importance >= 0.7) {
+          await saveObservation(
+            {
+              content: (payload['content'] as string) ?? '',
+              sessionId: this.sessionId,
+              type: 'episodic',
+              category: (payload['category'] as string) ?? 'session_promoted',
+              importance,
+              tags: ((payload['tags'] as string[]) ?? []).filter(t => t !== 'session_cache'),
+            },
+            this.vectorStore,
+            noOpGraph,
+          );
+          promoted++;
+        }
+      }
+
+      return { promoted, expired: all.length - promoted };
+    } catch (err) {
+      logger.debug('SessionContextIndex.promoteAndExpire: failed (non-fatal)', { error: String(err) });
+      return { promoted: 0, expired: 0 };
+    }
+  }
+
+  /**
+   * Convert a raw Qdrant payload into a SessionEvent.
+   * Extracts type, phase, and agent from the tags array.
+   */
+  private resultToSessionEvent(
+    payload: Record<string, unknown>,
+    projectId: string,
+  ): SessionEvent {
+    const tags = (payload['tags'] as string[]) ?? [];
+
+    const eventType = tags.find((t): t is SessionEvent['type'] =>
+      (KNOWN_EVENT_TYPES as readonly string[]).includes(t),
+    ) ?? 'phase_output';
+
+    const phase = tags.find((t) =>
+      (KNOWN_PHASES as readonly string[]).includes(t),
+    ) ?? 'unknown';
+
+    // Agent is stored at index 4 in the tags array:
+    // ['session_cache', sessionId, projectId, phase, agent, type]
+    const agent = tags[4] ?? 'unknown';
+
+    return {
+      type: eventType,
+      phase,
+      agent,
+      projectId,
+      content: (payload['content'] as string) ?? '',
+      created_at: (payload['created_at'] as number) ?? 0,
+    };
+  }
+}

package/src/context/session.ts

@@ -0,0 +1,152 @@
+// SessionManager — B9
+//
+// Manages session lifecycle against the SQLite sessions table.
+// Sessions are lightweight records that tie memory observations and
+// phase transitions to a chronological window so the context injector can
+// fetch "what was learned in this session" when assembling phase context.
+//
+// Design:
+//   - All operations are synchronous SQLite reads/writes (better-sqlite3).
+//   - Session IDs follow the format "session-{timestamp}-{random4}".
+//   - State is stored as a JSON blob in the `state` column; callers can
+//     stash arbitrary resumption data there.
+//   - observation_count is incremented via a single UPDATE statement so
+//     concurrent callers in the same process do not race.
+
+import { randomUUID } from 'node:crypto';
+import type { PipelineDB } from '../storage/sqlite.js';
+import type { Session, SessionRow, SessionState } from '../util/types.js';
+import { logger } from '../util/logger.js';
+
+// ---------------------------------------------------------------------------
+// Row deserialization helper
+// ---------------------------------------------------------------------------
+
+function rowToSession(row: SessionRow): Session {
+  let state: SessionState = {};
+  try {
+    state = JSON.parse(row.state) as SessionState;
+  } catch {
+    state = {};
+  }
+
+  return {
+    id: row.id,
+    projectId: row.project_id ?? null,
+    repoId: row.repo_id ?? null,
+    startedAt: row.started_at,
+    endedAt: row.ended_at ?? null,
+    state,
+    observationCount: row.observation_count,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// SessionManager
+// ---------------------------------------------------------------------------
+
+export class SessionManager {
+  constructor(private readonly db: PipelineDB) {}
+
+  /**
+   * Create and persist a new session.
+   * Returns the new session ID.
+   */
+  startSession(repoId?: string, projectId?: string): string {
+    const id = `session-${Date.now()}-${randomUUID().slice(0, 4)}`;
+    const now = Date.now();
+
+    this.db.run(
+      `INSERT INTO sessions (id, project_id, repo_id, started_at, ended_at, state, observation_count)
+       VALUES (?, ?, ?, ?, NULL, '{}', 0)`,
+      [id, projectId ?? null, repoId ?? null, now],
+    );
+
+    logger.debug('SessionManager: session started', { id, repoId, projectId });
+    return id;
+  }
+
+  /**
+   * Mark a session as ended. Idempotent — silently succeeds if already ended.
+   */
+  endSession(sessionId: string): void {
+    this.db.run(
+      `UPDATE sessions SET ended_at = ? WHERE id = ? AND ended_at IS NULL`,
+      [Date.now(), sessionId],
+    );
+    logger.debug('SessionManager: session ended', { sessionId });
+  }
+
+  /**
+   * Retrieve a session by ID.
+   * Returns null if no session with that ID exists.
+   */
+  getSession(sessionId: string): Session | null {
+    const row = this.db.get<SessionRow>(
+      `SELECT * FROM sessions WHERE id = ?`,
+      [sessionId],
+    );
+    if (!row) return null;
+    return rowToSession(row);
+  }
+
+  /**
+   * List sessions, optionally filtered by repoId.
+   * Results are ordered most-recent first.
+   */
+  listSessions(repoId?: string): Session[] {
+    let rows: SessionRow[];
+    if (repoId) {
+      rows = this.db.all<SessionRow>(
+        `SELECT * FROM sessions WHERE repo_id = ? ORDER BY started_at DESC`,
+        [repoId],
+      );
+    } else {
+      rows = this.db.all<SessionRow>(
+        `SELECT * FROM sessions ORDER BY started_at DESC`,
+      );
+    }
+    return rows.map(rowToSession);
+  }
+
+  /**
+   * Overwrite the JSON state blob for a session.
+   * Callers use this to persist resumption context (current task, agent
+   * outputs, phase position) so the supervisor can restore a paused session.
+   */
+  saveState(sessionId: string, state: Record<string, unknown>): void {
+    this.db.run(
+      `UPDATE sessions SET state = ? WHERE id = ?`,
+      [JSON.stringify(state), sessionId],
+    );
+    logger.debug('SessionManager: state saved', { sessionId });
+  }
+
+  /**
+   * Read the JSON state blob for a session.
+   * Returns null if the session does not exist.
+   */
+  restoreState(sessionId: string): Record<string, unknown> | null {
+    const row = this.db.get<{ state: string }>(
+      `SELECT state FROM sessions WHERE id = ?`,
+      [sessionId],
+    );
+    if (!row) return null;
+    try {
+      return JSON.parse(row.state) as Record<string, unknown>;
+    } catch {
+      return {};
+    }
+  }
+
+  /**
+   * Increment the observation_count for a session by 1.
+   * Called by the observation save path each time a memory is stored.
+   */
+  incrementObservations(sessionId: string): void {
+    this.db.run(
+      `UPDATE sessions SET observation_count = observation_count + 1 WHERE id = ?`,
+      [sessionId],
+    );
+  }
+}