forge-server 0.1.0

package/src/knowledge/store.ts

@@ -0,0 +1,262 @@
+// KnowledgeYamlStore — reads and writes .forge/knowledge/*.yaml files.
+//
+// Each file is a YAML array of KnowledgeItem objects. The store maps
+// KnowledgeCategory values to dedicated files so items can be retrieved
+// by category without loading the full corpus.
+//
+// Design notes:
+// - File I/O is synchronous. The MCP server is single-threaded; sync I/O
+//   keeps the surface area small and the error handling straightforward.
+// - The store never caches in memory — every read hits the filesystem.
+//   This ensures multi-process safety (e.g. CLI + server running together)
+//   and keeps memory pressure low for a local dev tool.
+// - getContentHash() enables cheap change detection by the knowledge-keeper
+//   agent without re-parsing every item.
+
+import { parse, stringify } from 'yaml';
+import {
+  readFileSync,
+  writeFileSync,
+  existsSync,
+  mkdirSync,
+  statSync,
+} from 'node:fs';
+import { join } from 'node:path';
+import { createHash } from 'node:crypto';
+import type { KnowledgeCategory, KnowledgeItem } from '../util/types.js';
+
+// Re-export so callers can import KnowledgeCategory from this module
+// without having to reach into util/types directly.
+export type { KnowledgeCategory, KnowledgeItem } from '../util/types.js';
+
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+
+const CATEGORY_FILES: Record<KnowledgeCategory, string> = {
+  gotcha: 'gotchas.yaml',
+  pattern: 'patterns.yaml',
+  decision: 'decisions.yaml',
+  convention: 'conventions.yaml',
+};
+
+const ALL_CATEGORIES: KnowledgeCategory[] = ['gotcha', 'pattern', 'decision', 'convention'];
+
+// ---------------------------------------------------------------------------
+// KnowledgeYamlStore
+// ---------------------------------------------------------------------------
+
+export class KnowledgeYamlStore {
+  private readonly knowledgeDir: string;
+
+  constructor(private readonly forgeDir: string) {
+    this.knowledgeDir = join(forgeDir, 'knowledge');
+  }
+
+  // ---------------------------------------------------------------------------
+  // Public API
+  // ---------------------------------------------------------------------------
+
+  /**
+   * Read all items from a single category file.
+   * Returns an empty array if the file does not exist or is empty.
+   */
+  readCategory(category: KnowledgeCategory): KnowledgeItem[] {
+    const filePath = this._filePath(category);
+    return this._readFile(filePath);
+  }
+
+  /**
+   * Read all items across every category file, concatenated into one array.
+   * Items from each category are returned in file order, categories in the
+   * order defined by ALL_CATEGORIES.
+   */
+  readAll(): KnowledgeItem[] {
+    const result: KnowledgeItem[] = [];
+    for (const category of ALL_CATEGORIES) {
+      result.push(...this.readCategory(category));
+    }
+    return result;
+  }
+
+  /**
+   * Overwrite a category file with the supplied items array.
+   * Creates the knowledge directory if it does not exist.
+   * Passing an empty array produces a file containing `[]`.
+   */
+  writeCategory(category: KnowledgeCategory, items: KnowledgeItem[]): void {
+    this._ensureDir();
+    const filePath = this._filePath(category);
+    const yaml = stringify(items, {
+      indent: 2,
+      // Use block scalars for long content strings so diffs are readable.
+      defaultStringType: 'PLAIN',
+    });
+    writeFileSync(filePath, yaml, 'utf8');
+  }
+
+  /**
+   * Append a single item to the appropriate category file.
+   * The category is derived from the item's id prefix:
+   *   gotcha-*    → gotcha
+   *   pattern-*   → pattern
+   *   decision-*  → decision
+   *   convention-* → convention
+   *
+   * If the item does not match any prefix, it falls through to the pattern
+   * file as a safe default.
+   */
+  addItem(item: KnowledgeItem): void {
+    const category = this._categoryFromItem(item);
+    const existing = this.readCategory(category);
+
+    // Deduplicate by id — if an item with this id already exists, replace it.
+    const idx = existing.findIndex((i) => i.id === item.id);
+    if (idx >= 0) {
+      existing[idx] = item;
+    } else {
+      existing.push(item);
+    }
+
+    this.writeCategory(category, existing);
+  }
+
+  /**
+   * Update an existing item by id across all category files.
+   * Applies the supplied partial updates to the first matching item.
+   * Returns true if an item was found and updated, false otherwise.
+   */
+  updateItem(id: string, updates: Partial<KnowledgeItem>): boolean {
+    for (const category of ALL_CATEGORIES) {
+      const items = this.readCategory(category);
+      const idx = items.findIndex((i) => i.id === id);
+      if (idx >= 0) {
+        items[idx] = {
+          ...items[idx]!,
+          ...updates,
+          // Always bump updated_at when the caller hasn't supplied it.
+          updated_at: updates.updated_at ?? Date.now(),
+        };
+        this.writeCategory(category, items);
+        return true;
+      }
+    }
+    return false;
+  }
+
+  /**
+   * Remove an item by id across all category files.
+   * Returns true if an item was found and removed, false otherwise.
+   */
+  removeItem(id: string): boolean {
+    for (const category of ALL_CATEGORIES) {
+      const items = this.readCategory(category);
+      const filtered = items.filter((i) => i.id !== id);
+      if (filtered.length < items.length) {
+        this.writeCategory(category, filtered);
+        return true;
+      }
+    }
+    return false;
+  }
+
+  /**
+   * Create the .forge/knowledge/ directory and initialise empty YAML files
+   * for every category. Existing files are left untouched — initialize() is
+   * safe to call multiple times.
+   */
+  initialize(): void {
+    this._ensureDir();
+    for (const category of ALL_CATEGORIES) {
+      const filePath = this._filePath(category);
+      if (!existsSync(filePath)) {
+        // Write an empty YAML array so the file is valid YAML from the start.
+        writeFileSync(filePath, '[]\n', 'utf8');
+      }
+    }
+  }
+
+  /**
+   * Compute a SHA-256 hash over the concatenated contents of all category
+   * files. Files that do not exist contribute an empty string to the hash.
+   * The hash changes whenever any item is added, updated, or removed, making
+   * it useful for cheap change detection without re-parsing every item.
+   */
+  getContentHash(): string {
+    const hash = createHash('sha256');
+    for (const category of ALL_CATEGORIES) {
+      const filePath = this._filePath(category);
+      if (existsSync(filePath)) {
+        const content = readFileSync(filePath, 'utf8');
+        hash.update(content);
+      } else {
+        hash.update('');
+      }
+    }
+    return hash.digest('hex');
+  }
+
+  // ---------------------------------------------------------------------------
+  // Private helpers
+  // ---------------------------------------------------------------------------
+
+  /** Absolute path to the YAML file for a given category. */
+  private _filePath(category: KnowledgeCategory): string {
+    return join(this.knowledgeDir, CATEGORY_FILES[category]);
+  }
+
+  /** Ensure the knowledge directory exists. */
+  private _ensureDir(): void {
+    if (!existsSync(this.knowledgeDir)) {
+      mkdirSync(this.knowledgeDir, { recursive: true });
+    }
+  }
+
+  /**
+   * Parse a YAML file and return its items.
+   * Returns [] when the file is missing, empty, or contains a YAML null
+   * (which is what an empty file or `---` alone produces).
+   */
+  private _readFile(filePath: string): KnowledgeItem[] {
+    if (!existsSync(filePath)) {
+      return [];
+    }
+
+    // Guard against zero-byte files — parse('') returns null in the yaml
+    // package, not an empty array.
+    const stat = statSync(filePath);
+    if (stat.size === 0) {
+      return [];
+    }
+
+    const content = readFileSync(filePath, 'utf8');
+    const parsed: unknown = parse(content);
+
+    if (parsed === null || parsed === undefined) {
+      return [];
+    }
+
+    if (!Array.isArray(parsed)) {
+      throw new Error(
+        `KnowledgeYamlStore: expected array at ${filePath}, got ${typeof parsed}`,
+      );
+    }
+
+    // Cast: the YAML is authoritative. Callers writing items go through our
+    // typed addItem/writeCategory methods so the shape is always correct.
+    return parsed as KnowledgeItem[];
+  }
+
+  /**
+   * Derive the KnowledgeCategory for a given item based on its id prefix.
+   * Falls back to 'pattern' when the id does not match a known prefix.
+   */
+  private _categoryFromItem(item: KnowledgeItem): KnowledgeCategory {
+    const id = item.id.toLowerCase();
+    if (id.startsWith('gotcha')) return 'gotcha';
+    if (id.startsWith('pattern')) return 'pattern';
+    if (id.startsWith('decision')) return 'decision';
+    if (id.startsWith('convention')) return 'convention';
+    return 'pattern';
+  }
+}

package/src/learning/confidence.ts

@@ -0,0 +1,193 @@
+// ConfidenceManager — B12
+//
+// Manages confidence score lifecycle for knowledge items stored in Qdrant.
+//
+// Confidence represents how well-validated a knowledge item is:
+//   0.0  – pure speculation, has never proven useful
+//   0.5  – moderate confidence, has been seen in a couple of contexts
+//   1.0  – very high confidence, validated across many projects
+//
+// Two operations:
+//
+//   decayAll(vectorStore, decayRate)
+//     Applies time-based confidence decay to all items that have not been
+//     accessed recently. Items accessed within the last 30 days are not
+//     decayed. The default decay rate is 0.01 per run (about 1 percentage
+//     point). Items whose confidence would fall below MIN_CONFIDENCE (0.1)
+//     are floored there rather than deleted — deletion is a policy decision
+//     for operators, not an automatic side effect of decay.
+//
+//   boost(vectorStore, knowledgeId, amount)
+//     Increases the confidence of a specific item by `amount` (default 0.05).
+//     Also bumps accessed_at and access_count. Capped at 1.0.
+//
+// Both methods are async and fail silently on Qdrant errors — confidence
+// management is best-effort. It must never block the pipeline.
+
+import type { QdrantVectorStore } from '../storage/qdrant-store.js';
+import type { KnowledgePayload } from '../util/types.js';
+import { logger } from '../util/logger.js';
+
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+
+/** Items accessed within this window (ms) are not decayed. */
+const DECAY_GRACE_PERIOD_MS = 30 * 24 * 60 * 60 * 1000; // 30 days
+
+/** Items whose confidence falls at or below this value are not decayed further. */
+const MIN_CONFIDENCE = 0.1;
+
+/** Maximum confidence a single boost can reach. */
+const MAX_CONFIDENCE = 1.0;
+
+/** How many knowledge items to scroll per Qdrant page during decay. */
+const SCROLL_PAGE_SIZE = 100;
+
+// ---------------------------------------------------------------------------
+// ConfidenceManager
+// ---------------------------------------------------------------------------
+
+export class ConfidenceManager {
+  /**
+   * Apply time-based confidence decay to all knowledge items in Qdrant.
+   *
+   * Algorithm:
+   * 1. Scroll through all knowledge items in pages of SCROLL_PAGE_SIZE.
+   * 2. For each item whose accessed_at is older than DECAY_GRACE_PERIOD_MS:
+   *    a. Compute new_confidence = max(MIN_CONFIDENCE, confidence - decayRate).
+   *    b. If new_confidence != confidence, call setPayload to update.
+   * 3. Return the total count of items whose confidence was updated.
+   *
+   * This method is intentionally slow (one HTTP call per decayed item) because
+   * it is designed to run as a background task, not in the hot path. A batch
+   * update path can be optimised once the volume of knowledge items justifies it.
+   */
+  async decayAll(
+    vectorStore: QdrantVectorStore,
+    decayRate: number = 0.01,
+  ): Promise<number> {
+    const healthy = await vectorStore.isHealthy();
+    if (!healthy) {
+      logger.warn('ConfidenceManager.decayAll: Qdrant unavailable, skipping decay');
+      return 0;
+    }
+
+    const now = Date.now();
+    const graceCutoff = now - DECAY_GRACE_PERIOD_MS;
+    let decayedCount = 0;
+
+    // Use filterKnowledge (scroll with empty filter) to paginate through all
+    // knowledge items. setKnowledgePayload is used to patch confidence values.
+    // Both methods are real exports on QdrantVectorStore.
+
+    let page: Array<{ id: string; payload: Record<string, unknown> }>;
+    try {
+      page = await vectorStore.filterKnowledge({}, SCROLL_PAGE_SIZE);
+    } catch (err) {
+      logger.warn('ConfidenceManager.decayAll: initial scroll failed, stopping', {
+        error: String(err),
+      });
+      return 0;
+    }
+
+    for (const point of page) {
+      const p = point.payload as Partial<KnowledgePayload>;
+      const accessedAt = p.accessed_at ?? 0;
+      const confidence = p.confidence ?? 0;
+
+      // Skip recently accessed items
+      if (accessedAt >= graceCutoff) continue;
+
+      // Skip items already at floor
+      if (confidence <= MIN_CONFIDENCE) continue;
+
+      const newConfidence = Math.max(MIN_CONFIDENCE, confidence - decayRate);
+
+      // Round to 4 decimal places to avoid floating-point drift
+      const rounded = Math.round(newConfidence * 10000) / 10000;
+      if (rounded === confidence) continue; // No change needed
+
+      try {
+        await vectorStore.setKnowledgePayload(point.id, { confidence: rounded });
+        decayedCount++;
+
+        logger.debug('ConfidenceManager.decayAll: item decayed', {
+          id: point.id,
+          oldConfidence: confidence,
+          newConfidence: rounded,
+        });
+      } catch (err) {
+        // Non-fatal — log and continue to next item
+        logger.debug('ConfidenceManager.decayAll: failed to update item', {
+          id: point.id,
+          error: String(err),
+        });
+      }
+    }
+
+    logger.info('ConfidenceManager.decayAll: complete', {
+      decayedCount,
+      decayRate,
+    });
+
+    return decayedCount;
+  }
+
+  /**
+   * Boost the confidence of a specific knowledge item.
+   *
+   * Updates:
+   *   confidence  += amount (capped at MAX_CONFIDENCE)
+   *   accessed_at  = now
+   *   access_count += 1
+   *
+   * Fails silently on Qdrant errors — boosting is a best-effort operation.
+   */
+  async boost(
+    vectorStore: QdrantVectorStore,
+    knowledgeId: string,
+    amount: number = 0.05,
+  ): Promise<void> {
+    const healthy = await vectorStore.isHealthy();
+    if (!healthy) {
+      logger.debug('ConfidenceManager.boost: Qdrant unavailable, skip', { knowledgeId });
+      return;
+    }
+
+    try {
+      let currentConfidence = 0.5;
+      let currentAccessCount = 0;
+
+      const existing = await vectorStore.getKnowledge(knowledgeId);
+      if (!existing) {
+        logger.debug('ConfidenceManager.boost: item not found', { knowledgeId });
+        return;
+      }
+      const payload = existing as Partial<KnowledgePayload>;
+      currentConfidence = (typeof payload.confidence === 'number' ? payload.confidence : null) ?? 0.5;
+      currentAccessCount = (typeof payload.access_count === 'number' ? payload.access_count : null) ?? 0;
+
+      const newConfidence = Math.min(MAX_CONFIDENCE, currentConfidence + amount);
+      const rounded = Math.round(newConfidence * 10000) / 10000;
+
+      await vectorStore.setKnowledgePayload(knowledgeId, {
+        confidence: rounded,
+        accessed_at: Date.now(),
+        access_count: currentAccessCount + 1,
+      });
+
+      logger.debug('ConfidenceManager.boost: boosted', {
+        knowledgeId,
+        oldConfidence: currentConfidence,
+        newConfidence: rounded,
+        newAccessCount: currentAccessCount + 1,
+      });
+    } catch (err) {
+      logger.warn('ConfidenceManager.boost: failed (non-fatal)', {
+        knowledgeId,
+        error: String(err),
+      });
+    }
+  }
+}