2ndbrain 2026.1.30 → 2026.1.32

package/src/embeddings/engine.js

@@ -0,0 +1,281 @@
+/**
+ * Embeddings engine -- manages pgvector-backed semantic search infrastructure.
+ *
+ * Handles startup configuration resolution per spec section 11.4:
+ *   1. Resolve dimensions from env var or model defaults
+ *   2. First-time setup: create extension, tables, index
+ *   3. Model switch: drop/recreate vector column, queue re-embedding
+ *   4. No change: skip
+ *
+ * Only creates pgvector tables when EMBEDDING_PROVIDER is set.
+ */
+
+/**
+ * Default vector dimensions for known OpenAI embedding models.
+ */
+const MODEL_DIMENSION_DEFAULTS = {
+  'text-embedding-3-small': 1536,
+  'text-embedding-3-large': 3072,
+  'text-embedding-ada-002': 1536,
+};
+
+class EmbeddingsEngine {
+  /**
+   * @param {object} deps
+   * @param {object} deps.db     - Database query interface ({ query(sql, params) }).
+   * @param {object} deps.config - Application configuration.
+   * @param {object} deps.logger - Logger instance.
+   */
+  constructor({ db, config, logger }) {
+    this.db = db;
+    this.config = config;
+    this.logger = logger;
+    this._dimensions = null;
+  }
+
+  /**
+   * Returns true when the embedding provider is configured.
+   *
+   * @returns {boolean}
+   */
+  isEnabled() {
+    return Boolean(this.config.EMBEDDING_PROVIDER);
+  }
+
+  /**
+   * Run startup configuration resolution.
+   *
+   * 1. Resolve dimensions: from EMBEDDING_DIMENSIONS env var, or model
+   *    defaults (text-embedding-3-small=1536, text-embedding-3-large=3072,
+   *    text-embedding-ada-002=1536).  Fails startup if the model is unknown
+   *    and no explicit dimension is provided.
+   * 2. First-time setup: CREATE EXTENSION IF NOT EXISTS vector, create
+   *    embedding_config and embeddings tables, create HNSW index, insert
+   *    config row.
+   * 3. Model switch: log warning, drop+recreate vector column with new
+   *    dimensions, recreate index, update config.  All existing rows become
+   *    NULL-vector and are re-embedded by the background worker.
+   * 4. No change: skip.
+   */
+  async initialize() {
+    if (!this.isEnabled()) {
+      this.logger.info('embeddings', 'Embedding provider not configured; embeddings disabled.');
+      return;
+    }
+
+    const provider = this.config.EMBEDDING_PROVIDER;
+    const model = this.config.EMBEDDING_MODEL || 'text-embedding-3-small';
+    const dimensions = this._resolveDimensions(model);
+    this._dimensions = dimensions;
+
+    this.logger.info(
+      'embeddings',
+      `Initializing embeddings: provider=${provider} model=${model} dimensions=${dimensions}`,
+    );
+
+    // Ensure the pgvector extension is available
+    await this.db.query('CREATE EXTENSION IF NOT EXISTS vector');
+
+    // Check whether the embedding_config table already exists
+    const tableCheck = await this.db.query(
+      `SELECT EXISTS (
+         SELECT FROM information_schema.tables
+         WHERE table_schema = 'public'
+           AND table_name   = 'embedding_config'
+       ) AS table_exists`,
+    );
+
+    if (!tableCheck.rows[0].table_exists) {
+      await this._firstTimeSetup(provider, model, dimensions);
+      return;
+    }
+
+    // Table exists -- check for an existing config row
+    const configRow = await this.db.query(
+      'SELECT provider, model, dimensions FROM embedding_config WHERE id = 1',
+    );
+
+    if (configRow.rows.length === 0) {
+      // Table present but empty -- treat as first-time setup
+      await this._firstTimeSetup(provider, model, dimensions);
+      return;
+    }
+
+    const current = configRow.rows[0];
+
+    if (
+      current.provider === provider &&
+      current.model === model &&
+      current.dimensions === dimensions
+    ) {
+      // Configuration unchanged
+      this.logger.info('embeddings', 'Embedding configuration unchanged.');
+      return;
+    }
+
+    // Configuration differs -- perform model switch
+    await this._handleModelSwitch(current, { provider, model, dimensions });
+  }
+
+  /**
+   * Queue an entity for background embedding generation.
+   * Inserts a row with a NULL vector; the background worker will fill it in.
+   *
+   * @param {string} entityType - Entity type (e.g. 'message', 'node', 'journal', 'issue').
+   * @param {number} entityId   - Primary key of the source entity.
+   */
+  async queueEmbedding(entityType, entityId) {
+    if (!this.isEnabled()) {
+      return;
+    }
+
+    await this.db.query(
+      `INSERT INTO embeddings (entity_type, entity_id)
+       VALUES ($1, $2)
+       ON CONFLICT (entity_type, entity_id) DO NOTHING`,
+      [entityType, entityId],
+    );
+  }
+
+  // ---------------------------------------------------------------------------
+  // Internal helpers
+  // ---------------------------------------------------------------------------
+
+  /**
+   * Resolve the target vector dimensions from the EMBEDDING_DIMENSIONS env var
+   * or the known model defaults.
+   *
+   * @param {string} model - Embedding model name.
+   * @returns {number} Resolved dimension count.
+   * @throws {Error} When dimensions cannot be determined.
+   */
+  _resolveDimensions(model) {
+    if (this.config.EMBEDDING_DIMENSIONS) {
+      const dim = parseInt(this.config.EMBEDDING_DIMENSIONS, 10);
+      if (Number.isNaN(dim) || dim <= 0) {
+        throw new Error(
+          `Invalid EMBEDDING_DIMENSIONS value: "${this.config.EMBEDDING_DIMENSIONS}"`,
+        );
+      }
+      return dim;
+    }
+
+    const defaultDim = MODEL_DIMENSION_DEFAULTS[model];
+    if (!defaultDim) {
+      throw new Error(
+        `Unknown embedding model "${model}" and EMBEDDING_DIMENSIONS is not set. ` +
+        `Set EMBEDDING_DIMENSIONS explicitly or use a known model: ` +
+        `${Object.keys(MODEL_DIMENSION_DEFAULTS).join(', ')}`,
+      );
+    }
+
+    return defaultDim;
+  }
+
+  /**
+   * First-time setup: create the embedding_config and embeddings tables,
+   * the HNSW index, and the initial config row.
+   *
+   * @param {string} provider   - Embedding provider name.
+   * @param {string} model      - Embedding model name.
+   * @param {number} dimensions - Vector dimension count.
+   */
+  async _firstTimeSetup(provider, model, dimensions) {
+    this.logger.info('embeddings', 'First-time embedding setup: creating tables and index.');
+
+    // Create the single-row configuration table
+    await this.db.query(`
+      CREATE TABLE IF NOT EXISTS embedding_config (
+        id          INTEGER PRIMARY KEY DEFAULT 1 CHECK (id = 1),
+        provider    TEXT NOT NULL,
+        model       TEXT NOT NULL,
+        dimensions  INTEGER NOT NULL,
+        created_at  TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+        updated_at  TIMESTAMPTZ NOT NULL DEFAULT NOW()
+      )
+    `);
+
+    // Create the embeddings table with the resolved vector dimension.
+    // NOTE: The dimension is a validated integer, not user input; string
+    // interpolation in the DDL statement is safe here because parameterized
+    // DDL is not supported by PostgreSQL for column type definitions.
+    await this.db.query(`
+      CREATE TABLE IF NOT EXISTS embeddings (
+        id          SERIAL PRIMARY KEY,
+        created_at  TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+        updated_at  TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+        entity_type TEXT NOT NULL,
+        entity_id   INTEGER NOT NULL,
+        vector      VECTOR(${dimensions}),
+        UNIQUE(entity_type, entity_id)
+      )
+    `);
+
+    // HNSW index for fast approximate nearest-neighbor search (cosine distance)
+    await this.db.query(`
+      CREATE INDEX IF NOT EXISTS idx_embeddings_vector
+      ON embeddings USING hnsw (vector vector_cosine_ops)
+    `);
+
+    // Insert (or update) the config row
+    await this.db.query(
+      `INSERT INTO embedding_config (provider, model, dimensions)
+       VALUES ($1, $2, $3)
+       ON CONFLICT (id) DO UPDATE
+       SET provider = $1, model = $2, dimensions = $3, updated_at = NOW()`,
+      [provider, model, dimensions],
+    );
+
+    this.logger.info('embeddings', 'Embedding tables and index created successfully.');
+  }
+
+  /**
+   * Handle a model configuration change.
+   *
+   * Drops the existing vector column and HNSW index, recreates them with the
+   * new dimension, and updates the config row.  All existing embedding rows
+   * are left with a NULL vector so the background worker re-generates them.
+   *
+   * @param {object} oldConfig - Previous { provider, model, dimensions }.
+   * @param {object} newConfig - New { provider, model, dimensions }.
+   */
+  async _handleModelSwitch(oldConfig, newConfig) {
+    this.logger.warn(
+      'embeddings',
+      `Embedding model changed from ${oldConfig.provider}/${oldConfig.model} ` +
+      `(${oldConfig.dimensions}d) to ${newConfig.provider}/${newConfig.model} ` +
+      `(${newConfig.dimensions}d). All existing embeddings will be dropped and re-generated.`,
+    );
+
+    // Drop the HNSW index
+    await this.db.query('DROP INDEX IF EXISTS idx_embeddings_vector');
+
+    // Drop and recreate the vector column with the new dimension
+    await this.db.query('ALTER TABLE embeddings DROP COLUMN vector');
+    await this.db.query(
+      `ALTER TABLE embeddings ADD COLUMN vector VECTOR(${newConfig.dimensions})`,
+    );
+
+    // Recreate the HNSW index
+    await this.db.query(`
+      CREATE INDEX idx_embeddings_vector
+      ON embeddings USING hnsw (vector vector_cosine_ops)
+    `);
+
+    // Update the config row
+    await this.db.query(
+      `UPDATE embedding_config
+       SET provider = $1, model = $2, dimensions = $3, updated_at = NOW()
+       WHERE id = 1`,
+      [newConfig.provider, newConfig.model, newConfig.dimensions],
+    );
+
+    this.logger.info(
+      'embeddings',
+      'Model switch complete. All embeddings queued for re-generation.',
+    );
+  }
+}
+
+export { EmbeddingsEngine };
+export default EmbeddingsEngine;

package/src/embeddings/worker.js

@@ -0,0 +1,221 @@
+import { generateEmbedding } from '../mcp/embed-server.js';
+
+/**
+ * Mapping of entity types to the SQL query that retrieves the text content
+ * to be embedded for a given entity_id.
+ */
+const ENTITY_TEXT_SOURCES = {
+  message: {
+    query: 'SELECT content AS text FROM conversation_messages WHERE id = $1',
+  },
+  node: {
+    query: `SELECT name || COALESCE(' ' || note, '') AS text FROM knowledge_nodes WHERE id = $1`,
+  },
+  journal: {
+    query: 'SELECT note AS text FROM journal WHERE id = $1',
+  },
+  issue: {
+    query: 'SELECT note AS text FROM issues WHERE id = $1',
+  },
+  spec: {
+    query: 'SELECT note AS text FROM specifications WHERE id = $1',
+  },
+};
+
+/** Maximum rows to process in a single iteration. */
+const BATCH_SIZE = 10;
+
+/** Milliseconds between processing iterations. */
+const POLL_INTERVAL_MS = 5_000;
+
+/**
+ * Background embedding worker -- periodically processes rows in the
+ * embeddings table that have a NULL vector, generates the embedding via
+ * the configured API, and stores the result (spec section 11.4).
+ */
+class EmbeddingWorker {
+  /**
+   * @param {object} deps
+   * @param {object} deps.db     - Database query interface ({ query(sql, params) }).
+   * @param {object} deps.config - Application configuration.
+   * @param {object} deps.logger - Logger instance.
+   */
+  constructor({ db, config, logger }) {
+    this.db = db;
+    this.config = config;
+    this.logger = logger;
+
+    /** @type {ReturnType<typeof setTimeout>|null} */
+    this._timer = null;
+
+    /** Whether the worker loop is active. */
+    this._running = false;
+
+    /** Guard to prevent overlapping iterations. */
+    this._processing = false;
+  }
+
+  /**
+   * Start the periodic embedding worker loop.
+   * Processes up to {@link BATCH_SIZE} NULL-vector rows every
+   * {@link POLL_INTERVAL_MS} milliseconds.
+   */
+  start() {
+    if (this._running) {
+      return;
+    }
+
+    this._running = true;
+    this.logger.info('embedding-worker', 'Starting background embedding worker.');
+    this._scheduleNext();
+  }
+
+  /**
+   * Stop the worker loop gracefully.  Any in-flight iteration will finish
+   * before the loop fully halts.
+   */
+  stop() {
+    this._running = false;
+
+    if (this._timer !== null) {
+      clearTimeout(this._timer);
+      this._timer = null;
+    }
+
+    this.logger.info('embedding-worker', 'Embedding worker stopped.');
+  }
+
+  // ---------------------------------------------------------------------------
+  // Internal
+  // ---------------------------------------------------------------------------
+
+  /**
+   * Schedule the next processing iteration after POLL_INTERVAL_MS.
+   */
+  _scheduleNext() {
+    if (!this._running) {
+      return;
+    }
+
+    this._timer = setTimeout(async () => {
+      this._timer = null;
+
+      // Skip if the previous iteration is still running
+      if (this._processing) {
+        this._scheduleNext();
+        return;
+      }
+
+      try {
+        this._processing = true;
+        await this._processQueue();
+      } catch (err) {
+        this.logger.error(
+          'embedding-worker',
+          `Unexpected error in worker loop: ${err.message}`,
+        );
+      } finally {
+        this._processing = false;
+        this._scheduleNext();
+      }
+    }, POLL_INTERVAL_MS);
+  }
+
+  /**
+   * Fetch and process a batch of rows with NULL vectors.
+   */
+  async _processQueue() {
+    const result = await this.db.query(
+      `SELECT id, entity_type, entity_id
+       FROM embeddings
+       WHERE vector IS NULL
+       ORDER BY created_at ASC
+       LIMIT $1`,
+      [BATCH_SIZE],
+    );
+
+    if (result.rows.length === 0) {
+      return;
+    }
+
+    this.logger.debug(
+      'embedding-worker',
+      `Processing ${result.rows.length} pending embedding(s).`,
+    );
+
+    for (const row of result.rows) {
+      try {
+        await this._processRow(row);
+      } catch (err) {
+        // Log the failure and continue with the next row
+        this.logger.error(
+          'embedding-worker',
+          `Failed to generate embedding for ${row.entity_type}:${row.entity_id}: ${err.message}`,
+        );
+      }
+    }
+  }
+
+  /**
+   * Process a single embedding row: look up the source text, call the
+   * embedding API, and store the resulting vector.
+   *
+   * @param {{ id: number, entity_type: string, entity_id: number }} row
+   */
+  async _processRow(row) {
+    const { id, entity_type: entityType, entity_id: entityId } = row;
+
+    // Resolve the query for this entity type
+    const source = ENTITY_TEXT_SOURCES[entityType];
+    if (!source) {
+      this.logger.warn(
+        'embedding-worker',
+        `Unknown entity type "${entityType}" for embedding ${id}; skipping.`,
+      );
+      return;
+    }
+
+    // Fetch the text content from the source table
+    const textResult = await this.db.query(source.query, [entityId]);
+
+    if (textResult.rows.length === 0) {
+      this.logger.warn(
+        'embedding-worker',
+        `Source entity ${entityType}:${entityId} not found; removing orphaned embedding row ${id}.`,
+      );
+      await this.db.query('DELETE FROM embeddings WHERE id = $1', [id]);
+      return;
+    }
+
+    const text = textResult.rows[0].text;
+    if (!text || text.trim().length === 0) {
+      this.logger.debug(
+        'embedding-worker',
+        `Empty text for ${entityType}:${entityId}; skipping embedding generation.`,
+      );
+      return;
+    }
+
+    // Generate the embedding vector via the configured API
+    const { vector } = await generateEmbedding(text, this.config);
+
+    // Format as a pgvector literal: [0.123,0.456,...]
+    const vectorLiteral = `[${vector.join(',')}]`;
+
+    // Update the row with the computed vector
+    await this.db.query(
+      `UPDATE embeddings
+       SET vector = $1::vector, updated_at = NOW()
+       WHERE id = $2`,
+      [vectorLiteral, id],
+    );
+
+    this.logger.debug(
+      'embedding-worker',
+      `Generated embedding for ${entityType}:${entityId} (${vector.length} dimensions).`,
+    );
+  }
+}
+
+export { EmbeddingWorker };
+export default EmbeddingWorker;