@geraldmaron/construct 1.0.18 → 1.0.20

package/lib/embed/jobs/vector-sync.mjs

@@ -1,198 +0,0 @@
-/**
- * lib/embed/jobs/vector-sync.mjs — Automatic embedding pipeline for observations.
- *
- * Runs on a schedule (default: every 5 minutes) to embed new/modified observations
- * and store them in Postgres (if available) and/or the local vector index.
- *
- * Design:
- *   - Scans .cx/observations/ for items not yet embedded
- *   - Embeds each using the configured model (local ONNX, OpenAI, Ollama)
- *   - Stores in Postgres construct_observations table (if DATABASE_URL set)
- *   - Falls back to local vectors.json if Postgres unavailable
- *   - Batched processing to avoid overwhelming the embedding API
- *   - Deduplicated via content hash comparison
- */
-import { existsSync, readFileSync, writeFileSync, statSync, mkdirSync } from 'node:fs';
-import { join, dirname } from 'node:path';
-import { homedir } from 'node:os';
-
-const DEFAULT_BATCH_SIZE = 32;
-const DEFAULT_INTERVAL_MS = 5 * 60 * 1000;
-
-/**
- * Find observations that haven't been embedded yet.
- * @param {string} rootDir
- * @returns {Array<{ id: string, project: string, role: string, category: string, summary: string, content: string, tags: string[], confidence: number, source: string, filePath: string }>}
- */
-function findUnembeddedObservations(rootDir) {
-  const obsDir = join(rootDir, '.cx', 'observations');
-  if (!existsSync(obsDir)) return [];
-
-  const indexPath = join(obsDir, 'index.json');
-  if (!existsSync(indexPath)) return [];
-
-  let index;
-  try {
-    index = JSON.parse(readFileSync(indexPath, 'utf8'));
-  } catch (err) {
-    process.stderr.write('[jobs/vector-sync.mjs] findUnembedded: ' + (err?.message ?? String(err)) + '\n');
-    return [];
-  }
-
-  // Check which observations have been embedded
-  const vectorsPath = join(obsDir, 'vectors.json');
-  const embeddedIds = new Set();
-  if (existsSync(vectorsPath)) {
-    try {
-      const vectors = JSON.parse(readFileSync(vectorsPath, 'utf8'));
-      for (const v of (vectors.records || vectors)) {
-        if (v.id) embeddedIds.add(v.id);
-      }
-    } catch { /* ignore */ }
-  }
-
-  // Also check Postgres if available
-  // (This is handled by the caller via VectorClient.isHealthy())
-
-  return (index || [])
-    .filter((obs) => obs.id && !embeddedIds.has(obs.id))
-    .map((obs) => ({
-      ...obs,
-      filePath: join(obsDir, `${obs.id}.json`),
-    }))
-    .filter((obs) => existsSync(obs.filePath));
-}
-
-/**
- * Run the vector sync job.
- * @param {object} opts
- * @param {string} opts.rootDir - Root directory for observation store
- * @param {object} opts.env - Environment variables
- * @param {object} opts.logger - Optional logger with debug/info/error methods
- * @param {object} opts.vectorClient - Optional VectorClient instance for SQL storage
- * @returns {Promise<{ synced: number, errors: number, durationMs: number }>}
- */
-export async function runVectorSync({ rootDir, env = process.env, logger, vectorClient } = {}) {
-  const startTime = Date.now();
-  const batchSize = parseInt(env.CONSTRUCT_EMBEDDING_BATCH_SIZE, 10) || DEFAULT_BATCH_SIZE;
-
-  try {
-    const unembedded = findUnembeddedObservations(rootDir);
-    if (!unembedded.length) {
-      logger?.debug?.('Vector sync: no new observations to embed');
-      return { synced: 0, errors: 0, durationMs: Date.now() - startTime };
-    }
-
-    let synced = 0;
-    let errors = 0;
-
-    // Dynamically import embedding engine
-    const { embedBatch } = await import('../../storage/embeddings-engine.mjs');
-
-    for (let i = 0; i < unembedded.length; i += batchSize) {
-      const batch = unembedded.slice(i, i + batchSize);
-      const texts = batch.map((o) => `${o.summary}\n${o.content}`);
-
-      try {
-        const embeddings = await embedBatch(texts, { env });
-
-        for (let j = 0; j < batch.length; j++) {
-          const obs = batch[j];
-          const embedding = embeddings[j].embedding;
-
-          // Store in Postgres if available
-          if (vectorClient && await vectorClient.isHealthy()) {
-            try {
-              await vectorClient.storeObservation({
-                id: obs.id,
-                project: obs.project || rootDir.split('/').pop(),
-                role: obs.role,
-                category: obs.category,
-                summary: obs.summary,
-                content: obs.content,
-                tags: obs.tags || [],
-                confidence: obs.confidence ?? 0.8,
-                source: obs.source,
-                embedding,
-              });
-            } catch (err) {
-              logger?.error?.(`Vector sync: failed to store observation ${obs.id} in SQL: ${err.message}`);
-              errors++;
-              continue;
-            }
-          }
-
-          // Always update local vectors.json
-          updateLocalVectorIndex(rootDir, obs, embedding);
-          synced++;
-        }
-      } catch (err) {
-        logger?.error?.(`Vector sync: batch embedding failed: ${err.message}`);
-        errors += batch.length;
-      }
-    }
-
-    const durationMs = Date.now() - startTime;
-    logger?.info?.(`Vector sync: embedded ${synced} observation(s) in ${durationMs}ms (${errors} errors)`);
-    return { synced, errors, durationMs };
-  } catch (err) {
-    const durationMs = Date.now() - startTime;
-    logger?.error?.(`Vector sync failed: ${err.message}`);
-    return { synced: 0, errors: 1, durationMs };
-  }
-}
-
-/**
- * Update the local vectors.json index with a new embedding.
- * @param {string} rootDir
- * @param {object} obs - Observation record
- * @param {Float32Array} embedding
- */
-function updateLocalVectorIndex(rootDir, obs, embedding) {
-  const obsDir = join(rootDir, '.cx', 'observations');
-  const vectorsPath = join(obsDir, 'vectors.json');
-
-  let vectors;
-  if (existsSync(vectorsPath)) {
-    try {
-      vectors = JSON.parse(readFileSync(vectorsPath, 'utf8'));
-    } catch (err) {
-      process.stderr.write('[jobs/vector-sync.mjs] updateLocalVectorIndex: ' + (err?.message ?? String(err)) + '\n');
-      vectors = { version: 1, model: 'hashing-bow-v1', updatedAt: null, records: [] };
-    }
-  } else {
-    vectors = { version: 1, model: 'hashing-bow-v1', updatedAt: null, records: [] };
-  }
-
-  // Add or update the record
-  const existingIdx = vectors.records.findIndex((r) => r.id === obs.id);
-  const record = {
-    id: obs.id,
-    embedding: Array.from(embedding),
-    summary: obs.summary,
-    content: obs.content,
-    tags: obs.tags || [],
-    project: obs.project,
-    role: obs.role,
-    category: obs.category,
-    updatedAt: new Date().toISOString(),
-  };
-
-  if (existingIdx >= 0) {
-    vectors.records[existingIdx] = record;
-  } else {
-    vectors.records.push(record);
-  }
-
-  // Cap at 1000 records (sliding window)
-  if (vectors.records.length > 1000) {
-    vectors.records = vectors.records.slice(-1000);
-  }
-
-  vectors.updatedAt = new Date().toISOString();
-
-  mkdirSync(dirname(vectorsPath), { recursive: true });
-  writeFileSync(vectorsPath, JSON.stringify(vectors, null, 2), 'utf8');
-}
-
-export { DEFAULT_BATCH_SIZE, DEFAULT_INTERVAL_MS };

package/lib/knowledge/postgres-search.mjs

@@ -1,132 +0,0 @@
-/**
- * lib/knowledge/postgres-search.mjs — Tag-aware vector search via pgvector + GIN.
- *
- * Runs when DATABASE_URL is configured. Chooses between two execution paths
- * based on tag selectivity:
- *
- * - High selectivity (tag matches <5% of corpus): GIN prefilter first, then
- *   HNSW ANN on the filtered set.
- * - Low selectivity (tag matches ≥5% of corpus): HNSW ANN first with
- *   hnsw.iterative_scan=relaxed_order, postfilter by tag.
- *
- * Falls back gracefully when DATABASE_URL is absent or the pgvector extension
- * is not installed; callers should use knowledgeSearch() from search.mjs in
- * that case.
- */
-
-// Selectivity threshold: if the tagged fraction exceeds this, use postfilter.
-const HIGH_SELECTIVITY_THRESHOLD = 0.05;
-
-/**
- * Execute a tag-filtered vector search against Postgres + pgvector.
- *
- * @param {object} opts
- * @param {Float32Array|number[]} opts.queryEmbedding   — query vector
- * @param {string}   opts.project       — project id (construct_documents.project)
- * @param {string[]} [opts.tags]        — tag filter
- * @param {'any'|'all'} [opts.tagMatch] — 'any' (default) or 'all'
- * @param {number}   [opts.topK]        — result limit (default 20)
- * @param {object}   opts.db            — postgres client (from lib/db.mjs or postgres pkg)
- * @returns {Promise<Array<{id, title, score}>>}
- */
-export async function postgresTagSearch({ queryEmbedding, project, tags = [], tagMatch = 'any', topK = 20, db }) {
-  if (!db) throw new Error('postgres-search: db client required');
-  if (!queryEmbedding?.length) throw new Error('postgres-search: queryEmbedding required');
-
-  const vecLiteral = `[${Array.from(queryEmbedding).join(',')}]`;
-
-  if (!tags.length) {
-    // No tag filter — plain ANN.
-    const rows = await db`
-      select d.id, d.title, e.embedding <=> ${vecLiteral}::vector as distance
-      from construct_embeddings e
-      join construct_documents d on d.id = e.document_id
-      where d.project = ${project}
-      order by e.embedding <=> ${vecLiteral}::vector
-      limit ${topK}
-    `;
-    return rows.map((r) => ({ id: r.id, title: r.title, score: 1 - parseFloat(r.distance) }));
-  }
-
-  // Estimate selectivity for this tag set.
-  const tagJsonb = JSON.stringify(tags);
-  const [{ total }] = await db`select count(*) as total from construct_documents where project = ${project}`;
-  const totalCount = parseInt(total, 10) || 1;
-
-  let taggedCount;
-  if (tagMatch === 'all') {
-    const [{ cnt }] = await db`
-      select count(*) as cnt from construct_documents
-      where project = ${project} and tags @> ${tagJsonb}::jsonb
-    `;
-    taggedCount = parseInt(cnt, 10);
-  } else {
-    // 'any' — count docs with at least one matching tag.
-    const [{ cnt }] = await db`
-      select count(*) as cnt from construct_documents
-      where project = ${project}
-        and (${db.unsafe(tags.map((_, i) => `tags ? $${i + 2}`).join(' or '))})
-    `;
-    taggedCount = parseInt(cnt, 10);
-  }
-
-  const selectivity = taggedCount / totalCount;
-
-  if (selectivity < HIGH_SELECTIVITY_THRESHOLD) {
-    // High selectivity: GIN prefilter → ANN on filtered set.
-    if (tagMatch === 'all') {
-      const rows = await db`
-        with filtered as (
-          select id from construct_documents
-          where project = ${project} and tags @> ${tagJsonb}::jsonb
-        )
-        select d.id, d.title, e.embedding <=> ${vecLiteral}::vector as distance
-        from construct_embeddings e
-        join filtered f on f.id = e.document_id
-        join construct_documents d on d.id = e.document_id
-        order by e.embedding <=> ${vecLiteral}::vector
-        limit ${topK}
-      `;
-      return rows.map((r) => ({ id: r.id, title: r.title, score: 1 - parseFloat(r.distance) }));
-    }
-    // 'any' — OR-expand tag list.
-    const tagConditions = tags.map((t) => `tags ? '${t.replace(/'/g, "''")}'`).join(' or ');
-    const rows = await db.unsafe(`
-      with filtered as (
-        select id from construct_documents
-        where project = $1 and (${tagConditions})
-      )
-      select d.id, d.title, e.embedding <=> $2::vector as distance
-      from construct_embeddings e
-      join filtered f on f.id = e.document_id
-      join construct_documents d on d.id = e.document_id
-      order by e.embedding <=> $2::vector
-      limit $3
-    `, [project, vecLiteral, topK]);
-    return rows.map((r) => ({ id: r.id, title: r.title, score: 1 - parseFloat(r.distance) }));
-  }
-
-  // Low selectivity: HNSW iterative scan → postfilter.
-  await db`set hnsw.iterative_scan = relaxed_order`;
-  if (tagMatch === 'all') {
-    const rows = await db`
-      select d.id, d.title, e.embedding <=> ${vecLiteral}::vector as distance
-      from construct_embeddings e
-      join construct_documents d on d.id = e.document_id
-      where d.project = ${project} and d.tags @> ${tagJsonb}::jsonb
-      order by e.embedding <=> ${vecLiteral}::vector
-      limit ${topK}
-    `;
-    return rows.map((r) => ({ id: r.id, title: r.title, score: 1 - parseFloat(r.distance) }));
-  }
-  const tagConditions = tags.map((t) => `d.tags ? '${t.replace(/'/g, "''")}'`).join(' or ');
-  const rows = await db.unsafe(`
-    select d.id, d.title, e.embedding <=> $1::vector as distance
-    from construct_embeddings e
-    join construct_documents d on d.id = e.document_id
-    where d.project = $2 and (${tagConditions})
-    order by e.embedding <=> $1::vector
-    limit $3
-  `, [vecLiteral, project, topK]);
-  return rows.map((r) => ({ id: r.id, title: r.title, score: 1 - parseFloat(r.distance) }));
-}

package/lib/services/pattern-promotion-service.mjs

@@ -1,167 +0,0 @@
-#!/usr/bin/env node
-/**
- * lib/services/pattern-promotion-service.mjs — Bounded pattern replication engine.
- *
- * Analyzes observations, scores candidate patterns, promotes high-quality ones to permanent
- * learned patterns in role skill files. Prevents unbounded growth with hard limits.
- *
- * Limits:
- * - Max 15 learned patterns per role
- * - Min confidence 0.85, usage 3, score 0.8
- * - Semantic + exact deduplication
- * - Pruning of low-effectiveness patterns
- */
-
-import fs from 'node:fs';
-import path from 'node:path';
-
-const ROOT_DIR = process.cwd();
-const SKILLS_DIR = path.join(ROOT_DIR, 'skills', 'roles');
-const OBS_DIR = path.join(process.cwd(), '.cx', 'observations', 'agent-outcomes');
-
-const MAX_PATTERNS = 15;
-const MIN_CONFIDENCE = 0.85;
-const MIN_USAGE = 3;
-const MIN_SCORE = 0.8;
-const MAX_TOKENS_FILE = 2500; // Approx tokens
-
-const LEARNED_SECTION_MARKER = '## Learned Patterns';
-
-function scorePattern(obs) {
-  const confidenceWeight = 0.3;
-  const usageWeight = 0.2;
-  const successWeight = 0.4;
-  const crossProjectWeight = 0.1;
-
-  const normalizedUsage = Math.min((obs.usage_count || 0) / 10, 1.0);
-  const crossProject = (obs.projects && obs.projects.length >= 2) ? 1.0 : 0.5;
-  const successRate = obs.success_rate || 0.5;
-
-  return (
-    (obs.confidence || 0) * confidenceWeight +
-    normalizedUsage * usageWeight +
-    successRate * successWeight +
-    crossProject * crossProjectWeight
-  );
-}
-
-function normalizeSummary(summary) {
-  return summary.toLowerCase().replace(/[^\w\s]/g, '').trim();
-}
-
-function isDuplicate(newObs, existingPatterns) {
-  const newNorm = normalizeSummary(newObs.summary);
-  for (const pat of existingPatterns) {
-    const patNorm = normalizeSummary(pat.summary);
-    if (newNorm.includes(patNorm.slice(0, Math.min(50, patNorm.length))) ||
-        patNorm.includes(newNorm.slice(0, Math.min(50, newNorm.length)))) {
-      return true;
-    }
-  }
-  return false;
-}
-
-function parseLearnedPatterns(content) {
-  const match = content.match(new RegExp(`${LEARNED_SECTION_MARKER}[\\s\\S]*?(?=## |$)`));
-  if (!match) return [];
-
-  const section = match[0];
-  return section.split('\n\n').slice(1).filter(Boolean).map(block => {
-    const lines = block.split('\n');
-    return {
-      summary: lines[0].replace(/^###\s*L\d+\.\s*/, ''),
-      content: block
-    };
-  });
-}
-
-function generatePatternEntry(obs, index) {
-  return `### L${index}. ${obs.summary}
-**Context**: ${obs.context || 'General'}
-**Effective Action**: ${obs.content.slice(0, 120)}...
-**Evidence**: Score ${(obs.score || 0).toFixed(2)}, used ${(obs.usage_count || 0)} times, ${(obs.projects || []).length} projects
-*Last reinforced: ${new Date().toISOString().split('T')[0]}*
-
-`;
-}
-
-function updateSkillFile(roleName, candidate) {
-  const skillFile = path.join(SKILLS_DIR, `${roleName}.md`);
-  if (!fs.existsSync(skillFile)) {
-    console.warn(`Skill file not found: ${skillFile}`);
-    return false;
-  }
-
-  let content = fs.readFileSync(skillFile, 'utf8');
-
-  // Check token limit (rough char/4)
-  if (content.length > MAX_TOKENS_FILE * 4) {
-    console.warn(`Skill file too large: ${skillFile}`);
-    return false;
-  }
-
-  const existingPatterns = parseLearnedPatterns(content);
-  if (existingPatterns.length >= MAX_PATTERNS || isDuplicate(candidate, existingPatterns)) {
-    console.log(`Skipped ${candidate.summary} (dupe or limit reached)`);
-    return false;
-  }
-
-  const entry = generatePatternEntry(candidate, existingPatterns.length + 1);
-  let newContent;
-
-  const sectionExists = content.includes(LEARNED_SECTION_MARKER);
-  if (sectionExists) {
-    const markerIndex = content.indexOf(LEARNED_SECTION_MARKER);
-    const afterMarker = content.slice(markerIndex);
-    const endMatch = afterMarker.match(/\n\n## /) || afterMarker.match(/$/);
-    const insertPos = markerIndex + afterMarker.indexOf(endMatch[0]);
-    newContent = content.slice(0, insertPos) + '\n\n' + entry + content.slice(insertPos);
-  } else {
-    newContent = content + '\n\n' + LEARNED_SECTION_MARKER + '\n\n' + entry;
-  }
-
-  fs.writeFileSync(skillFile, newContent);
-  console.log(`Added pattern to ${roleName}.md: ${candidate.summary}`);
-  return true;
-}
-
-function getCandidates() {
-  if (!fs.existsSync(OBS_DIR)) return [];
-
-  const files = fs.readdirSync(OBS_DIR).filter(f => f.endsWith('.json'));
-  const candidates = [];
-
-  for (const file of files) {
-    try {
-      const obs = JSON.parse(fs.readFileSync(path.join(OBS_DIR, file), 'utf8'));
-      if (obs.category === 'pattern' &&
-          (obs.confidence || 0) >= MIN_CONFIDENCE &&
-          (obs.usage_count || 0) >= MIN_USAGE) {
-        obs.score = scorePattern(obs);
-        if (obs.score >= MIN_SCORE) {
-          candidates.push(obs);
-        }
-      }
-    } catch (e) {
-      // Skip invalid
-    }
-  }
-
-  return candidates.sort((a, b) => b.score - a.score).slice(0, 5); // Top 5
-}
-
-export function promotePatterns() {
-  const candidates = getCandidates();
-  let promoted = 0;
-
-  for (const candidate of candidates) {
-    const roleName = candidate.role.replace(/^cx-/, '') || 'engineer'; // Fallback
-    if (updateSkillFile(roleName, candidate)) promoted++;
-  }
-
-  console.log(`Pattern promotion complete: ${promoted}/${candidates.length} promoted`);
-}
-
-if (import.meta.url === `file://${process.argv[1]}`) {
-  promotePatterns();
-}