@geraldmaron/construct 1.0.0 → 1.0.2

package/lib/services/pattern-promotion-service.mjs

@@ -0,0 +1,167 @@
+#!/usr/bin/env node
+/**
+ * lib/services/pattern-promotion-service.mjs — Bounded pattern replication engine.
+ *
+ * Analyzes observations, scores candidate patterns, promotes high-quality ones to permanent
+ * learned patterns in role skill files. Prevents unbounded growth with hard limits.
+ *
+ * Limits:
+ * - Max 15 learned patterns per role
+ * - Min confidence 0.85, usage 3, score 0.8
+ * - Semantic + exact deduplication
+ * - Pruning of low-effectiveness patterns
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+
+const ROOT_DIR = process.cwd();
+const SKILLS_DIR = path.join(ROOT_DIR, 'skills', 'roles');
+const OBS_DIR = path.join(process.cwd(), '.cx', 'observations', 'agent-outcomes');
+
+const MAX_PATTERNS = 15;
+const MIN_CONFIDENCE = 0.85;
+const MIN_USAGE = 3;
+const MIN_SCORE = 0.8;
+const MAX_TOKENS_FILE = 2500; // Approx tokens
+
+const LEARNED_SECTION_MARKER = '## Learned Patterns';
+
+function scorePattern(obs) {
+  const confidenceWeight = 0.3;
+  const usageWeight = 0.2;
+  const successWeight = 0.4;
+  const crossProjectWeight = 0.1;
+
+  const normalizedUsage = Math.min((obs.usage_count || 0) / 10, 1.0);
+  const crossProject = (obs.projects && obs.projects.length >= 2) ? 1.0 : 0.5;
+  const successRate = obs.success_rate || 0.5;
+
+  return (
+    (obs.confidence || 0) * confidenceWeight +
+    normalizedUsage * usageWeight +
+    successRate * successWeight +
+    crossProject * crossProjectWeight
+  );
+}
+
+function normalizeSummary(summary) {
+  return summary.toLowerCase().replace(/[^\w\s]/g, '').trim();
+}
+
+function isDuplicate(newObs, existingPatterns) {
+  const newNorm = normalizeSummary(newObs.summary);
+  for (const pat of existingPatterns) {
+    const patNorm = normalizeSummary(pat.summary);
+    if (newNorm.includes(patNorm.slice(0, Math.min(50, patNorm.length))) ||
+        patNorm.includes(newNorm.slice(0, Math.min(50, newNorm.length)))) {
+      return true;
+    }
+  }
+  return false;
+}
+
+function parseLearnedPatterns(content) {
+  const match = content.match(new RegExp(`${LEARNED_SECTION_MARKER}[\\s\\S]*?(?=## |$)`));
+  if (!match) return [];
+
+  const section = match[0];
+  return section.split('\n\n').slice(1).filter(Boolean).map(block => {
+    const lines = block.split('\n');
+    return {
+      summary: lines[0].replace(/^###\s*L\d+\.\s*/, ''),
+      content: block
+    };
+  });
+}
+
+function generatePatternEntry(obs, index) {
+  return `### L${index}. ${obs.summary}
+**Context**: ${obs.context || 'General'}
+**Effective Action**: ${obs.content.slice(0, 120)}...
+**Evidence**: Score ${(obs.score || 0).toFixed(2)}, used ${(obs.usage_count || 0)} times, ${(obs.projects || []).length} projects
+*Last reinforced: ${new Date().toISOString().split('T')[0]}*
+
+`;
+}
+
+function updateSkillFile(roleName, candidate) {
+  const skillFile = path.join(SKILLS_DIR, `${roleName}.md`);
+  if (!fs.existsSync(skillFile)) {
+    console.warn(`Skill file not found: ${skillFile}`);
+    return false;
+  }
+
+  let content = fs.readFileSync(skillFile, 'utf8');
+
+  // Check token limit (rough char/4)
+  if (content.length > MAX_TOKENS_FILE * 4) {
+    console.warn(`Skill file too large: ${skillFile}`);
+    return false;
+  }
+
+  const existingPatterns = parseLearnedPatterns(content);
+  if (existingPatterns.length >= MAX_PATTERNS || isDuplicate(candidate, existingPatterns)) {
+    console.log(`Skipped ${candidate.summary} (dupe or limit reached)`);
+    return false;
+  }
+
+  const entry = generatePatternEntry(candidate, existingPatterns.length + 1);
+  let newContent;
+
+  const sectionExists = content.includes(LEARNED_SECTION_MARKER);
+  if (sectionExists) {
+    const markerIndex = content.indexOf(LEARNED_SECTION_MARKER);
+    const afterMarker = content.slice(markerIndex);
+    const endMatch = afterMarker.match(/\n\n## /) || afterMarker.match(/$/);
+    const insertPos = markerIndex + afterMarker.indexOf(endMatch[0]);
+    newContent = content.slice(0, insertPos) + '\n\n' + entry + content.slice(insertPos);
+  } else {
+    newContent = content + '\n\n' + LEARNED_SECTION_MARKER + '\n\n' + entry;
+  }
+
+  fs.writeFileSync(skillFile, newContent);
+  console.log(`Added pattern to ${roleName}.md: ${candidate.summary}`);
+  return true;
+}
+
+function getCandidates() {
+  if (!fs.existsSync(OBS_DIR)) return [];
+
+  const files = fs.readdirSync(OBS_DIR).filter(f => f.endsWith('.json'));
+  const candidates = [];
+
+  for (const file of files) {
+    try {
+      const obs = JSON.parse(fs.readFileSync(path.join(OBS_DIR, file), 'utf8'));
+      if (obs.category === 'pattern' &&
+          (obs.confidence || 0) >= MIN_CONFIDENCE &&
+          (obs.usage_count || 0) >= MIN_USAGE) {
+        obs.score = scorePattern(obs);
+        if (obs.score >= MIN_SCORE) {
+          candidates.push(obs);
+        }
+      }
+    } catch (e) {
+      // Skip invalid
+    }
+  }
+
+  return candidates.sort((a, b) => b.score - a.score).slice(0, 5); // Top 5
+}
+
+export function promotePatterns() {
+  const candidates = getCandidates();
+  let promoted = 0;
+
+  for (const candidate of candidates) {
+    const roleName = candidate.role.replace(/^cx-/, '') || 'engineer'; // Fallback
+    if (updateSkillFile(roleName, candidate)) promoted++;
+  }
+
+  console.log(`Pattern promotion complete: ${promoted}/${candidates.length} promoted`);
+}
+
+if (import.meta.url === `file://${process.argv[1]}`) {
+  promotePatterns();
+}

package/lib/services/telemetry-backend.mjs

@@ -0,0 +1,178 @@
+/**
+ * lib/services/telemetry-backend.mjs — local services stack management.
+ *
+ * Manages the local Postgres service (services/docker-compose.yml).
+ * In solo mode, Postgres runs as a Docker container with pgvector enabled.
+ *
+ * Remote telemetry backends (for trace ingestion) are configured via
+ * CONSTRUCT_TELEMETRY_URL / CONSTRUCT_TELEMETRY_PUBLIC_KEY /
+ * CONSTRUCT_TELEMETRY_SECRET_KEY env vars. This module handles local
+ * infrastructure only — it does not manage telemetry dashboard services.
+ */
+
+import { spawnSync } from 'node:child_process';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+
+import { getUserEnvPath, writeEnvValues } from '../env-config.mjs';
+
+/** Local Postgres port (within Construct's 54329-54339 reserved block). */
+export const POSTGRES_LOCAL_PORT = 54332;
+
+export function isRemoteTelemetry(url = '') {
+  if (!url) return false;
+  return !url.includes('localhost') && !url.includes('127.0.0.1');
+}
+
+export function servicesComposePath(rootDir) {
+  return path.join(rootDir, 'services', 'docker-compose.yml');
+}
+
+// Keep only the N most recent stash pairs (.dump + .json) in the stash dir.
+export function pruneStashDir(dir, keep = 3) {
+  try {
+    const dumps = fs.readdirSync(dir)
+      .filter((f) => f.endsWith('.dump'))
+      .sort()
+      .reverse();
+    for (const dump of dumps.slice(keep)) {
+      fs.rmSync(path.join(dir, dump), { force: true });
+      fs.rmSync(path.join(dir, dump.replace('.dump', '.json')), { force: true });
+    }
+  } catch { /* non-critical */ }
+}
+
+/**
+ * Verify that the Postgres service is reachable. Returns a status object.
+ * Verifies the Postgres service is up and ready.
+ */
+export async function verifyPostgresHealth({
+  port = POSTGRES_LOCAL_PORT,
+  fetchFn = globalThis.fetch,
+  maxRetries = 5,
+  intervalMs = 2000,
+} = {}) {
+  // We can't use fetch against Postgres (it's not HTTP), so just check pg_isready via docker
+  for (let attempt = 0; attempt < maxRetries; attempt++) {
+    try {
+      const result = spawnSync('docker', [
+        'exec', 'construct-postgres',
+        'pg_isready', '-U', 'construct', '-d', 'construct',
+      ], { stdio: 'pipe', timeout: 3000 });
+      if (result.status === 0) return { status: 'verified' };
+    } catch { /* not ready yet */ }
+    if (attempt < maxRetries - 1) await new Promise((r) => setTimeout(r, intervalMs));
+  }
+  return { status: 'unreachable' };
+}
+
+/**
+ * Verify telemetry backend keys (remote HTTP).
+ * 
+ */
+export async function verifyTelemetryKeys({
+  baseUrl,
+  publicKey,
+  secretKey,
+  composeRunner,
+  composeFile,
+  homeDir = os.homedir(),
+  maxRetries = 5,
+  intervalMs = 2000,
+  spawnSyncFn = spawnSync,
+  fetchFn = globalThis.fetch,
+  overallTimeoutMs = 0,
+} = {}) {
+  const resolvedBaseUrl =
+    baseUrl ??
+    process.env.CONSTRUCT_TELEMETRY_URL ??
+    '';
+  const resolvedPublicKey =
+    publicKey ??
+    process.env.CONSTRUCT_TELEMETRY_PUBLIC_KEY ??
+    '';
+  const resolvedSecretKey =
+    secretKey ??
+    process.env.CONSTRUCT_TELEMETRY_SECRET_KEY ??
+    '';
+
+  if (!resolvedPublicKey || !resolvedSecretKey || !resolvedBaseUrl) {
+    return { status: 'unconfigured' };
+  }
+
+  async function doVerify() {
+    const auth = `Basic ${Buffer.from(`${resolvedPublicKey}:${resolvedSecretKey}`).toString('base64')}`;
+    for (let attempt = 0; attempt < maxRetries; attempt++) {
+      try {
+        const controller = new AbortController();
+        const timer = setTimeout(() => controller.abort(), 3000);
+        const res = await fetchFn(`${resolvedBaseUrl}/api/public/health`, {
+          signal: controller.signal,
+        }).finally(() => clearTimeout(timer));
+        if (res.ok) break;
+      } catch { /* not ready yet */ }
+      if (attempt < maxRetries - 1) await new Promise((r) => setTimeout(r, intervalMs));
+    }
+
+    try {
+      const controller = new AbortController();
+      const timer = setTimeout(() => controller.abort(), 5000);
+      const res = await fetchFn(`${resolvedBaseUrl}/api/public/traces?limit=1`, {
+        headers: { Authorization: auth },
+        signal: controller.signal,
+      }).finally(() => clearTimeout(timer));
+      if (res.ok) return { status: 'verified' };
+    } catch {
+      return { status: 'unreachable' };
+    }
+    return { status: 'auth-failed', reseeded: false };
+  }
+
+  if (overallTimeoutMs > 0) {
+    return Promise.race([
+      doVerify(),
+      new Promise((_, reject) =>
+        setTimeout(() => reject(new Error('verifyTelemetryKeys timeout')), overallTimeoutMs),
+      ),
+    ]);
+  }
+  return doVerify();
+}
+
+/**
+ * Bring up the local Postgres service via services/docker-compose.yml.
+ * Starts the local Postgres service via services/docker-compose.yml.
+ */
+export async function startManagedServices({
+  rootDir,
+  homeDir = os.homedir(),
+  env = process.env,
+  composeRunner,
+  spawnDetached,
+} = {}) {
+  if (!rootDir) throw new Error('startManagedServices: rootDir is required');
+  if (!composeRunner) throw new Error('startManagedServices: composeRunner is required');
+  if (!spawnDetached) throw new Error('startManagedServices: spawnDetached is required');
+
+  const composeFile = servicesComposePath(rootDir);
+  if (!fs.existsSync(composeFile)) {
+    return { status: 'unavailable', note: `services compose file missing at ${composeFile}` };
+  }
+
+  const { logPath } = spawnDetached(
+    composeRunner.command,
+    [...composeRunner.argsPrefix, '-p', 'construct-services', '-f', composeFile, 'up', '-d'],
+    homeDir,
+    'services.log',
+  );
+
+  return {
+    status: 'started',
+    note: `Postgres started — connect at localhost:${POSTGRES_LOCAL_PORT}`,
+    composeFile,
+    logPath,
+  };
+}
+
+