@geraldmaron/construct 1.0.0 → 1.0.2

package/lib/embed/jobs/vector-sync.mjs

@@ -0,0 +1,198 @@
+/**
+ * lib/embed/jobs/vector-sync.mjs — Automatic embedding pipeline for observations.
+ *
+ * Runs on a schedule (default: every 5 minutes) to embed new/modified observations
+ * and store them in Postgres (if available) and/or the local vector index.
+ *
+ * Design:
+ *   - Scans .cx/observations/ for items not yet embedded
+ *   - Embeds each using the configured model (local ONNX, OpenAI, Ollama)
+ *   - Stores in Postgres construct_observations table (if DATABASE_URL set)
+ *   - Falls back to local vectors.json if Postgres unavailable
+ *   - Batched processing to avoid overwhelming the embedding API
+ *   - Deduplicated via content hash comparison
+ */
+import { existsSync, readFileSync, writeFileSync, statSync, mkdirSync } from 'node:fs';
+import { join, dirname } from 'node:path';
+import { homedir } from 'node:os';
+
+const DEFAULT_BATCH_SIZE = 32;
+const DEFAULT_INTERVAL_MS = 5 * 60 * 1000;
+
+/**
+ * Find observations that haven't been embedded yet.
+ * @param {string} rootDir
+ * @returns {Array<{ id: string, project: string, role: string, category: string, summary: string, content: string, tags: string[], confidence: number, source: string, filePath: string }>}
+ */
+function findUnembeddedObservations(rootDir) {
+  const obsDir = join(rootDir, '.cx', 'observations');
+  if (!existsSync(obsDir)) return [];
+
+  const indexPath = join(obsDir, 'index.json');
+  if (!existsSync(indexPath)) return [];
+
+  let index;
+  try {
+    index = JSON.parse(readFileSync(indexPath, 'utf8'));
+  } catch (err) {
+    process.stderr.write('[jobs/vector-sync.mjs] findUnembedded: ' + (err?.message ?? String(err)) + '\n');
+    return [];
+  }
+
+  // Check which observations have been embedded
+  const vectorsPath = join(obsDir, 'vectors.json');
+  const embeddedIds = new Set();
+  if (existsSync(vectorsPath)) {
+    try {
+      const vectors = JSON.parse(readFileSync(vectorsPath, 'utf8'));
+      for (const v of (vectors.records || vectors)) {
+        if (v.id) embeddedIds.add(v.id);
+      }
+    } catch { /* ignore */ }
+  }
+
+  // Also check Postgres if available
+  // (This is handled by the caller via VectorClient.isHealthy())
+
+  return (index || [])
+    .filter((obs) => obs.id && !embeddedIds.has(obs.id))
+    .map((obs) => ({
+      ...obs,
+      filePath: join(obsDir, `${obs.id}.json`),
+    }))
+    .filter((obs) => existsSync(obs.filePath));
+}
+
+/**
+ * Run the vector sync job.
+ * @param {object} opts
+ * @param {string} opts.rootDir - Root directory for observation store
+ * @param {object} opts.env - Environment variables
+ * @param {object} opts.logger - Optional logger with debug/info/error methods
+ * @param {object} opts.vectorClient - Optional VectorClient instance for SQL storage
+ * @returns {Promise<{ synced: number, errors: number, durationMs: number }>}
+ */
+export async function runVectorSync({ rootDir, env = process.env, logger, vectorClient } = {}) {
+  const startTime = Date.now();
+  const batchSize = parseInt(env.CONSTRUCT_EMBEDDING_BATCH_SIZE, 10) || DEFAULT_BATCH_SIZE;
+
+  try {
+    const unembedded = findUnembeddedObservations(rootDir);
+    if (!unembedded.length) {
+      logger?.debug?.('Vector sync: no new observations to embed');
+      return { synced: 0, errors: 0, durationMs: Date.now() - startTime };
+    }
+
+    let synced = 0;
+    let errors = 0;
+
+    // Dynamically import embedding engine
+    const { embedBatch } = await import('../../storage/embeddings-engine.mjs');
+
+    for (let i = 0; i < unembedded.length; i += batchSize) {
+      const batch = unembedded.slice(i, i + batchSize);
+      const texts = batch.map((o) => `${o.summary}\n${o.content}`);
+
+      try {
+        const embeddings = await embedBatch(texts, { env });
+
+        for (let j = 0; j < batch.length; j++) {
+          const obs = batch[j];
+          const embedding = embeddings[j].embedding;
+
+          // Store in Postgres if available
+          if (vectorClient && await vectorClient.isHealthy()) {
+            try {
+              await vectorClient.storeObservation({
+                id: obs.id,
+                project: obs.project || rootDir.split('/').pop(),
+                role: obs.role,
+                category: obs.category,
+                summary: obs.summary,
+                content: obs.content,
+                tags: obs.tags || [],
+                confidence: obs.confidence ?? 0.8,
+                source: obs.source,
+                embedding,
+              });
+            } catch (err) {
+              logger?.error?.(`Vector sync: failed to store observation ${obs.id} in SQL: ${err.message}`);
+              errors++;
+              continue;
+            }
+          }
+
+          // Always update local vectors.json
+          updateLocalVectorIndex(rootDir, obs, embedding);
+          synced++;
+        }
+      } catch (err) {
+        logger?.error?.(`Vector sync: batch embedding failed: ${err.message}`);
+        errors += batch.length;
+      }
+    }
+
+    const durationMs = Date.now() - startTime;
+    logger?.info?.(`Vector sync: embedded ${synced} observation(s) in ${durationMs}ms (${errors} errors)`);
+    return { synced, errors, durationMs };
+  } catch (err) {
+    const durationMs = Date.now() - startTime;
+    logger?.error?.(`Vector sync failed: ${err.message}`);
+    return { synced: 0, errors: 1, durationMs };
+  }
+}
+
+/**
+ * Update the local vectors.json index with a new embedding.
+ * @param {string} rootDir
+ * @param {object} obs - Observation record
+ * @param {Float32Array} embedding
+ */
+function updateLocalVectorIndex(rootDir, obs, embedding) {
+  const obsDir = join(rootDir, '.cx', 'observations');
+  const vectorsPath = join(obsDir, 'vectors.json');
+
+  let vectors;
+  if (existsSync(vectorsPath)) {
+    try {
+      vectors = JSON.parse(readFileSync(vectorsPath, 'utf8'));
+    } catch (err) {
+      process.stderr.write('[jobs/vector-sync.mjs] updateLocalVectorIndex: ' + (err?.message ?? String(err)) + '\n');
+      vectors = { version: 1, model: 'hashing-bow-v1', updatedAt: null, records: [] };
+    }
+  } else {
+    vectors = { version: 1, model: 'hashing-bow-v1', updatedAt: null, records: [] };
+  }
+
+  // Add or update the record
+  const existingIdx = vectors.records.findIndex((r) => r.id === obs.id);
+  const record = {
+    id: obs.id,
+    embedding: Array.from(embedding),
+    summary: obs.summary,
+    content: obs.content,
+    tags: obs.tags || [],
+    project: obs.project,
+    role: obs.role,
+    category: obs.category,
+    updatedAt: new Date().toISOString(),
+  };
+
+  if (existingIdx >= 0) {
+    vectors.records[existingIdx] = record;
+  } else {
+    vectors.records.push(record);
+  }
+
+  // Cap at 1000 records (sliding window)
+  if (vectors.records.length > 1000) {
+    vectors.records = vectors.records.slice(-1000);
+  }
+
+  vectors.updatedAt = new Date().toISOString();
+
+  mkdirSync(dirname(vectorsPath), { recursive: true });
+  writeFileSync(vectorsPath, JSON.stringify(vectors, null, 2), 'utf8');
+}
+
+export { DEFAULT_BATCH_SIZE, DEFAULT_INTERVAL_MS };

package/lib/embed/notifications.mjs

@@ -0,0 +1,75 @@
+/**
+ * lib/embed/notifications.mjs — embed action notification bus.
+ *
+ * Decoupled event emitter for embed daemon → dashboard SSE.
+ * The dashboard server subscribes to this bus; the daemon emits events.
+ * Slack stub is wired but no-ops without config.
+ */
+
+import { EventEmitter } from 'node:events';
+
+const bus = new EventEmitter();
+bus.setMaxListeners(50);
+
+/**
+ * Emit a toast notification for an embed action.
+ *
+ * @param {object} event
+ * @param {string} event.type    - 'info' | 'success' | 'warning' | 'error'
+ * @param {string} event.source  - job name (e.g. 'docs-lifecycle', 'roadmap', 'snapshot')
+ * @param {string} event.message - human-readable description
+ * @param {object} [event.meta]  - optional structured metadata
+ */
+export function emitEmbedNotification(event) {
+  bus.emit('embed:action', {
+    ...event,
+    ts: Date.now(),
+  });
+}
+
+/**
+ * Subscribe to embed notifications.
+ * @param {(event: object) => void} handler
+ * @returns {() => void} unsubscribe function
+ */
+export function onEmbedNotification(handler) {
+  bus.on('embed:action', handler);
+  return () => bus.off('embed:action', handler);
+}
+
+/**
+ * Stub Slack notification path.
+ * Posts to Slack when SLACK_WEBHOOK_URL is set, otherwise no-ops.
+ *
+ * @param {object} event - same shape as emitEmbedNotification
+ * @param {object} [env] - process.env override for testing
+ */
+export async function notifySlack(event, env = process.env) {
+  const webhookUrl = env.SLACK_EMBED_WEBHOOK_URL ?? env.SLACK_WEBHOOK_URL;
+  if (!webhookUrl) return { sent: false, reason: 'no-webhook-url' };
+
+  try {
+    const body = JSON.stringify({
+      text: `[${event.source}] ${event.message}`,
+      blocks: [
+        {
+          type: 'section',
+          text: {
+            type: 'mrkdwn',
+            text: `*${event.source}* — ${event.message}`,
+          },
+        },
+      ],
+    });
+
+    const res = await fetch(webhookUrl, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body,
+    });
+
+    return { sent: res.ok, status: res.status };
+  } catch (err) {
+    return { sent: false, reason: err.message };
+  }
+}

package/lib/embed/output.mjs

@@ -0,0 +1,79 @@
+/**
+ * lib/embed/output.mjs — snapshot output dispatcher.
+ *
+ * Routes a rendered snapshot to one or more configured output targets:
+ *   - markdown: write to a file
+ *   - slack:    post to a channel via the slack provider (authority-guarded)
+ *   - log:      print to stdout (default / fallback)
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+import { renderMarkdown } from './snapshot.mjs';
+
+/**
+ * Dispatch a snapshot to all configured outputs.
+ * @param {object} snapshot              - Raw snapshot object from SnapshotEngine
+ * @param {object[]} outputs             - Output configs from embed config
+ * @param {ProviderRegistry} registry
+ * @param {AuthorityGuard} [authorityGuard] - Optional guard; if omitted, Slack posts proceed unchecked
+ */
+export async function dispatchOutputs(snapshot, outputs, registry, authorityGuard, rootDir) {
+  const markdown = renderMarkdown(snapshot);
+  const results = [];
+
+  for (const output of outputs ?? []) {
+    try {
+      switch (output.type) {
+        case 'markdown': {
+          const rel = output.path ?? '.cx/snapshot.md';
+          const filePath = path.isAbsolute(rel) ? rel : path.resolve(rootDir ?? process.cwd(), rel);
+          fs.mkdirSync(path.dirname(filePath), { recursive: true });
+          fs.writeFileSync(filePath, markdown, 'utf8');
+          results.push({ type: 'markdown', path: filePath, status: 'ok' });
+          break;
+        }
+        case 'slack': {
+          const provider = registry?.get(output.provider ?? 'slack');
+          if (!provider) throw new Error(`Slack provider not registered`);
+
+          // Authority check — Slack posts are externalPost actions
+          if (authorityGuard) {
+            const decision = await authorityGuard.check('externalPost', {
+              description: `Snapshot post to Slack channel ${output.channel}`,
+              payload: { channel: output.channel },
+            });
+            if (!decision.allowed) {
+              results.push({
+                type: 'slack',
+                channel: output.channel,
+                status: 'queued',
+                queueId: decision.queueId,
+                reason: decision.reason,
+              });
+              break;
+            }
+          }
+
+          await provider.write({
+            type: 'message',
+            channel: output.channel,
+            text: markdown.slice(0, 3000), // Slack message limit
+          });
+          results.push({ type: 'slack', channel: output.channel, status: 'ok' });
+          break;
+        }
+        case 'log':
+        default: {
+          process.stdout.write(markdown + '\n');
+          results.push({ type: 'log', status: 'ok' });
+          break;
+        }
+      }
+    } catch (err) {
+      results.push({ type: output.type, status: 'error', error: err.message });
+    }
+  }
+
+  return results;
+}

package/lib/embed/providers/github.mjs

@@ -0,0 +1,295 @@
+/**
+ * lib/embed/providers/github.mjs — GitHub provider for embed mode.
+ *
+ * Reads PRs, issues, commits, repo metadata, README, and top-level docs from
+ * one or more repositories using the GitHub REST API. Zero external deps —
+ * uses Node's built-in fetch.
+ *
+ * Supported refs:
+ *   prs        Open pull requests
+ *   issues     Open issues (excludes PRs)
+ *   commits    Recent commits on the default branch
+ *   meta       Repo metadata (name, description, topics, language, visibility)
+ *   readme     README file content (decoded from base64)
+ *   docs       Markdown files in repo root and /docs directory
+ */
+
+const GITHUB_API = 'https://api.github.com';
+
+// Max chars to store per doc file — keeps observations scannable
+const DOC_CONTENT_LIMIT = 8_000;
+
+export class GitHubProvider {
+  #token;
+  #fetchFn;
+
+  constructor({ token, fetchFn = globalThis.fetch } = {}) {
+    if (!token) throw new Error('GitHubProvider requires a token');
+    this.#token = token;
+    this.#fetchFn = fetchFn;
+  }
+
+  /**
+   * Default sources used when no embed.yaml is present.
+   * Returns one source per enrichment group:
+   *   - activity:  prs + issues + commits (live work)
+   *   - context:   meta + readme + docs   (repo understanding)
+   */
+  defaultSources(env = {}) {
+    const raw = env.GITHUB_REPOS ?? env.GITHUB_REPO ?? '';
+    const repos = raw.split(',').map((r) => r.trim()).filter(Boolean);
+    if (!repos.length) return [];
+    return [
+      {
+        provider: 'github',
+        repos,
+        refs: ['prs', 'issues', 'commits'],
+        limit: 25,
+      },
+      {
+        provider: 'github',
+        repos,
+        refs: ['meta', 'readme', 'docs'],
+        limit: 25,
+      },
+    ];
+  }
+
+  /**
+   * @param {string} ref      - 'prs' | 'issues' | 'commits' | 'meta' | 'readme' | 'docs'
+   * @param {object} opts     - source config (repo, repos, branch, limit)
+   * @returns {Promise<Item[]>}
+   */
+  async read(ref, opts = {}) {
+    const repos = resolveRepos(opts);
+    if (!repos.length) {
+      throw new Error(`GitHub source requires "repo" or "repos" field`);
+    }
+
+    const limit = Number(opts.limit ?? 25);
+    const results = [];
+
+    for (const repo of repos) {
+      try {
+        const items = await this.#readRepo(ref, repo, { branch: opts.branch, limit });
+        results.push(...items);
+      } catch (err) {
+        // Surface per-repo errors as items so the snapshot shows them
+        results.push({
+          type: 'error',
+          source: 'github',
+          repo,
+          ref,
+          message: err.message,
+        });
+      }
+    }
+
+    return results;
+  }
+
+  async #readRepo(ref, repo, { branch, limit }) {
+    switch (ref) {
+      case 'prs':      return this.#listPRs(repo, limit);
+      case 'issues':   return this.#listIssues(repo, limit);
+      case 'commits':  return this.#listCommits(repo, { branch, limit });
+      case 'meta':     return this.#fetchMeta(repo);
+      case 'readme':   return this.#fetchReadme(repo);
+      case 'docs':     return this.#fetchDocs(repo);
+      default:         throw new Error(`GitHub provider: unknown ref "${ref}"`);
+    }
+  }
+
+  async #listPRs(repo, limit) {
+    const data = await this.#get(`/repos/${repo}/pulls?state=open&per_page=${limit}&sort=updated&direction=desc`);
+    return data.map((pr) => ({
+      type: 'pr',
+      source: 'github',
+      repo,
+      number: pr.number,
+      title: pr.title,
+      author: pr.user?.login,
+      state: pr.state,
+      draft: pr.draft ?? false,
+      url: pr.html_url,
+      createdAt: pr.created_at,
+      updatedAt: pr.updated_at,
+      labels: (pr.labels ?? []).map((l) => l.name),
+      reviewers: (pr.requested_reviewers ?? []).map((r) => r.login),
+      summary: `PR #${pr.number}: ${pr.title}`,
+    }));
+  }
+
+  async #listIssues(repo, limit) {
+    // GitHub issues endpoint returns PRs too — filter them out
+    const data = await this.#get(`/repos/${repo}/issues?state=open&per_page=${limit}&sort=updated&direction=desc`);
+    return data
+      .filter((issue) => !issue.pull_request)
+      .map((issue) => ({
+        type: 'issue',
+        source: 'github',
+        repo,
+        number: issue.number,
+        title: issue.title,
+        author: issue.user?.login,
+        state: issue.state,
+        url: issue.html_url,
+        createdAt: issue.created_at,
+        updatedAt: issue.updated_at,
+        labels: (issue.labels ?? []).map((l) => l.name),
+        summary: `Issue #${issue.number}: ${issue.title}`,
+      }));
+  }
+
+  async #listCommits(repo, { branch, limit }) {
+    const branchParam = branch ? `&sha=${encodeURIComponent(branch)}` : '';
+    const data = await this.#get(`/repos/${repo}/commits?per_page=${limit}${branchParam}`);
+    return data.map((c) => ({
+      type: 'commit',
+      source: 'github',
+      repo,
+      hash: c.sha,
+      subject: c.commit?.message?.split('\n')[0] ?? '',
+      author: c.commit?.author?.name ?? c.author?.login ?? 'unknown',
+      date: c.commit?.author?.date,
+      url: c.html_url,
+      summary: `${c.sha.slice(0, 7)} ${c.commit?.message?.split('\n')[0] ?? ''}`,
+    }));
+  }
+
+  async #fetchMeta(repo) {
+    const data = await this.#get(`/repos/${repo}`);
+    return [{
+      type: 'meta',
+      source: 'github',
+      repo,
+      name: data.name,
+      fullName: data.full_name,
+      description: data.description ?? '',
+      topics: data.topics ?? [],
+      language: data.language ?? null,
+      visibility: data.visibility ?? (data.private ? 'private' : 'public'),
+      defaultBranch: data.default_branch ?? 'main',
+      url: data.html_url,
+      pushedAt: data.pushed_at,
+      summary: `Repo ${data.full_name}: ${data.description ?? 'no description'}`,
+    }];
+  }
+
+  async #fetchReadme(repo) {
+    let data;
+    try {
+      data = await this.#get(`/repos/${repo}/readme`);
+    } catch (err) {
+      // 404 = no README — not an error worth surfacing
+      if (err.message.includes('404')) return [];
+      throw err;
+    }
+    const content = decodeBase64(data.content ?? '').slice(0, DOC_CONTENT_LIMIT);
+    return [{
+      type: 'doc',
+      source: 'github',
+      repo,
+      path: data.path ?? 'README.md',
+      url: data.html_url ?? `https://github.com/${repo}/blob/HEAD/${data.path ?? 'README.md'}`,
+      content,
+      summary: `README for ${repo}`,
+    }];
+  }
+
+  async #fetchDocs(repo) {
+    const results = [];
+
+    // Collect .md files from repo root and /docs, in parallel
+    const [rootTree, docsTree] = await Promise.all([
+      this.#getTree(repo, ''),
+      this.#getTree(repo, 'docs'),
+    ]);
+
+    const mdFiles = [
+      ...rootTree.filter((f) => f.type === 'file' && /\.md$/i.test(f.path) && !isReadme(f.path)),
+      ...docsTree.filter((f) => f.type === 'file' && /\.md$/i.test(f.path)),
+    ];
+
+    // Fetch files concurrently (cap at 10 to stay well under rate limits)
+    const batch = mdFiles.slice(0, 10);
+    const fetched = await Promise.allSettled(
+      batch.map((f) => this.#fetchFile(repo, f.path))
+    );
+
+    for (let i = 0; i < fetched.length; i++) {
+      const r = fetched[i];
+      if (r.status === 'fulfilled' && r.value) results.push(r.value);
+    }
+
+    return results;
+  }
+
+  async #getTree(repo, dir) {
+    try {
+      const path = dir ? `/repos/${repo}/contents/${dir}` : `/repos/${repo}/contents`;
+      const data = await this.#get(path);
+      return Array.isArray(data) ? data : [];
+    } catch (err) {
+      process.stderr.write('[providers/github.mjs] getTree: ' + (err?.message ?? String(err)) + '\n');
+      return [];
+    }
+  }
+
+  async #fetchFile(repo, path) {
+    try {
+      const data = await this.#get(`/repos/${repo}/contents/${path}`);
+      if (data.encoding !== 'base64' || !data.content) return null;
+      const content = decodeBase64(data.content).slice(0, DOC_CONTENT_LIMIT);
+      return {
+        type: 'doc',
+        source: 'github',
+        repo,
+        path,
+        url: data.html_url ?? `https://github.com/${repo}/blob/HEAD/${path}`,
+        content,
+        summary: `Doc ${path} in ${repo}`,
+      };
+    } catch (err) {
+      process.stderr.write('[providers/github.mjs] fetchFile: ' + (err?.message ?? String(err)) + '\n');
+      return null;
+    }
+  }
+
+  async #get(path) {
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), 10_000);
+    try {
+      const res = await this.#fetchFn(`${GITHUB_API}${path}`, {
+        headers: {
+          Authorization: `Bearer ${this.#token}`,
+          Accept: 'application/vnd.github+json',
+          'X-GitHub-Api-Version': '2022-11-28',
+        },
+        signal: controller.signal,
+      });
+      if (!res.ok) {
+        const body = await res.text().catch(() => '');
+        throw new Error(`GitHub API ${res.status}: ${body.slice(0, 200)}`);
+      }
+      return res.json();
+    } finally {
+      clearTimeout(timer);
+    }
+  }
+}
+
+function resolveRepos(opts) {
+  if (Array.isArray(opts.repos)) return opts.repos.filter(Boolean);
+  if (opts.repo) return [opts.repo];
+  return [];
+}
+
+function decodeBase64(str) {
+  // GitHub base64 includes newlines — strip them first
+  return Buffer.from(str.replace(/\s/g, ''), 'base64').toString('utf8');
+}
+
+function isReadme(path) {
+  return /^readme(\.\w+)?$/i.test(path);
+}