@geraldmaron/construct 1.0.0 → 1.0.2

package/lib/beads-optimistic.mjs

@@ -0,0 +1,365 @@
+/**
+ * lib/beads-optimistic.mjs — Optimistic locking and concurrent read support for beads.
+ *
+ * Addresses the beads lock contention issue by:
+ * 1. Allowing concurrent reads (no lock required)
+ * 2. Using optimistic locking for writes (version-based conflict detection)
+ * 3. Automatic retry with exponential backoff on conflict
+ * 4. Read-replicas for team mode (when available)
+ *
+ * Replaces the exclusive lock model with a more scalable approach
+ * while maintaining consistency guarantees.
+ */
+
+import { spawnSync } from 'node:child_process';
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+// ---------------------------------------------------------------------------
+// Configuration
+// ---------------------------------------------------------------------------
+
+const DEFAULT_RETRY_OPTIONS = {
+  maxRetries: 5,
+  baseDelayMs: 50,
+  maxDelayMs: 2000,
+  backoffMultiplier: 2,
+};
+
+// ---------------------------------------------------------------------------
+// Version tracking for optimistic locking
+// ---------------------------------------------------------------------------
+
+/**
+ * Read the current version of a bead from the database.
+ * In Dolt, this uses the commit hash as a logical version.
+ */
+export async function getBeadVersion(beadId, cwd = process.cwd()) {
+  try {
+    const result = spawnSync(
+      'bd',
+      ['show', beadId, '--json'],
+      { cwd, encoding: 'utf8', timeout: 5000 }
+    );
+    
+    if (result.status !== 0) return null;
+    
+    const data = JSON.parse(result.stdout);
+    // Dolt tracks commit hashes; use as version
+    return data.commitHash || data.version || Date.now().toString();
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Execute a write operation with optimistic locking.
+ * 
+ * @param {Object} options
+ * @param {string} options.beadId - The bead being modified
+ * @param {Function} options.execute - Async function that performs the write
+ * @param {string} options.expectedVersion - Version we expect the bead to have
+ * @param {Object} options.retry - Retry configuration
+ * @returns {Promise<{success: boolean, result: any, attempts: number}>}
+ */
+export async function optimisticWrite({
+  beadId,
+  execute,
+  expectedVersion,
+  retry = DEFAULT_RETRY_OPTIONS,
+  cwd = process.cwd(),
+} = {}) {
+  let attempts = 0;
+  let delay = retry.baseDelayMs;
+  
+  while (attempts < retry.maxRetries) {
+    attempts++;
+    
+    // Verify version hasn't changed (optimistic check)
+    const currentVersion = await getBeadVersion(beadId, cwd);
+    
+    if (expectedVersion && currentVersion !== expectedVersion) {
+      // Conflict detected - another writer modified the bead
+      if (attempts >= retry.maxRetries) {
+        return {
+          success: false,
+          error: `Optimistic lock conflict after ${attempts} attempts. Bead ${beadId} was modified by another process.`,
+          attempts,
+          currentVersion,
+          expectedVersion,
+        };
+      }
+      
+      // Wait with exponential backoff and jitter
+      const jitter = Math.random() * 50;
+      await new Promise(r => setTimeout(r, delay + jitter));
+      delay = Math.min(delay * retry.backoffMultiplier, retry.maxDelayMs);
+      continue;
+    }
+    
+    try {
+      // Execute the write operation
+      const result = await execute();
+      return { success: true, result, attempts };
+    } catch (error) {
+      if (attempts >= retry.maxRetries) {
+        return {
+          success: false,
+          error: error.message,
+          attempts,
+        };
+      }
+      
+      // Retry on transient errors
+      const jitter = Math.random() * 50;
+      await new Promise(r => setTimeout(r, delay + jitter));
+      delay = Math.min(delay * retry.backoffMultiplier, retry.maxDelayMs);
+    }
+  }
+  
+  return {
+    success: false,
+    error: `Max retries (${retry.maxRetries}) exceeded`,
+    attempts,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Concurrent read operations (lock-free)
+// ---------------------------------------------------------------------------
+
+/**
+ * Execute a read operation without acquiring any lock.
+ * Reads are inherently safe to run concurrently.
+ */
+export async function concurrentRead(execute, cwd = process.cwd()) {
+  try {
+    const result = await execute();
+    return { success: true, result };
+  } catch (error) {
+    return {
+      success: false,
+      error: error.message,
+    };
+  }
+}
+
+/**
+ * Batch read multiple beads concurrently.
+ * Much faster than sequential reads when querying multiple items.
+ */
+export async function batchReadBeads(beadIds, cwd = process.cwd()) {
+  const promises = beadIds.map(id => 
+    concurrentRead(async () => {
+      const result = spawnSync(
+        'bd',
+        ['show', id, '--json'],
+        { cwd, encoding: 'utf8', timeout: 5000 }
+      );
+      if (result.status !== 0) throw new Error(`Failed to read ${id}`);
+      return JSON.parse(result.stdout);
+    }, cwd)
+  );
+  
+  const results = await Promise.all(promises);
+  
+  return {
+    success: results.every(r => r.success),
+    results: results.map(r => r.result),
+    errors: results.filter(r => !r.success).map(r => r.error),
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Enhanced beads client with optimistic locking
+// ---------------------------------------------------------------------------
+
+/**
+ * Update a bead with optimistic locking.
+ * Primary implementation for lock-free updates.
+ */
+export async function updateBeadOptimistic(
+  beadId,
+  updates,
+  options = {}
+) {
+  const { actor = 'construct', cwd = process.cwd(), notes } = options;
+  
+  // Get current version before attempting update
+  const currentVersion = await getBeadVersion(beadId, cwd);
+  
+  if (!currentVersion) {
+    return {
+      success: false,
+      error: `Bead ${beadId} not found`,
+    };
+  }
+  
+  return optimisticWrite({
+    beadId,
+    expectedVersion: currentVersion,
+    cwd,
+    execute: async () => {
+      const args = ['update', beadId];
+      
+      if (updates.status) args.push('--status', updates.status);
+      if (updates.assignee) args.push('--assignee', updates.assignee);
+      if (notes) args.push('--notes', notes);
+      if (updates.labels) {
+        for (const label of updates.labels) {
+          args.push('--label', label);
+        }
+      }
+      
+      const result = spawnSync('bd', args, {
+        cwd,
+        encoding: 'utf8',
+        timeout: 10000,
+        env: { ...process.env, BEADS_ACTOR: actor },
+      });
+      
+      if (result.status !== 0) {
+        throw new Error(result.stderr || 'Update failed');
+      }
+      
+      return { beadId, updated: true };
+    },
+  });
+}
+
+/**
+ * Claim a bead with optimistic locking.
+ * Handles the common case of concurrent claim attempts.
+ */
+export async function claimBeadOptimistic(beadId, options = {}) {
+  const { actor = 'construct', cwd = process.cwd() } = options;
+  
+  return updateBeadOptimistic(
+    beadId,
+    { assignee: actor },
+    { actor, cwd, notes: `Claimed by ${actor}` }
+  );
+}
+
+// ---------------------------------------------------------------------------
+// Statistics and monitoring
+// ---------------------------------------------------------------------------
+
+/**
+ * Get statistics about lock contention.
+ * Useful for monitoring and tuning.
+ */
+export async function getContentionStats(cwd = process.cwd()) {
+  const statsPath = path.join(cwd, '.beads', 'contention-stats.json');
+  
+  try {
+    if (fs.existsSync(statsPath)) {
+      return JSON.parse(fs.readFileSync(statsPath, 'utf8'));
+    }
+  } catch {
+    // Ignore read errors
+  }
+  
+  return {
+    totalOperations: 0,
+    conflicts: 0,
+    retries: 0,
+    avgAttempts: 1,
+    lastReset: new Date().toISOString(),
+  };
+}
+
+/**
+ * Record operation statistics.
+ */
+export function recordOperationStats(
+  operation,
+  { success, attempts, conflict = false },
+  cwd = process.cwd()
+) {
+  const statsPath = path.join(cwd, '.beads', 'contention-stats.json');
+  
+  try {
+    const current = fs.existsSync(statsPath)
+      ? JSON.parse(fs.readFileSync(statsPath, 'utf8'))
+      : { totalOperations: 0, conflicts: 0, retries: 0, operations: {} };
+    
+    current.totalOperations++;
+    if (conflict) current.conflicts++;
+    if (attempts > 1) current.retries += (attempts - 1);
+    
+    // Per-operation stats
+    if (!current.operations[operation]) {
+      current.operations[operation] = { count: 0, conflicts: 0, avgAttempts: 1 };
+    }
+    const op = current.operations[operation];
+    op.count++;
+    if (conflict) op.conflicts++;
+    op.avgAttempts = (op.avgAttempts * (op.count - 1) + attempts) / op.count;
+    
+    fs.writeFileSync(statsPath, JSON.stringify(current, null, 2));
+  } catch {
+    // Best effort - don't fail the operation
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Migration path: backwards-compatible wrapper
+// ---------------------------------------------------------------------------
+
+/**
+ * Determine if we should use optimistic locking or fall back to legacy locking.
+ * Uses optimistic locking by default, unless explicitly disabled.
+ */
+export function shouldUseOptimisticLocking(env = process.env) {
+  if (env.CONSTRUCT_BEADS_LEGACY_LOCK === '1') return false;
+  if (env.CONSTRUCT_BEADS_OPTIMISTIC === 'off') return false;
+  return true;
+}
+
+/**
+ * Smart operation router that chooses the best strategy.
+ */
+export async function runBeadsOperation(operation, options = {}) {
+  const {
+    type = 'read', // 'read' | 'write' | 'claim' | 'update'
+    beadId,
+    execute,
+    fallbackToLegacy = true,
+    cwd = process.cwd(),
+  } = options;
+  
+  // Reads never need locking
+  if (type === 'read') {
+    return concurrentRead(execute, cwd);
+  }
+  
+  // Try optimistic locking if enabled
+  if (shouldUseOptimisticLocking() && beadId) {
+    const result = await optimisticWrite({
+      beadId,
+      execute,
+      cwd,
+    });
+    
+    recordOperationStats(operation, result, cwd);
+    
+    if (result.success || !fallbackToLegacy) {
+      return result;
+    }
+    
+    // Fall back to legacy locking on persistent failure
+    console.error(`[beads] Optimistic locking failed, falling back to legacy lock: ${result.error}`);
+  }
+  
+  // Legacy path: use the file-based lock
+  if (fallbackToLegacy) {
+    const { runBd } = await import('./beads-client.mjs');
+    return runBd([operation], { cwd });
+  }
+  
+  return { success: false, error: 'Operation failed and fallback disabled' };
+}

package/lib/bootstrap/built-ins.mjs

@@ -0,0 +1,136 @@
+/**
+ * lib/bootstrap/built-ins.mjs — first-party resource definitions.
+ *
+ * Registers the resources Construct ships with: Node ≥ 18, git, Docker, the
+ * local ONNX embedding model, and Postgres+pgvector. The probes are
+ * deliberately lightweight — they answer "is this thing present and
+ * approximately healthy?" — and never block. The install paths handle the
+ * messier details (Docker container creation, model download).
+ *
+ * Calling `registerBuiltInResources()` is idempotent; re-registration is a
+ * no-op so this file can be imported multiple times.
+ */
+
+import { spawnSync } from 'node:child_process';
+import { existsSync } from 'node:fs';
+import { join } from 'node:path';
+import os from 'node:os';
+import { registerResource, getResource } from './resources.mjs';
+
+function probeCommandVersion(cmd, args = ['--version']) {
+  const r = spawnSync(cmd, args, { encoding: 'utf8', timeout: 3000 });
+  if (r.status !== 0) return { present: false };
+  const text = (r.stdout || r.stderr || '').trim().split('\n')[0];
+  return { present: true, version: text || cmd };
+}
+
+function probeNodeMin18() {
+  const r = spawnSync(process.execPath, ['--version'], { encoding: 'utf8', timeout: 1000 });
+  if (r.status !== 0) return { present: false };
+  const v = (r.stdout || '').trim().replace(/^v/, '');
+  const major = Number(v.split('.')[0]);
+  return {
+    present: major >= 18,
+    version: v,
+    healthy: major >= 18,
+    detail: major < 18 ? `Node ${v} is too old; need ≥ 18` : null,
+  };
+}
+
+async function probePostgresContainer() {
+  const docker = spawnSync('docker', ['ps', '--format', '{{.Names}}\t{{.Image}}'], {
+    encoding: 'utf8', timeout: 5000,
+  });
+  if (docker.status !== 0) return { present: false, detail: 'docker not running' };
+  const lines = docker.stdout.trim().split('\n').filter(Boolean);
+  const constructPg = lines.find((l) => /construct.*postgres|construct-pg|pgvector/.test(l));
+  if (!constructPg) return { present: false, detail: 'construct-postgres container not running' };
+  const [name] = constructPg.split('\t');
+  return {
+    present: true,
+    location: `docker:${name}`,
+    healthy: true,
+    detail: 'managed Postgres+pgvector container',
+  };
+}
+
+async function probeEmbeddingModel() {
+  const cachePath = join(
+    os.homedir(),
+    '.construct', 'cache', 'embeddings',
+    'Xenova', 'all-MiniLM-L6-v2', 'onnx', 'model_quantized.onnx'
+  );
+  if (existsSync(cachePath)) {
+    return {
+      present: true,
+      location: cachePath,
+      healthy: true,
+      version: 'Xenova/all-MiniLM-L6-v2',
+    };
+  }
+  return { present: false, detail: 'will download on first embedding' };
+}
+
+let registered = false;
+
+export function registerBuiltInResources() {
+  if (registered) return;
+  registered = true;
+
+  if (!getResource('node-runtime')) {
+    registerResource({
+      id: 'node-runtime',
+      displayName: 'Node.js 18+',
+      required: true,
+      consentKey: 'BOOTSTRAP_NODE',
+      detect: probeNodeMin18,
+      fallback: () => '(none — Node is required to run Construct)',
+    });
+  }
+
+  if (!getResource('git')) {
+    registerResource({
+      id: 'git',
+      displayName: 'git',
+      required: false,
+      consentKey: 'BOOTSTRAP_GIT',
+      detect: () => probeCommandVersion('git'),
+      fallback: () => 'Construct features that read git history degrade to "no history"',
+    });
+  }
+
+  if (!getResource('docker')) {
+    registerResource({
+      id: 'docker',
+      displayName: 'Docker',
+      required: false,
+      consentKey: 'BOOTSTRAP_DOCKER',
+      detect: () => probeCommandVersion('docker'),
+      fallback: () => 'Construct cannot manage local Postgres or run the binary container',
+    });
+  }
+
+  if (!getResource('embedding-model-local')) {
+    registerResource({
+      id: 'embedding-model-local',
+      displayName: 'Local ONNX embedding model (Xenova/all-MiniLM-L6-v2)',
+      required: false,
+      consentKey: 'BOOTSTRAP_EMBEDDING_MODEL',
+      downloadSize: 50 * 1024 * 1024,
+      detect: probeEmbeddingModel,
+      fallback: () => 'Construct embeddings degrade to hashing-bow-v1 (lower retrieval quality)',
+    });
+  }
+
+  if (!getResource('postgres-pgvector')) {
+    registerResource({
+      id: 'postgres-pgvector',
+      displayName: 'Postgres with pgvector',
+      required: false,
+      consentKey: 'BOOTSTRAP_POSTGRES',
+      downloadSize: 250 * 1024 * 1024,
+      detect: probePostgresContainer,
+      fallback: () => 'Construct retrieval falls back to local JSON vector index (slower, no SQL search)',
+    });
+  }
+}

package/lib/bootstrap/lazy-install.mjs

@@ -0,0 +1,161 @@
+/**
+ * lib/bootstrap/lazy-install.mjs — on-demand resource installer with consent gating.
+ *
+ * When a runtime code path needs a resource that is not present, call
+ * `lazyInstall(id)`. The function:
+ *
+ *   1. Reads cached operator consent from config.env (consent key set by the
+ *      resource definition in `lib/bootstrap/resources.mjs`).
+ *   2. If consent is 'never', returns { installed: false, fallback: <text> }
+ *      without prompting or printing anything.
+ *   3. If a TTY is available, prompts the user with the resource displayName,
+ *      estimated download size, and [y/n/never].
+ *   4. On 'y' or an existing 'yes' consent: calls resource.install(), waits
+ *      for it to complete, updates config.env, and returns { installed: true }.
+ *   5. On 'n': returns { installed: false, fallback: <text> } without saving.
+ *   6. On 'never': writes 'never' to config.env and returns { installed: false }.
+ *
+ * Hooks that run without a TTY (most hook invocations) skip to step 2 and
+ * use the fallback silently.
+ *
+ * Consent is cached per-resource in config.env so the prompt never repeats
+ * across process restarts once the user has answered.
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+import os from 'node:os';
+import readline from 'node:readline';
+
+import { getResource, probeResource } from './resources.mjs';
+
+const CONFIG_ENV = path.join(os.homedir(), '.construct', 'config.env');
+
+function readConfigEnv() {
+  try {
+    return fs.readFileSync(CONFIG_ENV, 'utf8');
+  } catch {
+    return '';
+  }
+}
+
+function readConsentFromEnv(consentKey) {
+  const lines = readConfigEnv().split('\n');
+  for (const line of lines) {
+    const trimmed = line.trim();
+    if (!trimmed || trimmed.startsWith('#')) continue;
+    const eq = trimmed.indexOf('=');
+    if (eq === -1) continue;
+    const k = trimmed.slice(0, eq).trim();
+    const v = trimmed.slice(eq + 1).trim().replace(/^['"]|['"]$/g, '');
+    if (k === consentKey) return v;
+  }
+  return null;
+}
+
+function writeConsentToEnv(consentKey, value) {
+  let content = readConfigEnv();
+  const regex = new RegExp(`^${consentKey}=.*$`, 'm');
+  const line = `${consentKey}=${value}`;
+  if (regex.test(content)) {
+    content = content.replace(regex, line);
+  } else {
+    content = content.trimEnd() + (content ? '\n' : '') + line + '\n';
+  }
+  fs.mkdirSync(path.dirname(CONFIG_ENV), { recursive: true });
+  fs.writeFileSync(CONFIG_ENV, content, { mode: 0o600 });
+}
+
+function hasTty() {
+  return Boolean(process.stdin.isTTY && process.stderr.isTTY);
+}
+
+function formatBytes(bytes) {
+  if (!bytes) return '';
+  if (bytes < 1024 * 1024) return `${Math.round(bytes / 1024)} KB`;
+  return `${Math.round(bytes / (1024 * 1024))} MB`;
+}
+
+async function promptUser(question) {
+  return new Promise((resolve) => {
+    const rl = readline.createInterface({ input: process.stdin, output: process.stderr });
+    rl.question(question, (answer) => {
+      rl.close();
+      resolve(answer.trim().toLowerCase());
+    });
+  });
+}
+
+/**
+ * Attempt to install a resource lazily, respecting cached consent.
+ *
+ * @param {string} id - resource id from the registry
+ * @param {object} [opts]
+ * @param {boolean} [opts.silent] - skip TTY prompt, use cached consent only
+ * @returns {{ installed: boolean, fallback?: string, error?: string }}
+ */
+export async function lazyInstall(id, { silent = false } = {}) {
+  const resource = getResource(id);
+  if (!resource) {
+    return { installed: false, error: `unknown resource: ${id}` };
+  }
+
+  const probe = await probeResource(resource);
+  if (probe.present && probe.healthy) {
+    return { installed: true };
+  }
+
+  const { consentKey, displayName, downloadSize, fallback } = resource;
+  const fallbackMsg = typeof fallback === 'function' ? fallback() : (fallback || '');
+
+  const cached = readConsentFromEnv(consentKey);
+
+  if (cached === 'never') {
+    return { installed: false, fallback: fallbackMsg };
+  }
+
+  if (cached === 'yes') {
+    return await performInstall(resource, fallbackMsg);
+  }
+
+  if (silent || !hasTty()) {
+    return { installed: false, fallback: fallbackMsg };
+  }
+
+  const size = downloadSize ? ` (~${formatBytes(downloadSize)})` : '';
+  const answer = await promptUser(
+    `\n[construct] ${displayName} required${size}. Install now? [y/n/never] `
+  );
+
+  if (answer === 'y' || answer === 'yes') {
+    writeConsentToEnv(consentKey, 'yes');
+    return await performInstall(resource, fallbackMsg);
+  }
+
+  if (answer === 'never') {
+    writeConsentToEnv(consentKey, 'never');
+    process.stderr.write(
+      `[construct] "${displayName}" will not be prompted again. Using fallback: ${fallbackMsg}\n`
+    );
+    return { installed: false, fallback: fallbackMsg };
+  }
+
+  return { installed: false, fallback: fallbackMsg };
+}
+
+async function performInstall(resource, fallbackMsg) {
+  if (typeof resource.install !== 'function') {
+    return { installed: false, fallback: fallbackMsg, error: 'resource has no install()' };
+  }
+  try {
+    process.stderr.write(`[construct] Installing ${resource.displayName}...\n`);
+    const result = await resource.install();
+    if (result?.success) {
+      process.stderr.write(`[construct] ${resource.displayName} installed.\n`);
+      return { installed: true };
+    }
+    return { installed: false, fallback: fallbackMsg, error: result?.log };
+  } catch (err) {
+    return { installed: false, fallback: fallbackMsg, error: err?.message || String(err) };
+  }
+}