envseed 0.2.1 → 0.3.0

package/lib/container-replicator.mjs

@@ -16,6 +16,7 @@ import path from 'node:path';
 import { execSync, spawn } from 'node:child_process';
 import { DATA_DIR, INSTALL_DIR } from './utils.mjs';
 import { launchRedactionReview } from './redaction-review.mjs';
+import { querySource } from './source-query-client.mjs';
 
 const REPLICAS_DIR = path.join(DATA_DIR, 'replicas');
 const LOCK_DIR = path.join(DATA_DIR, '.locks');
@@ -440,6 +441,25 @@ If something is fundamentally broken, document it in env-status.json and do your
     envStatus = JSON.parse(fs.readFileSync(envStatusPath, 'utf8'));
   } catch {}
 
+  // If running remotely and Opus reported missing files, fetch them via source queries
+  if (process.env.ENVSEED_REMOTE === '1' && envStatus.missingFiles?.length > 0) {
+    const incidentId = process.env.ENVSEED_INCIDENT_ID;
+    const apiEndpoint = process.env.ENVSEED_API_ENDPOINT;
+    const apiKey = loadConfig()?.apiKey;
+    if (incidentId && apiEndpoint) {
+      for (const filePath of envStatus.missingFiles) {
+        try {
+          const result = await querySource(incidentId, filePath, apiEndpoint, apiKey || '');
+          if (result.success && result.data) {
+            const destPath = path.join(replicaOutputDir, '..', 'workspace', filePath);
+            fs.mkdirSync(path.dirname(destPath), { recursive: true });
+            fs.writeFileSync(destPath, result.data);
+          }
+        } catch {}
+      }
+    }
+  }
+
   // If Opus generated a Dockerfile, try building it
   if (fs.existsSync(generatedDockerfile)) {
     try {

package/lib/log-incident.mjs

@@ -295,24 +295,53 @@ async function main() {
     log('  (incident saved locally, upload can be retried with: envseed incident <id> upload)');
   }
 
-  // 6. Spawn simulation orchestrator
+  // 6. Start simulations (remote or local)
   log('5/5 Starting simulations...');
-  const config = {};
+  // Re-read config (already loaded at top for credential check, but use fresh copy)
+  const simConfig = {};
   try {
-    Object.assign(config, JSON.parse(fs.readFileSync(path.join(INSTALL_DIR, 'config.json'), 'utf8')));
+    Object.assign(simConfig, JSON.parse(fs.readFileSync(path.join(INSTALL_DIR, 'config.json'), 'utf8')));
   } catch {}
 
-  if (config.enableSimulations === false) {
+  const libDir = path.dirname(new URL(import.meta.url).pathname);
+
+  if (simConfig.enableSimulations === false) {
     log('  Simulations disabled in config');
+  } else if (simConfig.remoteReplication && s3Result.success) {
+    // Remote mode: POST /reconstruct to launch EC2 worker, spawn polling agent
+    log('  Triggering remote replication...');
+    try {
+      const reconstructRes = await httpPost(
+        `${simConfig.uploadEndpoint}/reconstruct/${incidentId}`,
+        { apiEndpoint: simConfig.uploadEndpoint },
+        simConfig.apiKey,
+      );
+      if (reconstructRes.success) {
+        log(`  Worker launched: instance ${reconstructRes.body?.instanceId || 'pending'}`);
+        log(`  Job ID: ${reconstructRes.body?.jobId || 'unknown'}`);
+
+        // Spawn polling agent to serve source queries
+        const pollingScript = path.join(libDir, 'polling-agent.mjs');
+        const pollingChild = spawn('node', [
+          pollingScript, incidentId, cwd, simConfig.uploadEndpoint, simConfig.apiKey,
+        ], { detached: true, stdio: 'ignore' });
+        pollingChild.unref();
+        log(`  Polling agent spawned (PID ${pollingChild.pid})`);
+      } else {
+        log(`  Remote replication failed: ${reconstructRes.error || 'unknown error'}`);
+        log('  Falling back to local simulations...');
+        spawnLocalOrchestrator(libDir, incidentId, log);
+      }
+    } catch (err) {
+      log(`  Remote replication error: ${err.message}`);
+      log('  Falling back to local simulations...');
+      spawnLocalOrchestrator(libDir, incidentId, log);
+    }
   } else {
-    const orchestratorScript = path.join(path.dirname(new URL(import.meta.url).pathname), 'simulation-orchestrator.mjs');
-    const child = spawn('node', [orchestratorScript, incidentId], {
-      detached: true,
-      stdio: 'ignore',
-    });
-    child.unref();
-    log(`  Orchestrator spawned (PID ${child.pid})`);
-    log(`  Check progress: envseed incident ${incidentId}`);
+    if (simConfig.remoteReplication && !s3Result.success) {
+      log('  Remote replication requires S3 upload — falling back to local');
+    }
+    spawnLocalOrchestrator(libDir, incidentId, log);
   }
 
   // Write initial status
@@ -320,8 +349,9 @@ async function main() {
     started: new Date().toISOString(),
     archivalComplete: true,
     s3Uploaded: s3Result.success,
-    simulationsStarted: config.enableSimulations !== false,
-    totalPlanned: config.simulationCount || 2,
+    remote: !!(simConfig.remoteReplication && s3Result.success),
+    simulationsStarted: simConfig.enableSimulations !== false,
+    totalPlanned: simConfig.simulationCount || 2,
     completed: 0,
     failed: 0,
     running: 0,
@@ -333,6 +363,50 @@ async function main() {
   if (s3Result.success) log(`S3 path: ${s3Result.s3Path}`);
 }
 
+function spawnLocalOrchestrator(libDir, incidentId, log) {
+  const orchestratorScript = path.join(libDir, 'simulation-orchestrator.mjs');
+  const child = spawn('node', [orchestratorScript, incidentId], {
+    detached: true,
+    stdio: 'ignore',
+  });
+  child.unref();
+  log(`  Local orchestrator spawned (PID ${child.pid})`);
+  log(`  Check progress: envseed incident ${incidentId}`);
+}
+
+async function httpPost(url, body, apiKey) {
+  const https = await import('node:https');
+  const { URL } = await import('node:url');
+  return new Promise((resolve, reject) => {
+    const parsed = new URL(url);
+    const bodyStr = JSON.stringify(body);
+    const req = https.request({
+      method: 'POST',
+      hostname: parsed.hostname,
+      path: parsed.pathname,
+      headers: {
+        'Content-Type': 'application/json',
+        'Content-Length': Buffer.byteLength(bodyStr),
+        'x-api-key': apiKey,
+      },
+    }, (res) => {
+      let data = '';
+      res.on('data', (chunk) => { data += chunk; });
+      res.on('end', () => {
+        try {
+          const parsed = JSON.parse(data);
+          resolve({ success: res.statusCode === 200, body: parsed, error: parsed.error });
+        } catch {
+          resolve({ success: false, error: data });
+        }
+      });
+    });
+    req.on('error', reject);
+    req.write(bodyStr);
+    req.end();
+  });
+}
+
 main().catch(err => {
   process.stderr.write(`log-incident error: ${err.message}\n${err.stack}\n`);
   process.exit(1);

package/lib/polling-agent.mjs

@@ -0,0 +1,249 @@
+#!/usr/bin/env node
+
+/**
+ * Polling agent — runs on the user's machine during remote replication.
+ * Polls the API for source-query requests from the remote worker,
+ * executes them locally (read_file, list_dir, glob), and replies.
+ *
+ * Spawned as detached process by log-incident.mjs.
+ *
+ * Usage: node polling-agent.mjs <incidentId> <projectDir> <apiEndpoint> <apiKey>
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+import https from 'node:https';
+import { URL } from 'node:url';
+import { globSync } from 'node:fs';
+
+const POLL_INTERVAL_MS = 3000;
+const STATUS_CHECK_INTERVAL_MS = 30000;
+const TIMEOUT_MS = 30 * 60 * 1000; // 30 minutes
+const IDLE_TIMEOUT_MS = 10 * 60 * 1000; // 10 min with no queries after replication starts
+const MAX_FILE_SIZE = 512 * 1024; // 512KB per file
+
+const [,, incidentId, projectDir, apiEndpoint, apiKey] = process.argv;
+
+if (!incidentId || !projectDir || !apiEndpoint || !apiKey) {
+  process.stderr.write('Usage: node polling-agent.mjs <incidentId> <projectDir> <apiEndpoint> <apiKey>\n');
+  process.exit(1);
+}
+
+const log = (msg) => process.stderr.write(`[polling-agent] ${msg}\n`);
+
+// ── Security: path validation ──
+
+function isWithinProject(filePath) {
+  const resolved = path.resolve(projectDir, filePath);
+  const realProjectDir = fs.realpathSync(projectDir);
+  try {
+    const realResolved = fs.realpathSync(resolved);
+    return realResolved.startsWith(realProjectDir + path.sep) || realResolved === realProjectDir;
+  } catch {
+    // File doesn't exist yet — check the resolved path without symlink resolution
+    return resolved.startsWith(realProjectDir + path.sep) || resolved === realProjectDir;
+  }
+}
+
+// ── Query execution ──
+
+function executeQuery(query) {
+  const { queryType, path: queryPath, args } = query;
+
+  // All paths must be within project directory
+  if (!isWithinProject(queryPath)) {
+    return { success: false, error: `Path outside project directory: ${queryPath}` };
+  }
+
+  const fullPath = path.resolve(projectDir, queryPath);
+
+  try {
+    switch (queryType) {
+      case 'read_file': {
+        if (!fs.existsSync(fullPath)) {
+          return { success: false, error: `File not found: ${queryPath}` };
+        }
+        const stat = fs.statSync(fullPath);
+        if (!stat.isFile()) {
+          return { success: false, error: `Not a file: ${queryPath}` };
+        }
+        if (stat.size > MAX_FILE_SIZE) {
+          return { success: false, error: `File too large (${stat.size} bytes, max ${MAX_FILE_SIZE}): ${queryPath}` };
+        }
+        const content = fs.readFileSync(fullPath, 'utf8');
+        return { success: true, data: content };
+      }
+
+      case 'list_dir': {
+        if (!fs.existsSync(fullPath)) {
+          return { success: false, error: `Directory not found: ${queryPath}` };
+        }
+        const entries = fs.readdirSync(fullPath, { withFileTypes: true });
+        const listing = entries.map(e => ({
+          name: e.name,
+          type: e.isDirectory() ? 'dir' : e.isFile() ? 'file' : 'other',
+        }));
+        return { success: true, data: JSON.stringify(listing) };
+      }
+
+      case 'glob': {
+        const pattern = args?.pattern || queryPath;
+        // Use Node's glob — restrict to project dir
+        const { globSync } = require('node:fs');
+        try {
+          const matches = globSync(pattern, { cwd: projectDir }).slice(0, 500);
+          return { success: true, data: JSON.stringify(matches) };
+        } catch {
+          // Fallback for older Node without fs.globSync
+          return { success: false, error: 'Glob not supported on this Node version' };
+        }
+      }
+
+      default:
+        return { success: false, error: `Unknown query type: ${queryType}` };
+    }
+  } catch (err) {
+    return { success: false, error: err.message };
+  }
+}
+
+// ── HTTP helpers ──
+
+function httpRequest(urlStr, options = {}) {
+  return new Promise((resolve, reject) => {
+    const url = new URL(urlStr);
+    const reqOptions = {
+      method: options.method || 'GET',
+      hostname: url.hostname,
+      path: url.pathname + url.search,
+      headers: {
+        'x-api-key': apiKey,
+        'Content-Type': 'application/json',
+        ...options.headers,
+      },
+    };
+
+    if (options.body) {
+      reqOptions.headers['Content-Length'] = Buffer.byteLength(options.body);
+    }
+
+    const req = https.request(reqOptions, (res) => {
+      let data = '';
+      res.on('data', (chunk) => { data += chunk; });
+      res.on('end', () => {
+        try {
+          resolve({ statusCode: res.statusCode, body: JSON.parse(data) });
+        } catch {
+          resolve({ statusCode: res.statusCode, body: data });
+        }
+      });
+    });
+    req.on('error', reject);
+    if (options.body) req.write(options.body);
+    req.end();
+  });
+}
+
+async function pollForQueries() {
+  try {
+    const res = await httpRequest(`${apiEndpoint}/source-query/${incidentId}/poll`);
+    if (res.statusCode === 200 && res.body?.queries?.length > 0) {
+      return res.body.queries;
+    }
+  } catch (err) {
+    log(`Poll error: ${err.message}`);
+  }
+  return [];
+}
+
+async function replyToQuery(queryId, result) {
+  try {
+    await httpRequest(`${apiEndpoint}/source-query/${incidentId}/reply`, {
+      method: 'POST',
+      body: JSON.stringify({
+        queryId,
+        success: result.success,
+        data: result.data || '',
+        error: result.error || '',
+      }),
+    });
+  } catch (err) {
+    log(`Reply error for ${queryId}: ${err.message}`);
+  }
+}
+
+async function checkReconstructionStatus() {
+  try {
+    const res = await httpRequest(`${apiEndpoint}/reconstruct/${incidentId}`);
+    if (res.statusCode === 200) {
+      const status = res.body?.status;
+      return status === 'completed' || status === 'failed' || status === 'terminated';
+    }
+  } catch {
+    // ignore — we'll keep polling
+  }
+  return false;
+}
+
+// ── Main loop ──
+
+async function main() {
+  log(`Starting for incident ${incidentId}`);
+  log(`Project dir: ${projectDir}`);
+  log(`API endpoint: ${apiEndpoint}`);
+
+  const startTime = Date.now();
+  let lastQueryTime = Date.now();
+  let totalQueries = 0;
+  let reconstructionStarted = false;
+
+  while (true) {
+    const elapsed = Date.now() - startTime;
+
+    // Hard timeout
+    if (elapsed > TIMEOUT_MS) {
+      log(`Timeout reached (${Math.round(TIMEOUT_MS / 60000)} min), exiting`);
+      break;
+    }
+
+    // Poll for queries
+    const queries = await pollForQueries();
+
+    if (queries.length > 0) {
+      lastQueryTime = Date.now();
+      reconstructionStarted = true;
+
+      for (const query of queries) {
+        log(`Query ${query.queryId}: ${query.queryType} ${query.path}`);
+        const result = executeQuery(query);
+        log(`  → ${result.success ? 'OK' : 'FAIL'}: ${result.success ? `${(result.data || '').length} bytes` : result.error}`);
+        await replyToQuery(query.queryId, result);
+        totalQueries++;
+      }
+    }
+
+    // Check if reconstruction is done (less frequently)
+    if (elapsed > 0 && elapsed % STATUS_CHECK_INTERVAL_MS < POLL_INTERVAL_MS) {
+      const done = await checkReconstructionStatus();
+      if (done) {
+        log(`Reconstruction finished, exiting (served ${totalQueries} queries)`);
+        break;
+      }
+    }
+
+    // Idle timeout — only after reconstruction has started
+    if (reconstructionStarted && (Date.now() - lastQueryTime > IDLE_TIMEOUT_MS)) {
+      log(`No queries for ${Math.round(IDLE_TIMEOUT_MS / 60000)} min, exiting`);
+      break;
+    }
+
+    await new Promise(resolve => setTimeout(resolve, POLL_INTERVAL_MS));
+  }
+
+  log(`Done. Served ${totalQueries} source queries.`);
+}
+
+main().catch(err => {
+  log(`Fatal error: ${err.message}`);
+  process.exit(1);
+});

package/lib/source-query-client.mjs

@@ -0,0 +1,99 @@
+/**
+ * Source query client — used by the remote worker to request files
+ * from the user's machine via the polling agent pattern.
+ *
+ * Flow: worker POSTs query → Lambda stores in DynamoDB → polling agent
+ * on user's machine picks it up, reads file, POSTs reply → worker polls
+ * for result.
+ */
+
+import https from 'node:https';
+import { URL } from 'node:url';
+
+const POLL_INTERVAL_MS = 2000;
+const POLL_TIMEOUT_MS = 60000; // 1 minute max wait per query
+
+function httpRequest(urlStr, options = {}) {
+  return new Promise((resolve, reject) => {
+    const url = new URL(urlStr);
+    const reqOptions = {
+      method: options.method || 'GET',
+      hostname: url.hostname,
+      path: url.pathname + url.search,
+      headers: {
+        'Content-Type': 'application/json',
+        ...options.headers,
+      },
+    };
+    if (options.body) {
+      reqOptions.headers['Content-Length'] = Buffer.byteLength(options.body);
+    }
+    const req = https.request(reqOptions, (res) => {
+      let data = '';
+      res.on('data', (chunk) => { data += chunk; });
+      res.on('end', () => {
+        try {
+          resolve({ statusCode: res.statusCode, body: JSON.parse(data) });
+        } catch {
+          resolve({ statusCode: res.statusCode, body: data });
+        }
+      });
+    });
+    req.on('error', reject);
+    if (options.body) req.write(options.body);
+    req.end();
+  });
+}
+
+/**
+ * Request a file from the user's machine via the source query system.
+ *
+ * @param {string} incidentId
+ * @param {string} filePath - relative path within the project
+ * @param {string} apiEndpoint - API Gateway URL
+ * @param {string} apiKey
+ * @param {object} options
+ * @param {string} options.queryType - 'read_file' | 'list_dir' | 'glob'
+ * @param {object} options.args - additional args (e.g. { pattern } for glob)
+ * @returns {Promise<{success: boolean, data?: string, error?: string}>}
+ */
+export async function querySource(incidentId, filePath, apiEndpoint, apiKey, options = {}) {
+  const { queryType = 'read_file', args = {} } = options;
+
+  // 1. Post the query
+  const postRes = await httpRequest(`${apiEndpoint}/source-query/${incidentId}`, {
+    method: 'POST',
+    headers: { 'x-api-key': apiKey },
+    body: JSON.stringify({ queryType, path: filePath, args }),
+  });
+
+  if (postRes.statusCode !== 200 || !postRes.body?.queryId) {
+    return { success: false, error: `Failed to post query: ${postRes.body?.error || postRes.statusCode}` };
+  }
+
+  const queryId = postRes.body.queryId;
+
+  // 2. Poll for result
+  const startTime = Date.now();
+  while (Date.now() - startTime < POLL_TIMEOUT_MS) {
+    await new Promise(resolve => setTimeout(resolve, POLL_INTERVAL_MS));
+
+    const resultRes = await httpRequest(
+      `${apiEndpoint}/source-query/${incidentId}/result/${queryId}`,
+      { headers: { 'x-api-key': apiKey } },
+    );
+
+    if (resultRes.statusCode === 200) {
+      const { status, data, error } = resultRes.body;
+      if (status === 'completed') {
+        return { success: true, data };
+      }
+      if (status === 'failed') {
+        return { success: false, error: error || 'Query failed' };
+      }
+      // status === 'pending' — keep polling
+    }
+  }
+
+  return { success: false, error: `Query timed out after ${POLL_TIMEOUT_MS / 1000}s` };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envseed",
-  "version": "0.2.1",
+  "version": "0.3.0",
   "description": "Cultivate AI safety evals from real Claude Code sessions",
   "type": "module",
   "bin": {

package/postinstall.mjs

@@ -34,6 +34,7 @@ const DEFAULT_CONFIG = {
   simulationModels: ['claude-haiku-4-5-20251001'],
   enableSimulations: true,
   replicaModel: 'claude-opus-4-6',
+  remoteReplication: false,
   redactionReview: true,
   proxyUrl: '',
   proxyToken: '',