@engineeros/connector 0.1.0

package/README.md

@@ -0,0 +1,25 @@
+# EngineerOS Connector
+
+The connector keeps an outbound WebSocket open from a repository-owning machine to EngineerOS and executes assigned Goals with Codex CLI.
+
+## Requirements
+
+- Node.js 22 or newer
+- Git
+- Codex CLI installed and authenticated (`codex login`)
+
+## Pair once
+
+Open the repository, choose **Connect a machine** in the EngineerOS Goal screen, and run the generated `npx @engineeros/connector pair ...` command. The one-time code expires after 10 minutes. A workspace-scoped connector token is stored below `~/.engineeros/connectors` with owner-only permissions where the operating system supports them.
+
+```shell
+npx --yes @engineeros/connector@latest pair PAIRING-CODE --url https://engineeros.example.com --workspace /path/to/repository
+```
+
+## Reconnect
+
+```shell
+npx --yes @engineeros/connector@latest start --workspace /path/to/repository
+```
+
+Each Goal runs in an isolated worktree below `~/.engineeros/runs`. Cancellation stops Codex. The connector returns the changed paths, bounded diff, and repository ZIP to EngineerOS; a human still performs independent attestation.

package/bin/engineeros-connector.mjs

@@ -0,0 +1,287 @@
+#!/usr/bin/env node
+import os from "node:os";
+import path from "node:path";
+import {
+  loadConfig,
+  resultUrl,
+  saveConfig,
+  socketUrl,
+} from "../src/config.mjs";
+import { executeAssignment, stopProcess } from "../src/runner.mjs";
+
+const { command, positional, flags } = parseArgs(process.argv.slice(2));
+
+if (command === "status") {
+  const config = await loadConfig(flags.workspace || process.cwd());
+  console.log(
+    config
+      ? `Paired as ${config.name} (${config.connector_id}) for ${config.workspace}`
+      : "Not paired",
+  );
+  process.exit(config ? 0 : 1);
+}
+
+let config;
+let firstMessage;
+if (command === "pair") {
+  const pairingCode = positional[0];
+  if (!pairingCode)
+    fail(
+      "Usage: engineeros-connector pair CODE --url URL [--name NAME] [--workspace PATH]",
+    );
+  const url = flags.url;
+  if (!url) fail("Pairing requires --url with the EngineerOS backend address.");
+  config = {
+    server_url: socketUrl(url),
+    workspace: path.resolve(flags.workspace || process.cwd()),
+    name:
+      flags.name ||
+      `${os.hostname()} - ${path.basename(path.resolve(flags.workspace || process.cwd()))}`,
+  };
+  firstMessage = {
+    type: "pair",
+    pairing_code: pairingCode,
+    name: config.name,
+    capabilities: {
+      codex_cli: true,
+      platform: process.platform,
+      workspace_name: path.basename(config.workspace),
+    },
+  };
+} else if (command === "start") {
+  config = await loadConfig(flags.workspace || process.cwd());
+  if (!config)
+    fail(
+      "This connector is not paired. Create a pairing command in EngineerOS first.",
+    );
+  firstMessage = {
+    type: "authenticate",
+    connector_id: config.connector_id,
+    token: config.token,
+  };
+} else {
+  fail("Use `engineeros-connector pair`, `start`, or `status`.");
+}
+
+let stopped = false;
+let active = null;
+const available = [];
+let socket;
+let pingTimer;
+let reconnectDelay = 1_000;
+let connectionRejected = false;
+
+process.on("SIGINT", async () => {
+  stopped = true;
+  await stopProcess(active?.child);
+  socket?.close();
+  process.exit(0);
+});
+
+await connect();
+
+async function connect() {
+  console.log(`Connecting ${config.name} to ${config.server_url}`);
+  connectionRejected = false;
+  socket = new WebSocket(config.server_url);
+  socket.addEventListener("open", () =>
+    socket.send(JSON.stringify(firstMessage)),
+  );
+  socket.addEventListener("message", async (event) => {
+    const message = JSON.parse(String(event.data));
+    if (message.type === "paired") {
+      config = {
+        ...config,
+        connector_id: message.connector.id,
+        token: message.token,
+      };
+      await saveConfig(config);
+      firstMessage = {
+        type: "authenticate",
+        connector_id: config.connector_id,
+        token: config.token,
+      };
+      console.log(`Paired. Connector ${config.connector_id} is online.`);
+      reconnectDelay = 1_000;
+      startPings();
+      return;
+    }
+    if (message.type === "authenticated") {
+      console.log("Connected and waiting for EngineerOS runs.");
+      reconnectDelay = 1_000;
+      startPings();
+      return;
+    }
+    if (message.type === "run.available") {
+      if (!available.includes(message.run_id)) available.push(message.run_id);
+      pump();
+      return;
+    }
+    if (message.type === "run.assignment") {
+      await execute(message);
+      return;
+    }
+    if (message.type === "run.cancelled" && active?.runId === message.run_id) {
+      console.log(`Run ${message.run_id} cancelled by EngineerOS.`);
+      active.cancelled = true;
+      await stopProcess(active.child);
+      return;
+    }
+    if (message.type === "connector.revoked") {
+      stopped = true;
+      console.error("This connector was revoked in EngineerOS.");
+      socket.close();
+      return;
+    }
+    if (message.type === "run.error") {
+      console.error(`EngineerOS: ${message.message}`);
+      if (active?.runId === message.run_id && !active.child) {
+        active = null;
+        pump();
+      }
+    }
+    if (message.type === "connection.error") {
+      connectionRejected = true;
+      console.error(`EngineerOS: ${message.message}`);
+    }
+  });
+  socket.addEventListener("close", () => {
+    clearInterval(pingTimer);
+    if (connectionRejected) {
+      stopped = true;
+      console.error(
+        "Connection rejected. Create a new pairing command in EngineerOS if this connector was revoked.",
+      );
+      return;
+    }
+    if (stopped) return;
+    console.error(
+      `Connection lost. Retrying in ${Math.round(reconnectDelay / 1_000)}s.`,
+    );
+    setTimeout(connect, reconnectDelay);
+    reconnectDelay = Math.min(30_000, reconnectDelay * 2);
+  });
+  socket.addEventListener("error", () => {});
+}
+
+function startPings() {
+  clearInterval(pingTimer);
+  pingTimer = setInterval(() => {
+    if (socket.readyState === WebSocket.OPEN) {
+      socket.send(
+        JSON.stringify({
+          type: "ping",
+          active_run_id: active?.runId ?? null,
+        }),
+      );
+    }
+  }, 10_000);
+}
+
+function pump() {
+  if (active || socket.readyState !== WebSocket.OPEN) return;
+  const runId = available.shift();
+  if (!runId) return;
+  active = { runId, child: null, cancelled: false };
+  socket.send(
+    JSON.stringify({
+      type: "run.claim",
+      run_id: runId,
+      runner_name: "Codex CLI",
+      metadata: {
+        hostname: os.hostname(),
+        workspace_name: path.basename(config.workspace),
+      },
+    }),
+  );
+}
+
+async function execute(assignment) {
+  const runId = assignment.run_id;
+  console.log(`Running Goal ${runId} with Codex CLI.`);
+  let progress = 10;
+  const heartbeat = setInterval(() => {
+    if (socket.readyState === WebSocket.OPEN && active?.runId === runId) {
+      progress = Math.min(90, progress + 5);
+      socket.send(
+        JSON.stringify({
+          type: "run.progress",
+          run_id: runId,
+          progress_percent: progress,
+          message: "Codex is working",
+        }),
+      );
+    }
+  }, 15_000);
+  try {
+    const result = await executeAssignment(assignment, config, {
+      onProcess: (child) => {
+        if (active?.runId === runId) active.child = child;
+      },
+      onEvent: (event) => {
+        const message =
+          event.message || event.item?.text || event.type || "Codex is working";
+        if (socket.readyState === WebSocket.OPEN) {
+          socket.send(
+            JSON.stringify({
+              type: "run.progress",
+              run_id: runId,
+              progress_percent: progress,
+              message: String(message).slice(0, 500),
+            }),
+          );
+        }
+      },
+    });
+    if (active?.cancelled) return;
+    const response = await fetch(
+      resultUrl(config.server_url, config.connector_id, runId),
+      {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+          Authorization: `Bearer ${config.token}`,
+        },
+        body: JSON.stringify(result),
+      },
+    );
+    if (!response.ok)
+      throw new Error(
+        `EngineerOS rejected the result (${response.status}): ${await response.text()}`,
+      );
+    console.log(`Run submitted. Review workspace: ${result.run_workspace}`);
+  } catch (error) {
+    if (!active?.cancelled && socket.readyState === WebSocket.OPEN) {
+      socket.send(
+        JSON.stringify({
+          type: "run.failed",
+          run_id: runId,
+          message: error instanceof Error ? error.message : String(error),
+        }),
+      );
+    }
+    if (!active?.cancelled)
+      console.error(error instanceof Error ? error.message : String(error));
+  } finally {
+    clearInterval(heartbeat);
+    active = null;
+    pump();
+  }
+}
+
+function parseArgs(args) {
+  const command = args.shift();
+  const positional = [];
+  const flags = {};
+  while (args.length) {
+    const value = args.shift();
+    if (!value.startsWith("--")) positional.push(value);
+    else flags[value.slice(2)] = args.shift();
+  }
+  return { command, positional, flags };
+}
+
+function fail(message) {
+  console.error(message);
+  process.exit(1);
+}

package/package.json

@@ -0,0 +1,35 @@
+{
+  "name": "@engineeros/connector",
+  "version": "0.1.0",
+  "description": "Connect a local Codex CLI workspace to EngineerOS over an outbound WebSocket.",
+  "private": false,
+  "type": "module",
+  "license": "UNLICENSED",
+  "files": [
+    "bin",
+    "src",
+    "README.md"
+  ],
+  "bin": {
+    "engineeros-connector": "bin/engineeros-connector.mjs"
+  },
+  "scripts": {
+    "start": "node ./bin/engineeros-connector.mjs",
+    "test": "node --test",
+    "type-check": "node --check ./bin/engineeros-connector.mjs && node --check ./src/runner.mjs"
+  },
+  "engines": {
+    "node": ">=22"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/vinpuli/engineerosv2.git",
+    "directory": "packages/engineeros-connector"
+  },
+  "bugs": {
+    "url": "https://github.com/vinpuli/engineerosv2/issues"
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}

package/src/config.mjs

@@ -0,0 +1,51 @@
+import { createHash } from "node:crypto";
+import { chmod, mkdir, readFile, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+
+export function configPath(workspace = process.cwd()) {
+  const key = createHash("sha256")
+    .update(path.resolve(workspace))
+    .digest("hex")
+    .slice(0, 24);
+  return path.join(os.homedir(), ".engineeros", "connectors", `${key}.json`);
+}
+
+export function socketUrl(value) {
+  const url = new URL(value);
+  if (url.protocol === "http:") url.protocol = "ws:";
+  if (url.protocol === "https:") url.protocol = "wss:";
+  if (!["ws:", "wss:"].includes(url.protocol)) {
+    throw new Error("EngineerOS URL must use http, https, ws, or wss.");
+  }
+  url.pathname = "/api/v1/agent-connectors/ws";
+  url.search = "";
+  url.hash = "";
+  return url.toString();
+}
+
+export function resultUrl(websocketUrl, connectorId, runId) {
+  const url = new URL(websocketUrl);
+  url.protocol = url.protocol === "wss:" ? "https:" : "http:";
+  url.pathname = `/api/v1/agent-connectors/${connectorId}/runs/${runId}/result`;
+  return url.toString();
+}
+
+export async function loadConfig(workspace = process.cwd()) {
+  try {
+    return JSON.parse(await readFile(configPath(workspace), "utf8"));
+  } catch (error) {
+    if (error?.code === "ENOENT") return null;
+    throw error;
+  }
+}
+
+export async function saveConfig(config) {
+  const target = configPath(config.workspace);
+  await mkdir(path.dirname(target), { recursive: true });
+  await writeFile(target, `${JSON.stringify(config, null, 2)}\n`, {
+    encoding: "utf8",
+    mode: 0o600,
+  });
+  if (process.platform !== "win32") await chmod(target, 0o600);
+}

package/src/runner.mjs

@@ -0,0 +1,359 @@
+import { createHash } from "node:crypto";
+import { spawn } from "node:child_process";
+import { mkdir, readFile, readdir, stat, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { promisify } from "node:util";
+import { deflateRaw } from "node:zlib";
+
+const deflate = promisify(deflateRaw);
+const MAX_ARCHIVE_BYTES = 25 * 1024 * 1024;
+
+export async function executeAssignment(assignment, config, callbacks) {
+  const runWorkspace = await prepareRunWorkspace(
+    config.workspace,
+    assignment.run_id,
+    assignment.base_revision,
+  );
+  const controller = launchCodex(
+    runWorkspace,
+    assignment.packet_markdown,
+    callbacks,
+  );
+  callbacks.onProcess?.(controller.child);
+  await controller.completed;
+  const changedFiles = await changedFilePaths(runWorkspace);
+  if (!changedFiles.length)
+    throw new Error("Codex completed without changing any files.");
+  const diffPatch = await boundedDiff(runWorkspace, changedFiles);
+  const archive = await repositoryArchive(runWorkspace);
+  const revision = `worktree-${createHash("sha256").update(diffPatch).digest("hex").slice(0, 55)}`;
+  return {
+    head_revision: revision,
+    repository_locator: assignment.repository_locator,
+    external_reference: `connector:${config.connector_id}/run:${assignment.run_id}`,
+    diff_patch: diffPatch,
+    changed_files: changedFiles,
+    repository_archive_base64: archive.toString("base64"),
+    repository_archive_media_type: "application/zip",
+    proof_evidence: [],
+  };
+}
+
+export async function stopProcess(child) {
+  if (!child || child.exitCode !== null) return;
+  if (process.platform === "win32") {
+    await run(
+      "taskkill",
+      ["/pid", String(child.pid), "/t", "/f"],
+      process.cwd(),
+      { allowFailure: true },
+    );
+  } else {
+    child.kill("SIGTERM");
+  }
+}
+
+async function prepareRunWorkspace(workspace, runId, baseRevision) {
+  const source = path.resolve(workspace);
+  const root = path.join(os.homedir(), ".engineeros", "runs");
+  const target = path.join(root, runId);
+  await mkdir(root, { recursive: true });
+  const isGit =
+    (
+      await run("git", ["rev-parse", "--is-inside-work-tree"], source, {
+        allowFailure: true,
+      })
+    ).code === 0;
+  if (isGit) {
+    const exists = await stat(target).then(
+      () => true,
+      () => false,
+    );
+    if (!exists) {
+      const base = await worktreeBase(source, baseRevision);
+      await run("git", ["worktree", "add", "--detach", target, base], source);
+    }
+    return target;
+  }
+  const exists = await stat(target).then(
+    () => true,
+    () => false,
+  );
+  if (!exists) {
+    await mkdir(target, { recursive: true });
+    await copyWorkspace(source, target);
+    await run("git", ["init"], target);
+    await run("git", ["config", "user.name", "EngineerOS Connector"], target);
+    await run(
+      "git",
+      ["config", "user.email", "connector@engineeros.local"],
+      target,
+    );
+    await run("git", ["add", "-A"], target);
+    await run(
+      "git",
+      ["commit", "--allow-empty", "-m", "EngineerOS run baseline"],
+      target,
+    );
+  }
+  return target;
+}
+
+function launchCodex(workspace, packet, callbacks) {
+  const command =
+    process.env.CODEX_BIN ||
+    (process.platform === "win32" ? "codex.cmd" : "codex");
+  const args = [
+    "exec",
+    "--json",
+    "--sandbox",
+    "workspace-write",
+    "-C",
+    workspace,
+    "-",
+  ];
+  const child = spawn(command, args, {
+    cwd: workspace,
+    env: process.env,
+    shell: process.platform === "win32",
+    stdio: ["pipe", "pipe", "pipe"],
+  });
+  const prompt = `${packet}\n\n## EngineerOS execution instruction\n\nImplement this frozen Goal completely in the current run workspace. Run the required verification. Do not commit, push, or modify files outside this workspace. End with a concise result and verification summary.\n`;
+  child.stdin.end(prompt);
+  let output = "";
+  let buffer = "";
+  child.stdout.setEncoding("utf8");
+  child.stdout.on("data", (chunk) => {
+    output += chunk;
+    buffer += chunk;
+    const lines = buffer.split(/\r?\n/);
+    buffer = lines.pop() ?? "";
+    for (const line of lines) {
+      if (!line.trim()) continue;
+      try {
+        const event = JSON.parse(line);
+        callbacks.onEvent?.(event);
+      } catch {
+        callbacks.onEvent?.({
+          type: "agent.output",
+          message: line.slice(0, 500),
+        });
+      }
+    }
+  });
+  child.stderr.setEncoding("utf8");
+  child.stderr.on("data", (chunk) => {
+    output += chunk;
+    callbacks.onEvent?.({
+      type: "agent.stderr",
+      message: chunk.trim().slice(0, 500),
+    });
+  });
+  const completed = new Promise((resolve, reject) => {
+    child.once("error", reject);
+    child.once("close", (code) => {
+      if (code === 0) resolve(output.slice(-20_000));
+      else
+        reject(
+          new Error(`Codex exited with code ${code}. ${output.slice(-1_000)}`),
+        );
+    });
+  });
+  return { child, completed };
+}
+
+async function changedFilePaths(workspace) {
+  const tracked = await run(
+    "git",
+    ["diff", "--name-only", "HEAD", "--"],
+    workspace,
+  );
+  const untracked = await run(
+    "git",
+    ["ls-files", "--others", "--exclude-standard"],
+    workspace,
+  );
+  return [
+    ...new Set(
+      `${tracked.stdout}\n${untracked.stdout}`
+        .split(/\r?\n/)
+        .map(normalizePath)
+        .filter(Boolean),
+    ),
+  ].sort();
+}
+
+async function boundedDiff(workspace, changedFiles) {
+  const tracked = await run(
+    "git",
+    ["diff", "--binary", "HEAD", "--"],
+    workspace,
+  );
+  const untracked = new Set(
+    (
+      await run(
+        "git",
+        ["ls-files", "--others", "--exclude-standard"],
+        workspace,
+      )
+    ).stdout
+      .split(/\r?\n/)
+      .map(normalizePath)
+      .filter(Boolean),
+  );
+  const parts = [tracked.stdout];
+  for (const relative of changedFiles.filter((item) => untracked.has(item))) {
+    const generated = await run(
+      "git",
+      ["diff", "--no-index", "--binary", "--", "/dev/null", relative],
+      workspace,
+      {
+        allowFailure: true,
+      },
+    );
+    parts.push(generated.stdout);
+  }
+  const diff = parts.filter(Boolean).join("\n");
+  if (!diff.trim()) throw new Error("The run produced no bounded diff.");
+  return diff;
+}
+
+async function repositoryArchive(workspace) {
+  const listed = await run(
+    "git",
+    ["ls-files", "-co", "--exclude-standard"],
+    workspace,
+  );
+  const files = [
+    ...new Set(listed.stdout.split(/\r?\n/).map(normalizePath).filter(Boolean)),
+  ].sort();
+  return createZip(
+    await Promise.all(
+      files.map(async (relative) => ({
+        name: relative,
+        data: await readFile(path.join(workspace, relative)),
+      })),
+    ),
+  );
+}
+
+async function worktreeBase(workspace, requested) {
+  if (!requested || requested.startsWith("greenfield:")) return "HEAD";
+  const exists = await run(
+    "git",
+    ["cat-file", "-e", `${requested}^{commit}`],
+    workspace,
+    { allowFailure: true },
+  );
+  if (exists.code !== 0) {
+    throw new Error(
+      `The frozen base revision ${requested} is not available in this repository.`,
+    );
+  }
+  return requested;
+}
+
+export async function createZip(files) {
+  const localParts = [];
+  const centralParts = [];
+  let offset = 0;
+  let total = 0;
+  for (const file of files) {
+    const name = Buffer.from(normalizePath(file.name), "utf8");
+    const data = Buffer.from(file.data);
+    total += data.length;
+    if (total > MAX_ARCHIVE_BYTES)
+      throw new Error("Repository archive exceeds the 25 MB connector limit.");
+    const compressed = await deflate(data);
+    const crc = crc32(data);
+    const local = Buffer.alloc(30);
+    local.writeUInt32LE(0x04034b50, 0);
+    local.writeUInt16LE(20, 4);
+    local.writeUInt16LE(0x0800, 6);
+    local.writeUInt16LE(8, 8);
+    local.writeUInt32LE(crc, 14);
+    local.writeUInt32LE(compressed.length, 18);
+    local.writeUInt32LE(data.length, 22);
+    local.writeUInt16LE(name.length, 26);
+    localParts.push(local, name, compressed);
+
+    const central = Buffer.alloc(46);
+    central.writeUInt32LE(0x02014b50, 0);
+    central.writeUInt16LE(20, 4);
+    central.writeUInt16LE(20, 6);
+    central.writeUInt16LE(0x0800, 8);
+    central.writeUInt16LE(8, 10);
+    central.writeUInt32LE(crc, 16);
+    central.writeUInt32LE(compressed.length, 20);
+    central.writeUInt32LE(data.length, 24);
+    central.writeUInt16LE(name.length, 28);
+    central.writeUInt32LE(offset, 42);
+    centralParts.push(central, name);
+    offset += local.length + name.length + compressed.length;
+  }
+  const central = Buffer.concat(centralParts);
+  const end = Buffer.alloc(22);
+  end.writeUInt32LE(0x06054b50, 0);
+  end.writeUInt16LE(files.length, 8);
+  end.writeUInt16LE(files.length, 10);
+  end.writeUInt32LE(central.length, 12);
+  end.writeUInt32LE(offset, 16);
+  return Buffer.concat([...localParts, central, end]);
+}
+
+async function copyWorkspace(source, target) {
+  for (const entry of await readdir(source, { withFileTypes: true })) {
+    if ([".git", ".engineeros", "node_modules"].includes(entry.name)) continue;
+    const from = path.join(source, entry.name);
+    const to = path.join(target, entry.name);
+    if (entry.isDirectory()) {
+      await mkdir(to, { recursive: true });
+      await copyWorkspace(from, to);
+    } else if (entry.isFile()) {
+      await writeFile(to, await readFile(from));
+    }
+  }
+}
+
+function normalizePath(value) {
+  return String(value ?? "")
+    .trim()
+    .replaceAll("\\", "/")
+    .replace(/^\.\//, "");
+}
+
+function run(command, args, cwd, options = {}) {
+  return new Promise((resolve, reject) => {
+    const child = spawn(command, args, {
+      cwd,
+      shell: false,
+      windowsHide: true,
+    });
+    let stdout = "";
+    let stderr = "";
+    child.stdout.setEncoding("utf8");
+    child.stderr.setEncoding("utf8");
+    child.stdout.on("data", (chunk) => (stdout += chunk));
+    child.stderr.on("data", (chunk) => (stderr += chunk));
+    child.once("error", reject);
+    child.once("close", (code) => {
+      const result = { code: code ?? 1, stdout, stderr };
+      if (code === 0 || options.allowFailure) resolve(result);
+      else
+        reject(
+          new Error(`${command} ${args.join(" ")} failed: ${stderr || stdout}`),
+        );
+    });
+  });
+}
+
+function crc32(buffer) {
+  let crc = 0xffffffff;
+  for (const byte of buffer) {
+    crc ^= byte;
+    for (let bit = 0; bit < 8; bit += 1)
+      crc = (crc >>> 1) ^ (0xedb88320 & -(crc & 1));
+  }
+  return (crc ^ 0xffffffff) >>> 0;
+}