@engineeros/connector 0.1.0 → 0.2.1

package/README.md

@@ -1,25 +1,29 @@
 # EngineerOS Connector
 
-The connector keeps an outbound WebSocket open from a repository-owning machine to EngineerOS and executes assigned Goals with Codex CLI.
+Connect a local Codex CLI workspace to EngineerOS through an outbound WebSocket.
 
-## Requirements
+## Onboard a workspace
 
-- Node.js 22 or newer
-- Git
-- Codex CLI installed and authenticated (`codex login`)
+Create a connection command from **Project steering -> Connect workspace**, then run it inside the local folder:
 
-## Pair once
+```sh
+npx --yes @engineeros/connector@latest pair PAIRING-CODE --url https://your-engineeros.example --workspace . --onboard
+```
 
-Open the repository, choose **Connect a machine** in the EngineerOS Goal screen, and run the generated `npx @engineeros/connector pair ...` command. The one-time code expires after 10 minutes. A workspace-scoped connector token is stored below `~/.engineeros/connectors` with owner-only permissions where the operating system supports them.
+The connector uploads a bounded ZIP snapshot for assessment, then stays online for rescans and Goal Runs. Onboarding never executes repository code. It excludes known secrets, dependency directories, build output, compiled binaries, files larger than 5 MB, agent-tool caches, Git metadata, and connector state before upload.
 
-```shell
-npx --yes @engineeros/connector@latest pair PAIRING-CODE --url https://engineeros.example.com --workspace /path/to/repository
-```
+An empty or document-only folder establishes a greenfield baseline. A code-bearing folder is assessed as brownfield. Use **Rescan** in Steering after the local workspace changes.
 
 ## Reconnect
 
-```shell
-npx --yes @engineeros/connector@latest start --workspace /path/to/repository
+```sh
+npx @engineeros/connector start --workspace .
 ```
 
-Each Goal runs in an isolated worktree below `~/.engineeros/runs`. Cancellation stops Codex. The connector returns the changed paths, bounded diff, and repository ZIP to EngineerOS; a human still performs independent attestation.
+Credentials are stored per workspace under `~/.engineeros/connectors` with owner-only permissions where supported.
+
+## Run Goals
+
+Keep the connector online to receive Goals assigned from EngineerOS. Each Goal runs in an isolated worktree below `~/.engineeros/runs`. Cancellation stops Codex. The connector returns changed paths, a bounded diff, and the exact repository ZIP; a human still performs independent attestation.
+
+Requirements: Node.js 22 or newer, Git, and an authenticated Codex CLI (`codex login`).

package/bin/engineeros-connector.mjs

@@ -6,8 +6,13 @@ import {
   resultUrl,
   saveConfig,
   socketUrl,
+  workspaceUrl,
 } from "../src/config.mjs";
-import { executeAssignment, stopProcess } from "../src/runner.mjs";
+import {
+  executeAssignment,
+  stopProcess,
+  workspaceSnapshot,
+} from "../src/runner.mjs";
 
 const { command, positional, flags } = parseArgs(process.argv.slice(2));
 
@@ -34,6 +39,8 @@ if (command === "pair") {
   config = {
     server_url: socketUrl(url),
     workspace: path.resolve(flags.workspace || process.cwd()),
+    onboard: flags.onboard === true,
+    onboarding_pending: flags.onboard === true,
     name:
       flags.name ||
       `${os.hostname()} - ${path.basename(path.resolve(flags.workspace || process.cwd()))}`,
@@ -70,6 +77,7 @@ let socket;
 let pingTimer;
 let reconnectDelay = 1_000;
 let connectionRejected = false;
+let snapshotInFlight = false;
 
 process.on("SIGINT", async () => {
   stopped = true;
@@ -104,12 +112,18 @@ async function connect() {
       console.log(`Paired. Connector ${config.connector_id} is online.`);
       reconnectDelay = 1_000;
       startPings();
+      if (config.onboarding_pending) void submitWorkspaceSnapshot();
       return;
     }
     if (message.type === "authenticated") {
       console.log("Connected and waiting for EngineerOS runs.");
       reconnectDelay = 1_000;
       startPings();
+      if (config.onboarding_pending) void submitWorkspaceSnapshot();
+      return;
+    }
+    if (message.type === "workspace.refresh") {
+      void submitWorkspaceSnapshot();
       return;
     }
     if (message.type === "run.available") {
@@ -135,6 +149,7 @@ async function connect() {
     }
     if (message.type === "run.error") {
       console.error(`EngineerOS: ${message.message}`);
+      if (!message.run_id) snapshotInFlight = false;
       if (active?.runId === message.run_id && !active.child) {
         active = null;
         pump();
@@ -164,6 +179,46 @@ async function connect() {
   socket.addEventListener("error", () => {});
 }
 
+async function submitWorkspaceSnapshot() {
+  if (snapshotInFlight || socket.readyState !== WebSocket.OPEN) return;
+  snapshotInFlight = true;
+  console.log("Inspecting the workspace without executing its code.");
+  try {
+    const snapshot = await workspaceSnapshot(config.workspace);
+    const response = await fetch(
+      workspaceUrl(config.server_url, config.connector_id),
+      {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+          Authorization: `Bearer ${config.token}`,
+        },
+        body: JSON.stringify(snapshot),
+      },
+    );
+    if (!response.ok) {
+      throw new Error(
+        `EngineerOS rejected the workspace (${response.status}): ${await response.text()}`,
+      );
+    }
+    const result = await response.json();
+    config = { ...config, onboarding_pending: false };
+    await saveConfig(config);
+    console.log(
+      `Sent ${snapshot.file_count} safe file(s); ${snapshot.excluded_file_count} sensitive or generated path(s) excluded.`,
+    );
+    console.log(
+      `Workspace assessed as ${result.workspace_kind}. EngineerOS System State is current.`,
+    );
+    snapshotInFlight = false;
+  } catch (error) {
+    snapshotInFlight = false;
+    console.error(
+      `Workspace assessment failed: ${error instanceof Error ? error.message : String(error)}`,
+    );
+  }
+}
+
 function startPings() {
   clearInterval(pingTimer);
   pingTimer = setInterval(() => {
@@ -276,6 +331,7 @@ function parseArgs(args) {
   while (args.length) {
     const value = args.shift();
     if (!value.startsWith("--")) positional.push(value);
+    else if (value === "--onboard") flags.onboard = true;
     else flags[value.slice(2)] = args.shift();
   }
   return { command, positional, flags };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@engineeros/connector",
-  "version": "0.1.0",
+  "version": "0.2.1",
   "description": "Connect a local Codex CLI workspace to EngineerOS over an outbound WebSocket.",
   "private": false,
   "type": "module",

package/src/config.mjs

@@ -31,6 +31,13 @@ export function resultUrl(websocketUrl, connectorId, runId) {
   return url.toString();
 }
 
+export function workspaceUrl(websocketUrl, connectorId) {
+  const url = new URL(websocketUrl);
+  url.protocol = url.protocol === "wss:" ? "https:" : "http:";
+  url.pathname = `/api/v1/agent-connectors/${connectorId}/workspace`;
+  return url.toString();
+}
+
 export async function loadConfig(workspace = process.cwd()) {
   try {
     return JSON.parse(await readFile(configPath(workspace), "utf8"));

package/src/runner.mjs

@@ -1,6 +1,6 @@
 import { createHash } from "node:crypto";
 import { spawn } from "node:child_process";
-import { mkdir, readFile, readdir, stat, writeFile } from "node:fs/promises";
+import { lstat, mkdir, readFile, readdir, stat, writeFile } from "node:fs/promises";
 import os from "node:os";
 import path from "node:path";
 import { promisify } from "node:util";
@@ -8,6 +8,78 @@ import { deflateRaw } from "node:zlib";
 
 const deflate = promisify(deflateRaw);
 const MAX_ARCHIVE_BYTES = 25 * 1024 * 1024;
+const MAX_SHAREABLE_FILE_BYTES = 5 * 1024 * 1024;
+const MAX_WORKSPACE_FILES = 5_000;
+const EXCLUDED_DIRECTORIES = new Set([
+  ".agents",
+  ".claude",
+  ".codex",
+  ".engineeros",
+  ".forge",
+  ".gemini",
+  ".git",
+  ".next",
+  ".pytest_cache",
+  ".ruff_cache",
+  ".venv",
+  "__pycache__",
+  "build",
+  "coverage",
+  "dist",
+  "node_modules",
+  "target",
+  "vendor",
+]);
+const EXCLUDED_PATH_PREFIXES = [".github/skills/"];
+const EXCLUDED_FILE_EXTENSIONS = new Set([
+  ".7z",
+  ".a",
+  ".bin",
+  ".class",
+  ".dll",
+  ".dylib",
+  ".exe",
+  ".gz",
+  ".jar",
+  ".lib",
+  ".o",
+  ".obj",
+  ".pyc",
+  ".pyo",
+  ".rar",
+  ".so",
+  ".tar",
+  ".tgz",
+  ".war",
+  ".zip",
+]);
+const CODE_EXTENSIONS = new Set([
+  ".c",
+  ".cpp",
+  ".cs",
+  ".go",
+  ".h",
+  ".java",
+  ".js",
+  ".jsx",
+  ".mjs",
+  ".php",
+  ".py",
+  ".rb",
+  ".rs",
+  ".sql",
+  ".ts",
+  ".tsx",
+]);
+const CODE_MARKERS = new Set([
+  "cargo.toml",
+  "composer.json",
+  "go.mod",
+  "package.json",
+  "pom.xml",
+  "pyproject.toml",
+  "requirements.txt",
+]);
 
 export async function executeAssignment(assignment, config, callbacks) {
   const runWorkspace = await prepareRunWorkspace(
@@ -54,6 +126,66 @@ export async function stopProcess(child) {
   }
 }
 
+export async function workspaceSnapshot(workspace) {
+  const root = path.resolve(workspace);
+  const gitFiles = await run(
+    "git",
+    ["ls-files", "-co", "--exclude-standard"],
+    root,
+    { allowFailure: true },
+  );
+  const candidates =
+    gitFiles.code === 0
+      ? gitFiles.stdout.split(/\r?\n/).map(normalizePath).filter(Boolean)
+      : await workspaceFiles(root);
+  const files = [];
+  let excludedFileCount = 0;
+  let totalBytes = 0;
+  for (const relative of [...new Set(candidates)].sort()) {
+    if (!isShareablePath(relative)) {
+      excludedFileCount += 1;
+      continue;
+    }
+    const absolute = path.join(root, relative);
+    const details = await lstat(absolute).catch(() => null);
+    if (!details?.isFile() || details.isSymbolicLink()) {
+      excludedFileCount += 1;
+      continue;
+    }
+    if (details.size > MAX_SHAREABLE_FILE_BYTES) {
+      excludedFileCount += 1;
+      continue;
+    }
+    totalBytes += details.size;
+    if (totalBytes > MAX_ARCHIVE_BYTES) {
+      throw new Error(
+        "Shareable workspace files exceed 25 MB. Exclude large generated or data files before onboarding.",
+      );
+    }
+    files.push({ name: relative, data: await readFile(absolute) });
+    if (files.length > MAX_WORKSPACE_FILES) {
+      throw new Error(
+        `Workspace exceeds the ${MAX_WORKSPACE_FILES.toLocaleString()}-file onboarding limit. Exclude generated files first.`,
+      );
+    }
+  }
+  const archive = await createZip(files);
+  const head = await run("git", ["rev-parse", "HEAD"], root, {
+    allowFailure: true,
+  });
+  return {
+    archive_base64: archive.toString("base64"),
+    media_type: "application/zip",
+    workspace_name: path.basename(root),
+    workspace_kind: files.some(({ name }) => isCodeBearing(name))
+      ? "brownfield"
+      : "greenfield",
+    file_count: files.length,
+    excluded_file_count: excludedFileCount,
+    head_revision: head.code === 0 ? head.stdout.trim().slice(0, 128) : null,
+  };
+}
+
 async function prepareRunWorkspace(workspace, runId, baseRevision) {
   const source = path.resolve(workspace);
   const root = path.join(os.homedir(), ".engineeros", "runs");
@@ -238,6 +370,58 @@ async function repositoryArchive(workspace) {
   );
 }
 
+async function workspaceFiles(root, current = root) {
+  const files = [];
+  for (const entry of await readdir(current, { withFileTypes: true })) {
+    const absolute = path.join(current, entry.name);
+    const relative = normalizePath(path.relative(root, absolute));
+    if (entry.isSymbolicLink()) continue;
+    if (entry.isDirectory()) {
+      if (!EXCLUDED_DIRECTORIES.has(entry.name.toLowerCase())) {
+        files.push(...(await workspaceFiles(root, absolute)));
+      }
+    } else if (entry.isFile()) {
+      files.push(relative);
+    }
+  }
+  return files;
+}
+
+function isShareablePath(relative) {
+  const normalized = normalizePath(relative);
+  if (!normalized || normalized === ".." || normalized.startsWith("../") || path.isAbsolute(normalized)) {
+    return false;
+  }
+  const parts = normalized.split("/");
+  if (parts.some((part) => EXCLUDED_DIRECTORIES.has(part.toLowerCase()))) {
+    return false;
+  }
+  if (
+    EXCLUDED_PATH_PREFIXES.some((prefix) =>
+      normalized.toLowerCase().startsWith(prefix),
+    )
+  ) {
+    return false;
+  }
+  const name = parts.at(-1)?.toLowerCase() ?? "";
+  if (EXCLUDED_FILE_EXTENSIONS.has(path.posix.extname(name))) return false;
+  if (name === ".env.example" || name === ".env.sample") return true;
+  if (
+    name === ".env" ||
+    name.startsWith(".env.") ||
+    [".npmrc", ".netrc", "credentials", "credentials.json"].includes(name) ||
+    /\.(?:key|pem|p12|pfx)$/i.test(name)
+  ) {
+    return false;
+  }
+  return true;
+}
+
+function isCodeBearing(relative) {
+  const name = path.posix.basename(relative).toLowerCase();
+  return CODE_MARKERS.has(name) || CODE_EXTENSIONS.has(path.posix.extname(name));
+}
+
 async function worktreeBase(workspace, requested) {
   if (!requested || requested.startsWith("greenfield:")) return "HEAD";
   const exists = await run(