nomarmy 0.1.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/NOTICE +25 -0
- package/README.md +484 -0
- package/bin/nomarmy.mjs +2248 -0
- package/config/agents.yml.example +63 -0
- package/config/common.env +31 -0
- package/config/profiles/bedrock-cheap.env +26 -0
- package/config/profiles/bedrock.env +28 -0
- package/config/profiles/cpu-linux.env +8 -0
- package/config/profiles/dgx-spark.env +12 -0
- package/config/profiles/macbook-pro.env +9 -0
- package/config/profiles/nvidia-linux.env +9 -0
- package/docker/Dockerfile +15 -0
- package/docker/Dockerfile.go +29 -0
- package/docker/Dockerfile.rust +19 -0
- package/e2e.sh +153 -0
- package/install.sh +125 -0
- package/lib/agents.mjs +285 -0
- package/lib/army.mjs +400 -0
- package/lib/budget.mjs +368 -0
- package/lib/claude-transcript.mjs +150 -0
- package/lib/config.mjs +193 -0
- package/lib/connect.mjs +409 -0
- package/lib/coordinator-instructions.mjs +23 -0
- package/lib/decompose.mjs +389 -0
- package/lib/dispatch-config.mjs +164 -0
- package/lib/dispatch-schema.mjs +280 -0
- package/lib/doctor.mjs +443 -0
- package/lib/evidence.mjs +679 -0
- package/lib/gguf.mjs +589 -0
- package/lib/hardware.mjs +476 -0
- package/lib/health.mjs +278 -0
- package/lib/model-catalog.mjs +71 -0
- package/lib/notifier-app.mjs +95 -0
- package/lib/notify.mjs +66 -0
- package/lib/openclaw-config.mjs +65 -0
- package/lib/openclaw-errors.mjs +40 -0
- package/lib/propose.mjs +110 -0
- package/lib/prune.mjs +77 -0
- package/lib/repo-query.mjs +267 -0
- package/lib/runs.mjs +150 -0
- package/lib/sabotage.mjs +128 -0
- package/lib/sandbox-images.mjs +434 -0
- package/lib/scan.mjs +1538 -0
- package/lib/schema.mjs +288 -0
- package/lib/scout.mjs +544 -0
- package/lib/sizing.mjs +1322 -0
- package/lib/slots.mjs +112 -0
- package/lib/statusline.mjs +126 -0
- package/lib/subscription-config.mjs +68 -0
- package/lib/subscription-setup.mjs +217 -0
- package/lib/transcript.mjs +195 -0
- package/lib/verify.mjs +700 -0
- package/mcp/server.mjs +4206 -0
- package/notifier/icon.swift +34 -0
- package/notifier/main.swift +52 -0
- package/notifier/nomarmy-icon.png +0 -0
- package/package.json +67 -0
- package/playbooks/feature.md +43 -0
- package/policies/coder.md +49 -0
- package/policies/orchestrator.md +35 -0
- package/policies/reviewer.md +35 -0
- package/policies/scout.md +65 -0
- package/scripts/configure-openclaw.sh +96 -0
- package/scripts/configure-orchestrator.sh +84 -0
- package/scripts/install-llama-cpp.sh +16 -0
- package/scripts/lib.sh +198 -0
- package/scripts/select-model.mjs +96 -0
- package/scripts/select-model.sh +4 -0
- package/scripts/setup-sandbox.sh +38 -0
- package/scripts/start-inference.sh +46 -0
- package/scripts/stop-inference.sh +5 -0
- package/scripts/uninstall.sh +6 -0
- package/scripts/verify-install.sh +68 -0
package/lib/sabotage.mjs
ADDED
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
// Detecting a worker that made the tests pass instead of making the code
|
|
2
|
+
// work.
|
|
3
|
+
//
|
|
4
|
+
// A real Senti job: the sandbox had no sqlglot, so the local worker stubbed
|
|
5
|
+
// it out of production code and skipped 5 test classes to get a green
|
|
6
|
+
// result, and another left `.sampling-gate-backup/` copies of source files
|
|
7
|
+
// in the repo. Each is a review flag here, not a verdict -- a new skip
|
|
8
|
+
// marker can be legitimate -- so the coordinator looks before it accepts:
|
|
9
|
+
//
|
|
10
|
+
// skip marker a test newly marked skip/xfail/ignore
|
|
11
|
+
// stubbed import production code newly catching ImportError or writing
|
|
12
|
+
// sys.modules, i.e. carrying on without a dependency
|
|
13
|
+
// shadow module a new file named like one of the project's own
|
|
14
|
+
// dependencies (a local sqlglot.py, pytest.py)
|
|
15
|
+
// stray backup a new file or directory named like a backup copy
|
|
16
|
+
|
|
17
|
+
import fs from "node:fs";
|
|
18
|
+
import path from "node:path";
|
|
19
|
+
|
|
20
|
+
export const SKIP_MARKER_RE = /(@pytest\.mark\.(?:skip|skipif|xfail)\b|\bpytest\.(?:skip|importorskip)\(|@unittest\.skip|\bunittest\.skip(?:If|Unless)?\(|\bself\.skipTest\(|\b(?:it|test|describe)\.skip\(|\bx(?:it|describe|test)\(|\bt\.Skip(?:Now)?\(|#\[ignore\]|@Disabled\b|@Ignore\b)/;
|
|
21
|
+
export const STUB_IMPORT_RE = /(\bsys\.modules\s*\[|\bexcept\s*\(?\s*(?:ModuleNotFoundError|ImportError)\b)/;
|
|
22
|
+
const IMPORT_LINE_RE = /^\s*(?:import\s+[A-Za-z_]|from\s+[A-Za-z_][\w.]*\s+import\b)/;
|
|
23
|
+
const EXCEPT_LINE_RE = /^\s*except\b/;
|
|
24
|
+
const BACKUP_RE = /(^|\/)[^/]*(?:backup|\.bak|\.orig)[^/]*(\/|$)|~$/i;
|
|
25
|
+
const CODE_EXT_RE = /\.(py|js|mjs|cjs|ts|tsx|jsx)$/;
|
|
26
|
+
|
|
27
|
+
const normalize = (name) => String(name).toLowerCase().replace(/[-.]/g, "_");
|
|
28
|
+
|
|
29
|
+
/** Added lines (without the leading "+") from `git diff -U0` output. */
|
|
30
|
+
export function addedLinesOf(diffText) {
|
|
31
|
+
return String(diffText ?? "").split("\n").filter((l) => l.startsWith("+") && !l.startsWith("+++")).map((l) => l.slice(1));
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* The project's declared dependency names, normalized (lowercase, "-" and
|
|
36
|
+
* "." as "_"), from requirements*.txt, pyproject.toml and package.json at
|
|
37
|
+
* the repo root and one level down (Senti keeps its Python ones in lambda/).
|
|
38
|
+
*/
|
|
39
|
+
export function loadDependencyNames(cwd) {
|
|
40
|
+
const names = new Set();
|
|
41
|
+
const dirs = [cwd];
|
|
42
|
+
try { for (const e of fs.readdirSync(cwd, { withFileTypes: true })) if (e.isDirectory() && !e.name.startsWith(".") && e.name !== "node_modules") dirs.push(path.join(cwd, e.name)); } catch { /* unreadable */ }
|
|
43
|
+
for (const dir of dirs) {
|
|
44
|
+
let files;
|
|
45
|
+
try { files = fs.readdirSync(dir); } catch { continue; }
|
|
46
|
+
for (const f of files) {
|
|
47
|
+
const file = path.join(dir, f);
|
|
48
|
+
try {
|
|
49
|
+
if (/^requirements.*\.txt$/.test(f)) {
|
|
50
|
+
for (const line of fs.readFileSync(file, "utf8").split("\n")) {
|
|
51
|
+
const m = /^\s*([A-Za-z0-9][A-Za-z0-9._-]*)/.exec(line.replace(/#.*/, ""));
|
|
52
|
+
if (m) names.add(normalize(m[1]));
|
|
53
|
+
}
|
|
54
|
+
} else if (f === "pyproject.toml") {
|
|
55
|
+
const text = fs.readFileSync(file, "utf8");
|
|
56
|
+
for (const m of text.matchAll(/^\s*["']([A-Za-z0-9][A-Za-z0-9._-]*)\s*[<>=~!\[;,"']/gm)) names.add(normalize(m[1]));
|
|
57
|
+
} else if (f === "package.json") {
|
|
58
|
+
const pkg = JSON.parse(fs.readFileSync(file, "utf8"));
|
|
59
|
+
for (const k of ["dependencies", "devDependencies", "peerDependencies"]) for (const n of Object.keys(pkg[k] ?? {})) names.add(normalize(n.replace(/^@[^/]+\//, "")));
|
|
60
|
+
}
|
|
61
|
+
} catch { /* one bad manifest never stops the rest */ }
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
return names;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Production-code lines that carry on without an import: any `except`
|
|
69
|
+
* around a try that holds only imports (see guardsOnlyImports), or an explicit
|
|
70
|
+
* ImportError/sys.modules workaround, or a dependency's own name set to
|
|
71
|
+
* None. The real stub was `try: import sqlglot` / `except Exception:` /
|
|
72
|
+
* `sqlglot = None`, so catching only ImportError missed it.
|
|
73
|
+
*/
|
|
74
|
+
// Whether the `except` at addedLines[i] guards nothing but imports. A try
|
|
75
|
+
// that also does the work -- `try: from table_catalog_embed import
|
|
76
|
+
// clear_missing_marks; clear_missing_marks(...)` / `except Exception as e:
|
|
77
|
+
// print("... (non-fatal)")`, a Senti file's own lazy-import idiom -- keeps
|
|
78
|
+
// a failure non-fatal; it isn't carrying on without a dependency, and was
|
|
79
|
+
// flagged as a stub (a false positive a reviewer had to clear).
|
|
80
|
+
function guardsOnlyImports(addedLines, i) {
|
|
81
|
+
const start = Math.max(0, i - 8);
|
|
82
|
+
let tryAt = -1;
|
|
83
|
+
for (let k = i - 1; k >= start; k--) if (/^\s*try\s*:/.test(addedLines[k])) { tryAt = k; break; }
|
|
84
|
+
// The try itself isn't in the added lines: only an import right above counts.
|
|
85
|
+
if (tryAt === -1) return addedLines.slice(Math.max(0, i - 4), i).some((l) => IMPORT_LINE_RE.test(l));
|
|
86
|
+
const body = addedLines.slice(tryAt + 1, i).filter((l) => l.trim() && !/^\s*#/.test(l));
|
|
87
|
+
return body.length > 0 && body.every((l) => IMPORT_LINE_RE.test(l));
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
function stubbedImportLines(addedLines, deps) {
|
|
91
|
+
const hits = [];
|
|
92
|
+
addedLines.forEach((line, i) => {
|
|
93
|
+
if (STUB_IMPORT_RE.test(line)) hits.push(line);
|
|
94
|
+
else if (EXCEPT_LINE_RE.test(line) && guardsOnlyImports(addedLines, i)) hits.push(line);
|
|
95
|
+
else {
|
|
96
|
+
const m = /^\s*([A-Za-z_]\w*)\s*=\s*None\b/.exec(line);
|
|
97
|
+
if (m && deps.has(normalize(m[1]))) hits.push(line);
|
|
98
|
+
}
|
|
99
|
+
});
|
|
100
|
+
return hits;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* @param {{ changes: { status: string, path: string, addedLines: string[] }[], isTestPathFn: (p: string) => boolean, dependencyNames?: Set<string>|string[] }} input
|
|
105
|
+
* @returns {{ flags: string[], reason: string } | null}
|
|
106
|
+
*/
|
|
107
|
+
export function detectTestSabotage({ changes = [], isTestPathFn, dependencyNames = [] }) {
|
|
108
|
+
const deps = dependencyNames instanceof Set ? dependencyNames : new Set([...dependencyNames].map(normalize));
|
|
109
|
+
const flags = [];
|
|
110
|
+
const clip = (s) => { const t = String(s).trim(); return t.length > 100 ? `${t.slice(0, 99)}…` : t; };
|
|
111
|
+
for (const c of changes) {
|
|
112
|
+
const isTest = isTestPathFn(c.path);
|
|
113
|
+
if (isTest) {
|
|
114
|
+
const skips = c.addedLines.filter((l) => SKIP_MARKER_RE.test(l));
|
|
115
|
+
if (skips.length) flags.push(`skip marker: ${c.path} newly skips ${skips.length} test(s) (${clip(skips[0])})`);
|
|
116
|
+
} else {
|
|
117
|
+
const stubs = stubbedImportLines(c.addedLines, deps);
|
|
118
|
+
if (stubs.length && CODE_EXT_RE.test(c.path)) flags.push(`stubbed import: production code in ${c.path} newly carries on without an import (${clip(stubs[0])})`);
|
|
119
|
+
}
|
|
120
|
+
if (c.status === "A") {
|
|
121
|
+
const base = normalize(path.basename(c.path).replace(CODE_EXT_RE, ""));
|
|
122
|
+
if (CODE_EXT_RE.test(c.path) && deps.has(base) && !isTest) flags.push(`shadow module: new file ${c.path} is named like the dependency "${base}", which would hide the real package`);
|
|
123
|
+
if (BACKUP_RE.test(c.path)) flags.push(`stray backup: new file ${c.path} looks like a backup copy, not part of the change`);
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
if (!flags.length) return null;
|
|
127
|
+
return { flags, reason: `${flags.length} sign(s) the tests may have been made to pass rather than the code made to work: ${flags.join("; ")}` };
|
|
128
|
+
}
|
|
@@ -0,0 +1,434 @@
|
|
|
1
|
+
// nomArmy per-language sandbox images (v1.3).
|
|
2
|
+
//
|
|
3
|
+
// The base sandbox (docker/Dockerfile) has python3/pip, but not any
|
|
4
|
+
// third-party PACKAGE a real repo needs (boto3, pandas, ...) -- and it can't,
|
|
5
|
+
// since those are repo-specific. A worker or verification run against a Go
|
|
6
|
+
// or Rust repository needs `go`/`cargo` inside the sandbox for the same
|
|
7
|
+
// underlying reason (the base image can't carry every toolchain), though
|
|
8
|
+
// there the fix is simpler: the toolchain itself is repo-independent, so one
|
|
9
|
+
// static image covers every Go repo and every Rust repo. None of this is
|
|
10
|
+
// worth baking into the one image every job uses regardless of language --
|
|
11
|
+
// a native compiler or a repo's own dependency set is real, non-trivial
|
|
12
|
+
// memory pressure competing with the same tight local-inference budget
|
|
13
|
+
// lib/sizing.mjs already fights to protect, and most installs will never
|
|
14
|
+
// touch Go, Rust, or a Python repo with real third-party dependencies.
|
|
15
|
+
//
|
|
16
|
+
// So these images are built lazily, on first use, and cached by Podman from
|
|
17
|
+
// then on (an ordinary tagged image, indistinguishable from one built by
|
|
18
|
+
// hand). The build itself needs network access to fetch packages/rustup/pip
|
|
19
|
+
// packages -- that happens once, on the HOST, via `podman build`; the
|
|
20
|
+
// resulting container a job actually runs in still gets --network none like
|
|
21
|
+
// every other sandbox (this is the actual fix for "pip install has no
|
|
22
|
+
// network in the sandbox": install at BUILD time, which has network, never
|
|
23
|
+
// at run time, which never does). This only ever affects which image
|
|
24
|
+
// nomArmy's OWN independent verification runs against (lib/verify.mjs) -- it
|
|
25
|
+
// does not change what sandbox the worker's own tool calls execute in, which
|
|
26
|
+
// is still whatever OpenClaw is configured with (see docs/experiments for
|
|
27
|
+
// why: `openclaw agent exec` has no per-call sandbox override today).
|
|
28
|
+
|
|
29
|
+
import crypto from "node:crypto";
|
|
30
|
+
import fs from "node:fs";
|
|
31
|
+
import os from "node:os";
|
|
32
|
+
import path from "node:path";
|
|
33
|
+
import { fileURLToPath } from "node:url";
|
|
34
|
+
import { execFileSync } from "node:child_process";
|
|
35
|
+
|
|
36
|
+
const HERE = path.dirname(fileURLToPath(import.meta.url));
|
|
37
|
+
// lib/ and docker/ are always siblings, whether running from the dev
|
|
38
|
+
// checkout or the installed copy under ~/.local/share/nomarmy-local-worker
|
|
39
|
+
// (installMcpCopy copies both) -- resolving relative to this file's own
|
|
40
|
+
// location works in either case with no extra configuration.
|
|
41
|
+
const DOCKER_DIR = path.join(HERE, "..", "docker");
|
|
42
|
+
|
|
43
|
+
/** Marker files, checked at the worktree root only (cheap, no recursive
|
|
44
|
+
* walk) -- this decides which SANDBOX IMAGE a job needs, a narrower and
|
|
45
|
+
* hotter-path question than lib/scan.mjs's full evidence-gathering pass. */
|
|
46
|
+
export const LANGUAGE_IMAGES = Object.freeze({
|
|
47
|
+
go: { image: "openclaw-nomarmy-coder-go:bookworm", dockerfile: "Dockerfile.go", markers: ["go.mod"] },
|
|
48
|
+
rust: { image: "openclaw-nomarmy-coder-rust:bookworm", dockerfile: "Dockerfile.rust", markers: ["Cargo.toml"] },
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Extra PATH entries a language's image needs for OpenClaw's own `exec` tool
|
|
53
|
+
* to find that toolchain -- verified live: `podman exec` on a freshly built
|
|
54
|
+
* Go image resolves `go` fine (the image's own baked ENV PATH includes it),
|
|
55
|
+
* but the SAME container driven through `openclaw agent exec` reported
|
|
56
|
+
* "go: not found" until `tools.exec.pathPrepend` carried these paths
|
|
57
|
+
* explicitly. OpenClaw's exec tool does not inherit a sandbox image's own
|
|
58
|
+
* ENV PATH on its own; this is the config knob that closes that gap. Paths
|
|
59
|
+
* match each Dockerfile's own ENV lines (Dockerfile.go's GOPATH=/home/node/go,
|
|
60
|
+
* Dockerfile.rust's CARGO_HOME=/home/node/.cargo). Python needs nothing here:
|
|
61
|
+
* `pip3 install --break-system-packages` (no --user) puts entry-point
|
|
62
|
+
* scripts on the image's already-default PATH.
|
|
63
|
+
*/
|
|
64
|
+
// Where a Node repo's installed packages live in its dependency image
|
|
65
|
+
// (nodeDependencyFiles below). /node_modules links here: Node's resolver,
|
|
66
|
+
// TypeScript's and `npm run`'s PATH all search parent folders up to /, so
|
|
67
|
+
// /workspace resolves packages from it without touching the worktree.
|
|
68
|
+
export const NODE_DEPS_ROOT = "/deps";
|
|
69
|
+
export const NODE_DEPS_DIR = `${NODE_DEPS_ROOT}/node_modules`;
|
|
70
|
+
export const NODE_DEPS_BIN = `${NODE_DEPS_DIR}/.bin`;
|
|
71
|
+
// npm inside a sandbox: its cache in /tmp, never the worktree, and no
|
|
72
|
+
// update check. With the cache resolving relative to the working directory,
|
|
73
|
+
// `cd lambda/x && npx tsc` left lambda/x/.npm/_update-notifier-last-checked
|
|
74
|
+
// in a real Senti commit. Environment variables beat any .npmrc.
|
|
75
|
+
export const SANDBOX_NPM_ENV = Object.freeze({
|
|
76
|
+
NPM_CONFIG_CACHE: "/tmp/.npm", npm_config_cache: "/tmp/.npm",
|
|
77
|
+
NPM_CONFIG_UPDATE_NOTIFIER: "false", npm_config_update_notifier: "false",
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
export const EXEC_PATH_PREPEND = Object.freeze({
|
|
81
|
+
go: ["/usr/local/go/bin", "/home/node/go/bin"],
|
|
82
|
+
rust: ["/home/node/.cargo/bin"],
|
|
83
|
+
node: [NODE_DEPS_BIN],
|
|
84
|
+
"python+node": [NODE_DEPS_BIN],
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Which requirements file(s) a Python repo's sandbox image should install:
|
|
89
|
+
* `.nomarmy.yml`'s `environment.python.requirements` if declared (a repo
|
|
90
|
+
* with several requirements files -- app, dev, a sub-package's own -- has no
|
|
91
|
+
* single conventional name this could guess), otherwise bare
|
|
92
|
+
* `requirements.txt` if that file actually exists. Never fabricated: a repo
|
|
93
|
+
* with neither declares nothing to install and gets the base image, same as
|
|
94
|
+
* before this existed.
|
|
95
|
+
*/
|
|
96
|
+
export function pythonRequirementsFor(cwd, config) {
|
|
97
|
+
const declared = config?.environment?.python?.requirements;
|
|
98
|
+
if (Array.isArray(declared) && declared.length > 0) return declared;
|
|
99
|
+
return cwd && fs.existsSync(path.join(cwd, "requirements.txt")) ? ["requirements.txt"] : [];
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/** One package directory's own install: { files, reason } (dir is "." or a relative path). */
|
|
103
|
+
function nodePackageAt(cwd, dir) {
|
|
104
|
+
const at = (f) => path.join(cwd, dir, f), rel = (f) => (dir === "." ? f : `${dir}/${f}`);
|
|
105
|
+
if (!fs.existsSync(at("package.json"))) return { files: [], reason: "no package.json" };
|
|
106
|
+
const lock = ["package-lock.json", "npm-shrinkwrap.json"].find((f) => fs.existsSync(at(f)));
|
|
107
|
+
if (!lock) {
|
|
108
|
+
const other = ["pnpm-lock.yaml", "yarn.lock", "bun.lockb", "bun.lock"].find((f) => fs.existsSync(at(f)));
|
|
109
|
+
return { files: [], reason: other ? `${other} isn't supported yet (npm lockfiles only)` : "no package-lock.json" };
|
|
110
|
+
}
|
|
111
|
+
try {
|
|
112
|
+
if (JSON.parse(fs.readFileSync(at("package.json"), "utf8")).workspaces) return { files: [], reason: "npm workspaces aren't supported yet" };
|
|
113
|
+
} catch { return { files: [], reason: "package.json doesn't parse" }; }
|
|
114
|
+
return { files: [rel("package.json"), rel(lock)], reason: null };
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
// Every directory holding an npm lockfile: git's tracked files when this is
|
|
118
|
+
// a checkout (which skips node_modules and anything ignored), else a
|
|
119
|
+
// shallow walk.
|
|
120
|
+
function lockfileDirs(cwd) {
|
|
121
|
+
const isLock = (f) => /(^|\/)(package-lock|npm-shrinkwrap)\.json$/.test(f) && !/(^|\/)node_modules\//.test(f);
|
|
122
|
+
let files = null;
|
|
123
|
+
try {
|
|
124
|
+
files = execFileSync("git", ["ls-files", "-z", "--", "*package-lock.json", "*npm-shrinkwrap.json"], { cwd, encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }).split("\0").filter(Boolean);
|
|
125
|
+
} catch { /* not a checkout */ }
|
|
126
|
+
if (files === null) {
|
|
127
|
+
files = [];
|
|
128
|
+
const walk = (dir, depth) => {
|
|
129
|
+
let entries = [];
|
|
130
|
+
try { entries = fs.readdirSync(path.join(cwd, dir), { withFileTypes: true }); } catch { return; }
|
|
131
|
+
for (const e of entries) {
|
|
132
|
+
const rel = dir === "." ? e.name : `${dir}/${e.name}`;
|
|
133
|
+
if (e.isDirectory()) { if (depth < 4 && e.name !== "node_modules" && !e.name.startsWith(".")) walk(rel, depth + 1); }
|
|
134
|
+
else files.push(rel);
|
|
135
|
+
}
|
|
136
|
+
};
|
|
137
|
+
walk(".", 0);
|
|
138
|
+
}
|
|
139
|
+
return [...new Set(files.filter(isLock).map((f) => (f.includes("/") ? f.slice(0, f.lastIndexOf("/")) : ".")))].sort();
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* A Node repo's lockfile installs: { files, packages, skipped, reason }.
|
|
144
|
+
* The sandbox has no network, so without this every JS/TS repo's tsc and
|
|
145
|
+
* tests failed on their first import (a real Senti review). Every package
|
|
146
|
+
* with its own npm lockfile is installed, not just the root: a frontend in
|
|
147
|
+
* ui/ and backends under lambda/, each with its own lockfile and no npm
|
|
148
|
+
* workspaces, is a common layout, and installing only the root left
|
|
149
|
+
* `cd ui && tsc` failing on every file (the same review, one step later).
|
|
150
|
+
*
|
|
151
|
+
* `packages` are the directories ("." for the root) that will install;
|
|
152
|
+
* `files` their package.json and lockfile paths; `skipped` those that
|
|
153
|
+
* won't, each with its reason. npm workspaces, yarn, pnpm and bun aren't
|
|
154
|
+
* installed yet. `environment.node.install: false` turns it off.
|
|
155
|
+
*/
|
|
156
|
+
export function nodeDependencyFiles(cwd, config) {
|
|
157
|
+
const none = (reason) => ({ files: [], packages: [], skipped: [], reason });
|
|
158
|
+
if (!cwd) return none("no package.json");
|
|
159
|
+
if (config?.environment?.node?.install === false) return none("environment.node.install is false");
|
|
160
|
+
const dirs = lockfileDirs(cwd);
|
|
161
|
+
if (!dirs.includes(".")) dirs.unshift(".");
|
|
162
|
+
const packages = [], skipped = [], files = [];
|
|
163
|
+
for (const dir of dirs) {
|
|
164
|
+
const found = nodePackageAt(cwd, dir);
|
|
165
|
+
if (found.files.length) { packages.push(dir); files.push(...found.files); }
|
|
166
|
+
else if (dir !== "." || fs.existsSync(path.join(cwd, "package.json"))) skipped.push({ dir, reason: found.reason });
|
|
167
|
+
}
|
|
168
|
+
const reason = packages.length ? null : (skipped[0]?.reason ?? "no package.json");
|
|
169
|
+
return { files, packages, skipped, reason };
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/**
|
|
173
|
+
* Give each non-root package in `worktree` a node_modules that resolves to
|
|
174
|
+
* its install in the dependency image: a symlink to /deps/<dir>/node_modules,
|
|
175
|
+
* dangling on the host, valid in the sandbox (both mount the worktree at
|
|
176
|
+
* /workspace). Node and TypeScript look for node_modules beside the package
|
|
177
|
+
* first, and /node_modules only covers the root. Never replaces a
|
|
178
|
+
* node_modules that's already there; nomArmy never commits these (see
|
|
179
|
+
* isRuntimeJunk in mcp/server.mjs). Returns the links it made.
|
|
180
|
+
*/
|
|
181
|
+
export function linkNodePackages(worktree, config, { env = process.env } = {}) {
|
|
182
|
+
if (env.NOMARMY_AGENT_IMAGE) return []; // someone else's image: /deps may not exist
|
|
183
|
+
const made = [];
|
|
184
|
+
for (const dir of nodeDependencyFiles(worktree, config).packages) {
|
|
185
|
+
if (dir === ".") continue;
|
|
186
|
+
const link = path.join(worktree, dir, "node_modules");
|
|
187
|
+
try { fs.lstatSync(link); continue; } catch { /* nothing there yet */ }
|
|
188
|
+
try { fs.symlinkSync(`${NODE_DEPS_ROOT}/${dir}/node_modules`, link, "dir"); made.push(`${dir}/node_modules`); } catch { /* best-effort */ }
|
|
189
|
+
}
|
|
190
|
+
return made;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/** Which of LANGUAGE_IMAGES' languages this worktree root looks like, or
|
|
194
|
+
* "python" if it declares/has installable Python dependencies, or null for
|
|
195
|
+
* anything else (including Node, which uses the default image and needs no
|
|
196
|
+
* detection at all). `config` is the already-loaded, already-validated
|
|
197
|
+
* .nomarmy.yml (or null) -- this never loads or parses it itself. */
|
|
198
|
+
export function detectPrimaryLanguage(cwd, config = null) {
|
|
199
|
+
if (!cwd || typeof cwd !== "string") return null;
|
|
200
|
+
for (const [lang, spec] of Object.entries(LANGUAGE_IMAGES)) {
|
|
201
|
+
if (spec.markers.some((m) => fs.existsSync(path.join(cwd, m)))) return lang;
|
|
202
|
+
}
|
|
203
|
+
const python = pythonRequirementsFor(cwd, config).length > 0;
|
|
204
|
+
const node = nodeDependencyFiles(cwd, config).files.length > 0;
|
|
205
|
+
if (python && node) return "python+node";
|
|
206
|
+
if (python) return "python";
|
|
207
|
+
if (node) return "node";
|
|
208
|
+
return null;
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
function defaultRun(cmd, args) {
|
|
212
|
+
return execFileSync(cmd, args, { encoding: "utf8" });
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
function imageExists(image, run) {
|
|
216
|
+
try {
|
|
217
|
+
return run("podman", ["images", "-q", image]).trim().length > 0;
|
|
218
|
+
} catch {
|
|
219
|
+
// Podman missing/unreachable is not this function's problem to solve --
|
|
220
|
+
// the caller's own sandbox-start path already handles that absence.
|
|
221
|
+
return false;
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
/** Builds LANGUAGE_IMAGES[lang]'s image if Podman doesn't already have it
|
|
226
|
+
* tagged. Idempotent and safe to call on every job -- the existence check
|
|
227
|
+
* is what makes repeat calls a no-op, not a memoized flag, so it is
|
|
228
|
+
* correct even across separate processes/restarts. Throws with a clear
|
|
229
|
+
* reason on failure; never silently falls back to a different image, since
|
|
230
|
+
* running Go/Rust commands in an image without that toolchain would look
|
|
231
|
+
* like a genuine test failure rather than the infrastructure gap it is. */
|
|
232
|
+
export function ensureLanguageImageBuilt(lang, { run = defaultRun } = {}) {
|
|
233
|
+
const spec = LANGUAGE_IMAGES[lang];
|
|
234
|
+
if (!spec) throw new Error(`unknown sandbox language: ${lang}`);
|
|
235
|
+
if (imageExists(spec.image, run)) return spec.image;
|
|
236
|
+
const dockerfilePath = path.join(DOCKER_DIR, spec.dockerfile);
|
|
237
|
+
try {
|
|
238
|
+
run("podman", ["build", "-t", spec.image, "-f", dockerfilePath, DOCKER_DIR]);
|
|
239
|
+
} catch (error) {
|
|
240
|
+
throw new Error(`failed to build the ${lang} sandbox image (${spec.image}): ${error.message}`);
|
|
241
|
+
}
|
|
242
|
+
return spec.image;
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
/**
|
|
246
|
+
* Unlike Go/Rust (one static image for every repo, since the toolchain is
|
|
247
|
+
* repo-independent), Python's problem is the repo's OWN dependencies -- a
|
|
248
|
+
* generic python3/pip image still fails on `import boto3`. So the image is
|
|
249
|
+
* built FROM the worktree as its context (so a Dockerfile COPY can reach the
|
|
250
|
+
* real requirements files at their real relative paths) and tagged by a hash
|
|
251
|
+
* of their content, not a fixed name: a dependency change is a cache miss
|
|
252
|
+
* (rebuilds once, then cached like every other job), and an unchanged
|
|
253
|
+
* requirements file across different repos or branches is a cache hit
|
|
254
|
+
* against the exact same tag, no coordination needed to make that happen.
|
|
255
|
+
*/
|
|
256
|
+
export function pythonImageTag(cwd, requirements) {
|
|
257
|
+
const hash = crypto.createHash("sha256");
|
|
258
|
+
for (const rel of requirements) {
|
|
259
|
+
hash.update(rel);
|
|
260
|
+
hash.update("\0");
|
|
261
|
+
// A requirements file the tag-computation can't read is not this
|
|
262
|
+
// function's problem to solve -- the build step right after will fail
|
|
263
|
+
// on the same missing file, with a real, specific error, instead of
|
|
264
|
+
// this one guessing at a substitute hash.
|
|
265
|
+
try { hash.update(fs.readFileSync(path.join(cwd, rel))); } catch { /* surfaces at build time */ }
|
|
266
|
+
hash.update("\0");
|
|
267
|
+
}
|
|
268
|
+
return `openclaw-nomarmy-coder-python-${hash.digest("hex").slice(0, 8)}:bookworm`;
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
function pythonDockerfile(requirements) {
|
|
272
|
+
const copies = requirements.map((rel, i) => `COPY ${rel} /tmp/reqs/req-${i}.txt`).join("\n");
|
|
273
|
+
const reqFlags = requirements.map((_, i) => `-r /tmp/reqs/req-${i}.txt`).join(" ");
|
|
274
|
+
// Same base as docker/Dockerfile plus build-essential: pandas/pyarrow/
|
|
275
|
+
// psycopg2-binary all ship manylinux wheels so compiling should not be
|
|
276
|
+
// needed, but a requirements file this hasn't been tested against might
|
|
277
|
+
// need it -- cheap insurance, same reasoning as the Rust image. bookworm's
|
|
278
|
+
// packaged pip refuses to install into the system environment at all
|
|
279
|
+
// (PEP 668) without --break-system-packages; this container IS the
|
|
280
|
+
// dedicated environment, there is no venv to prefer it over.
|
|
281
|
+
return `FROM node:24-bookworm-slim
|
|
282
|
+
ENV DEBIAN_FRONTEND=noninteractive
|
|
283
|
+
RUN apt-get update && apt-get install -y --no-install-recommends \\
|
|
284
|
+
bash ca-certificates curl git jq python3 python3-pip python3-venv ripgrep build-essential \\
|
|
285
|
+
&& rm -rf /var/lib/apt/lists/*
|
|
286
|
+
${copies}
|
|
287
|
+
RUN pip3 install --no-cache-dir --break-system-packages ${reqFlags}
|
|
288
|
+
USER node
|
|
289
|
+
WORKDIR /workspace
|
|
290
|
+
CMD ["sleep", "infinity"]
|
|
291
|
+
`;
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
/** Builds this worktree's Python dependency image if Podman doesn't already
|
|
295
|
+
* have it tagged. Returns null (never throws) when there is nothing to
|
|
296
|
+
* install -- the caller falls back to the default image, unchanged from
|
|
297
|
+
* before this existed. A build failure DOES throw, same as
|
|
298
|
+
* ensureLanguageImageBuilt and for the same reason: running against the
|
|
299
|
+
* wrong image would look like a genuine test failure, not the
|
|
300
|
+
* infrastructure gap it actually is. */
|
|
301
|
+
export function ensurePythonImageBuilt(cwd, config, { run = defaultRun } = {}) {
|
|
302
|
+
const requirements = pythonRequirementsFor(cwd, config);
|
|
303
|
+
if (!requirements.length) return null;
|
|
304
|
+
const image = pythonImageTag(cwd, requirements);
|
|
305
|
+
if (imageExists(image, run)) return image;
|
|
306
|
+
const dockerfilePath = path.join(os.tmpdir(), `nomarmy-python-${crypto.randomBytes(6).toString("hex")}.Dockerfile`);
|
|
307
|
+
fs.writeFileSync(dockerfilePath, pythonDockerfile(requirements));
|
|
308
|
+
try {
|
|
309
|
+
run("podman", ["build", "-t", image, "-f", dockerfilePath, cwd]);
|
|
310
|
+
} catch (error) {
|
|
311
|
+
throw new Error(`failed to build the python sandbox image (${image}) from ${requirements.join(", ")}: ${error.message}`);
|
|
312
|
+
} finally {
|
|
313
|
+
fs.rmSync(dockerfilePath, { force: true });
|
|
314
|
+
}
|
|
315
|
+
return image;
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
/**
|
|
319
|
+
* A repo's dependency image for Node, or Python and Node together: npm ci
|
|
320
|
+
* (and pip) at build time, on the host, which has network. Tagged by a hash
|
|
321
|
+
* of every dependency file, like pythonImageTag. The build context is a
|
|
322
|
+
* temp folder holding only those files, never the repo (whose own
|
|
323
|
+
* node_modules alone can be gigabytes).
|
|
324
|
+
*/
|
|
325
|
+
export function dependencyImageTag(cwd, files) {
|
|
326
|
+
const hash = crypto.createHash("sha256");
|
|
327
|
+
for (const rel of files) {
|
|
328
|
+
hash.update(rel); hash.update("\0");
|
|
329
|
+
try { hash.update(fs.readFileSync(path.join(cwd, rel))); } catch { /* surfaces at build time */ }
|
|
330
|
+
hash.update("\0");
|
|
331
|
+
}
|
|
332
|
+
return `openclaw-nomarmy-coder-deps-${hash.digest("hex").slice(0, 8)}:bookworm`;
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
export function dependencyDockerfile({ requirements = [], nodeFiles = [], nodePackages = null }) {
|
|
336
|
+
const reqCopies = requirements.map((_, i) => `COPY py/req-${i}.txt /tmp/reqs/req-${i}.txt`).join("\n");
|
|
337
|
+
const pip = requirements.length ? `RUN pip3 install --no-cache-dir --break-system-packages ${requirements.map((_, i) => `-r /tmp/reqs/req-${i}.txt`).join(" ")}` : "";
|
|
338
|
+
const packages = nodePackages ?? (nodeFiles.length ? ["."] : []);
|
|
339
|
+
// Owned by the sandbox's "node" user from the start and installed as it,
|
|
340
|
+
// so a tool that writes a cache under node_modules (vite, babel) can, and
|
|
341
|
+
// no chown -R layer doubles the image.
|
|
342
|
+
const nodeCopies = packages.length ? [`RUN mkdir -p ${NODE_DEPS_ROOT} && chown node:node ${NODE_DEPS_ROOT}`, ...nodeFiles.map((f) => `COPY --chown=node:node node/${f} ${NODE_DEPS_ROOT}/${f}`), "USER node"].join("\n") : "";
|
|
343
|
+
// One layer per package, each laid out at its repo path under /deps. A
|
|
344
|
+
// package that can't install (a private registry, say) leaves a marker
|
|
345
|
+
// instead of failing the image, so the others still work.
|
|
346
|
+
const installs = packages.map((dir) => {
|
|
347
|
+
const at = dir === "." ? NODE_DEPS_ROOT : `${NODE_DEPS_ROOT}/${dir}`;
|
|
348
|
+
return `RUN cd ${at} && (npm ci --no-audit --no-fund || touch .nomarmy-npm-ci-failed) && npm cache clean --force`;
|
|
349
|
+
}).join("\n");
|
|
350
|
+
// The root's packages are reachable from anywhere under /workspace via
|
|
351
|
+
// /node_modules; each other package gets a worktree link (linkNodePackages).
|
|
352
|
+
const root = packages.length ? ["USER root", ...(packages.includes(".") ? [`RUN ln -s ${NODE_DEPS_DIR} /node_modules`, `ENV PATH=${NODE_DEPS_BIN}:$PATH`] : [])].join("\n") : "";
|
|
353
|
+
return `FROM node:24-bookworm-slim
|
|
354
|
+
ENV DEBIAN_FRONTEND=noninteractive
|
|
355
|
+
RUN apt-get update && apt-get install -y --no-install-recommends \\
|
|
356
|
+
bash ca-certificates curl git jq python3 python3-pip python3-venv ripgrep build-essential \\
|
|
357
|
+
&& rm -rf /var/lib/apt/lists/*
|
|
358
|
+
${[reqCopies, pip, nodeCopies, installs, root].filter(Boolean).join("\n")}
|
|
359
|
+
USER node
|
|
360
|
+
WORKDIR /workspace
|
|
361
|
+
CMD ["sleep", "infinity"]
|
|
362
|
+
`;
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
/** Builds the dependency image if Podman doesn't have it; null when there's nothing to install. Throws on a failed build, like ensurePythonImageBuilt. */
|
|
366
|
+
export function ensureDependencyImageBuilt(cwd, config, { run = defaultRun } = {}) {
|
|
367
|
+
const requirements = pythonRequirementsFor(cwd, config);
|
|
368
|
+
const node = nodeDependencyFiles(cwd, config), nodeFiles = node.files;
|
|
369
|
+
if (!nodeFiles.length) return requirements.length ? ensurePythonImageBuilt(cwd, config, { run }) : null;
|
|
370
|
+
const all = [...requirements, ...nodeFiles];
|
|
371
|
+
const image = dependencyImageTag(cwd, all);
|
|
372
|
+
if (imageExists(image, run)) return image;
|
|
373
|
+
const context = fs.mkdtempSync(path.join(os.tmpdir(), "nomarmy-deps-"));
|
|
374
|
+
try {
|
|
375
|
+
fs.mkdirSync(path.join(context, "py")); fs.mkdirSync(path.join(context, "node"));
|
|
376
|
+
requirements.forEach((rel, i) => fs.copyFileSync(path.join(cwd, rel), path.join(context, "py", `req-${i}.txt`)));
|
|
377
|
+
for (const f of nodeFiles) {
|
|
378
|
+
fs.mkdirSync(path.dirname(path.join(context, "node", f)), { recursive: true });
|
|
379
|
+
fs.copyFileSync(path.join(cwd, f), path.join(context, "node", f));
|
|
380
|
+
}
|
|
381
|
+
fs.writeFileSync(path.join(context, "Dockerfile"), dependencyDockerfile({ requirements, nodeFiles, nodePackages: node.packages }));
|
|
382
|
+
run("podman", ["build", "-t", image, "-f", path.join(context, "Dockerfile"), context]);
|
|
383
|
+
} catch (error) {
|
|
384
|
+
throw new Error(`failed to build the dependency sandbox image (${image}) from ${all.join(", ")}: ${error.message}`);
|
|
385
|
+
} finally {
|
|
386
|
+
fs.rmSync(context, { recursive: true, force: true });
|
|
387
|
+
}
|
|
388
|
+
return image;
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
/**
|
|
392
|
+
* The image a verification run against `cwd` should use: an explicit
|
|
393
|
+
* override always wins (matches every other explicit-config-beats-
|
|
394
|
+
* auto-detect choice in this codebase); otherwise a detected Go/Rust repo
|
|
395
|
+
* gets its language image, a Python repo with real dependencies gets its
|
|
396
|
+
* own dependency-hashed image (both built lazily), and everything else gets
|
|
397
|
+
* `defaultImage` unchanged -- zero behavior change for Node repos, or
|
|
398
|
+
* Python repos that declare nothing to install.
|
|
399
|
+
*/
|
|
400
|
+
export function resolveSandboxImage({ cwd, explicitImage, defaultImage, config = null, run = defaultRun }) {
|
|
401
|
+
if (explicitImage) return explicitImage;
|
|
402
|
+
const lang = detectPrimaryLanguage(cwd, config);
|
|
403
|
+
if (!lang) return defaultImage;
|
|
404
|
+
if (lang === "python") return ensurePythonImageBuilt(cwd, config, { run }) ?? defaultImage;
|
|
405
|
+
if (lang === "node" || lang === "python+node") return ensureDependencyImageBuilt(cwd, config, { run }) ?? defaultImage;
|
|
406
|
+
return ensureLanguageImageBuilt(lang, { run });
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
/** What each npm package's node_modules is right now: "link", "dir" or "none", by package dir ("." is the root). */
|
|
410
|
+
export function nodeModulesState(worktree, config) {
|
|
411
|
+
const out = {};
|
|
412
|
+
for (const dir of nodeDependencyFiles(worktree, config).packages) {
|
|
413
|
+
try { out[dir] = fs.lstatSync(path.join(worktree, dir, "node_modules")).isSymbolicLink() ? "link" : "dir"; } catch { out[dir] = "none"; }
|
|
414
|
+
}
|
|
415
|
+
return out;
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
/**
|
|
419
|
+
* Packages whose node_modules became a real directory during the job. In
|
|
420
|
+
* the sandbox nothing can install (no network), so a new real node_modules
|
|
421
|
+
* means a tool ran outside it: a real Senti job on the Claude CLI ran
|
|
422
|
+
* `npm install` on the host, replacing the dependency link with macOS
|
|
423
|
+
* binaries, and verification then failed on them. The directories are
|
|
424
|
+
* removed and the links restored so verification runs against the image
|
|
425
|
+
* (they're never committed either way); the caller flags the job for review.
|
|
426
|
+
*/
|
|
427
|
+
export function repairHostInstalls(worktree, config, before) {
|
|
428
|
+
const after = nodeModulesState(worktree, config);
|
|
429
|
+
const replaced = Object.keys(after).filter((dir) => after[dir] === "dir" && before?.[dir] !== "dir");
|
|
430
|
+
for (const dir of replaced) fs.rmSync(path.join(worktree, dir, "node_modules"), { recursive: true, force: true });
|
|
431
|
+
if (replaced.length) linkNodePackages(worktree, config);
|
|
432
|
+
return replaced.map((dir) => (dir === "." ? "node_modules" : `${dir}/node_modules`));
|
|
433
|
+
}
|
|
434
|
+
|