@pirol/nido 0.2.1 → 0.3.0

package/README.md

@@ -75,8 +75,24 @@ Instead of an agent, run any command. It needs no key:
 | `files.exclude` | Patterns like in `.gitignore`; `.git` and `.nido` are always excluded |
 | `secrets` | Names of stored secrets, never values |
 | `network` | Hosts the sandbox may reach on port 443; everything else is blocked |
+| `storage` | Storage volumes, mounted at `/data/<name>` |
 | `output` | Folder that comes back after the run; default `output/` |
 
+## Storage
+
+Keep large data in storage, independently of sandboxes:
+
+```bash
+nido storage put ./data my-data   # upload straight to Cloud Storage in Frankfurt
+nido storage ls                   # volumes with file count and size
+nido storage ls my-data           # files in one volume
+nido storage rm my-data           # only once no sandbox uses it
+```
+
+Add `"storage": ["my-data"]` to `nido.json` (or `nido create shop --storage my-data`), and the
+sandbox reads and writes the data at `/data/my-data`, without copying. Volumes are attached when a
+sandbox is created; after changing `storage`, run `nido run --fresh`.
+
 ## Secrets
 
 ```bash
@@ -109,11 +125,11 @@ Paths inside a sandbox must be below `/sandbox`.
 
 ## Limits in the beta
 
-- Up to 5 sandboxes per account.
+- Up to 5 sandboxes per account, and 10 storage volumes with 100 GB in total.
 - No interactive shell yet; use `nido exec`.
 - The agent runs in the foreground. If the connection drops, `nido pull` fetches what is already
   in `output/`.
-- No persistent workspaces or backups yet. Keep anything important in `output/`.
+- No backups or templates of a sandbox yet. Keep anything important in `output/` or in storage.
 
 ## Development
 

package/dist/api.js

@@ -1,6 +1,8 @@
 // Client for pirol-agents-api. Files travel as tar.gz streams, command
 // output comes back as NDJSON while the command runs.
 import { spawn } from "node:child_process";
+import { createReadStream } from "node:fs";
+import { request } from "node:https";
 import { Readable } from "node:stream";
 import { loadConfig } from "./config.js";
 export class ApiError extends Error {
@@ -152,4 +154,37 @@ export class Api {
     removeSecret(name) {
         return this.json("DELETE", `/v1/secrets/${encodeURIComponent(name)}`);
     }
+    storageList() {
+        return this.json("GET", "/v1/storage");
+    }
+    storageFiles(name) {
+        return this.json("GET", `/v1/storage/${encodeURIComponent(name)}`);
+    }
+    async storageCreate(name) {
+        return (await this.json("PUT", `/v1/storage/${encodeURIComponent(name)}`)).result;
+    }
+    /** One upload URL per file; each takes exactly the given number of bytes. */
+    async storageUploads(name, files) {
+        return (await this.json("POST", `/v1/storage/${encodeURIComponent(name)}/uploads`, { files })).uploads;
+    }
+    storageRemove(name) {
+        return this.json("DELETE", `/v1/storage/${encodeURIComponent(name)}`);
+    }
+}
+/** Sends a local file to an upload URL from storageUploads (straight to Cloud Storage). */
+export function putFile(url, file, size) {
+    return new Promise((resolve, reject) => {
+        const req = request(url, { method: "PUT", headers: { "content-length": size } }, (res) => {
+            let body = "";
+            res.on("data", (d) => (body += d.toString("utf8")));
+            res.on("end", () => res.statusCode === 200 || res.statusCode === 201
+                ? resolve()
+                : reject(new Error(`upload failed: HTTP ${res.statusCode} ${body.slice(0, 200)}`)));
+        });
+        req.on("error", reject);
+        if (size === 0)
+            req.end();
+        else
+            createReadStream(file).on("error", reject).pipe(req);
+    });
 }

package/dist/cli.js

@@ -2,6 +2,7 @@
 import { readFileSync, realpathSync } from "node:fs";
 import { fileURLToPath } from "node:url";
 import { copy, createSandbox, exec, init, list, login, logout, pull, runFolder, secretList, secretRemove, secretSet, simple, whoami, } from "./commands.js";
+import { storageLs, storagePut, storageRm } from "./storage.js";
 const HELP = `nido — run agents in isolated sandboxes (closed beta)
 
 Start
@@ -14,13 +15,18 @@ Folder workflow
   nido pull                    bring output/ back again
 
 Sandboxes
-  nido create <name> [--image <ref>] [--secret <name>]… [--network <host>]…
+  nido create <name> [--image <ref>] [--secret <name>]… [--network <host>]… [--storage <name>]…
   nido ls                      list your sandboxes
   nido exec <name> -- <cmd>…   run a command
   nido cp <src> <dst>          copy files; write the sandbox side as <name>:<path>
   nido logs <name>             sandbox logs
   nido stop|start|rm <name>
 
+Storage (lives on without sandboxes, mounted at /data/<name>)
+  nido storage put <folder|file> <name>   upload, straight to Cloud Storage
+  nido storage ls [name]
+  nido storage rm <name>
+
 Secrets (stored once, never readable again)
   nido secret set <name> [--from-gh]   anthropic, openai, github
   nido secret ls
@@ -69,6 +75,7 @@ async function main(argv) {
                 image: flags.get("image")?.[0],
                 secrets: flags.get("secret") ?? [],
                 network: flags.get("network") ?? [],
+                storage: flags.get("storage") ?? [],
             });
             return 0;
         case "ls":
@@ -93,6 +100,8 @@ async function main(argv) {
             return 0;
         case "secret":
             return secret(args, flags);
+        case "storage":
+            return storage(args);
         default:
             throw new Error(`Unknown command "${cmd}". See: nido help`);
     }
@@ -109,7 +118,19 @@ async function secret(args, flags) {
         throw new Error("Usage: nido secret set <name> | nido secret ls | nido secret rm <name>");
     return 0;
 }
-const VALUE_FLAGS = new Set(["image", "secret", "network"]);
+async function storage(args) {
+    const [sub, a, b] = args;
+    if (sub === "put" && a && b)
+        await storagePut(a, b);
+    else if (sub === "ls")
+        await storageLs(a);
+    else if (sub === "rm" && a)
+        await storageRm(a);
+    else
+        throw new Error("Usage: nido storage put <folder|file> <name> | nido storage ls [name] | nido storage rm <name>");
+    return 0;
+}
+const VALUE_FLAGS = new Set(["image", "secret", "network", "storage"]);
 export function parse(argv) {
     const flags = new Map();
     const args = [];

package/dist/commands.js

@@ -164,7 +164,13 @@ async function ensureSandbox(api, m, fresh) {
     }
     if (!info) {
         say(`Creating sandbox ${m.name}…`);
-        const created = await api.create({ name: m.name, image: m.imageSet ? m.image : undefined, secrets: m.secrets, network: m.network });
+        const created = await api.create({
+            name: m.name,
+            image: m.imageSet ? m.image : undefined,
+            secrets: m.secrets,
+            network: m.network,
+            storage: m.storage,
+        });
         say(`Sandbox ${m.name} ready in ${(created.ready_ms / 1000).toFixed(1)} s`);
         return true;
     }

package/dist/manifest.js

@@ -25,7 +25,7 @@ export function parseManifest(raw, folderName) {
     if (!raw || typeof raw !== "object" || Array.isArray(raw))
         throw new Error(`${MANIFEST} must be an object`);
     const m = raw;
-    const known = ["name", "agent", "task", "command", "image", "files", "secrets", "network", "output"];
+    const known = ["name", "agent", "task", "command", "image", "files", "secrets", "network", "storage", "output"];
     const unknown = Object.keys(m).filter((key) => !known.includes(key) && !key.startsWith("$"));
     if (unknown.length)
         throw new Error(`${MANIFEST}: unknown field ${unknown.map((k) => `"${k}"`).join(", ")}`);
@@ -65,6 +65,11 @@ export function parseManifest(raw, folderName) {
         exclude,
         secrets: stringList(m, "secrets"),
         network: stringList(m, "network"),
+        storage: stringList(m, "storage").map((name) => {
+            if (!isVolumeName(name))
+                throw new Error(`${MANIFEST}: "${name}" in "storage" is not a valid name`);
+            return name;
+        }),
         output,
     };
 }
@@ -78,6 +83,10 @@ export function sandboxName(input) {
         .replace(/-+$/, "");
     return name || "nest";
 }
+/** Storage names, like sandbox names: lowercase letters, digits and dashes, at most 40. */
+export function isVolumeName(name) {
+    return /^[a-z0-9]([a-z0-9-]{0,38}[a-z0-9])?$/.test(name);
+}
 function optionalString(m, key) {
     const value = m[key];
     if (value === undefined)

package/dist/storage.js

@@ -0,0 +1,115 @@
+// nido storage: volumes in Cloud Storage that live independently of
+// sandboxes. Files go straight to Cloud Storage; the API only hands out an
+// upload URL per file.
+import { existsSync, statSync } from "node:fs";
+import { basename, dirname, join, resolve } from "node:path";
+import { Api, putFile } from "./api.js";
+import { excludeMatcher, listFiles } from "./files.js";
+import { isVolumeName } from "./manifest.js";
+const say = (text) => void process.stderr.write(text + "\n");
+const BATCH = 500;
+const PARALLEL = 8;
+export function checkVolumeName(name) {
+    if (!isVolumeName(name))
+        throw new Error(`"${name}" is not a valid storage name: lowercase letters, digits and dashes, at most 40 characters`);
+    return name;
+}
+/** Local files to upload: a folder's contents, or one file. Paths use "/". */
+export function localFiles(source) {
+    const full = resolve(source);
+    if (!existsSync(full))
+        throw new Error(`${source} does not exist`);
+    if (statSync(full).isFile())
+        return { root: dirname(full), files: [{ path: basename(full), size: statSync(full).size }] };
+    const paths = listFiles(full, excludeMatcher([]));
+    return { root: full, files: paths.map((path) => ({ path, size: statSync(join(full, path)).size })) };
+}
+/** nido storage put <folder|file> <name> */
+export async function storagePut(source, name) {
+    checkVolumeName(name);
+    const { root, files } = localFiles(source);
+    if (!files.length)
+        throw new Error(`${source} has no files`);
+    const api = Api.fromConfig();
+    if ((await api.storageCreate(name)) === "created")
+        say(`Created storage ${name}`);
+    const total = files.reduce((sum, f) => sum + f.size, 0);
+    say(`Uploading ${files.length} file${files.length === 1 ? "" : "s"} (${size(total)}) to ${name}…`);
+    let done = 0;
+    let sent = 0;
+    const started = Date.now();
+    const progress = () => {
+        if (process.stderr.isTTY)
+            process.stderr.write(`\r  ${done}/${files.length} files, ${size(sent)} of ${size(total)}   `);
+    };
+    for (let i = 0; i < files.length; i += BATCH) {
+        const batch = files.slice(i, i + BATCH);
+        const queue = await api.storageUploads(name, batch);
+        const bySize = new Map(batch.map((f) => [f.path, f.size]));
+        const worker = async () => {
+            for (let item = queue.shift(); item; item = queue.shift()) {
+                const bytes = bySize.get(item.path);
+                await uploadWithRetry(api, name, item, join(root, item.path), bytes);
+                done += 1;
+                sent += bytes;
+                progress();
+            }
+        };
+        await Promise.all(Array.from({ length: PARALLEL }, worker));
+    }
+    if (process.stderr.isTTY)
+        process.stderr.write("\n");
+    const seconds = (Date.now() - started) / 1000;
+    say(`Done in ${seconds.toFixed(1)} s. Sandboxes see it at /data/${name} with "storage": ["${name}"] in nido.json`);
+}
+async function uploadWithRetry(api, name, item, file, bytes) {
+    let url = item.url;
+    for (let attempt = 1;; attempt++) {
+        try {
+            return await putFile(url, file, bytes);
+        }
+        catch (err) {
+            if (attempt === 3)
+                throw new Error(`${item.path}: ${err.message}`);
+            // A failed session cannot take the whole file again; ask for a new one.
+            url = (await api.storageUploads(name, [{ path: item.path, size: bytes }]))[0].url;
+        }
+    }
+}
+/** nido storage ls [name] */
+export async function storageLs(name) {
+    const api = Api.fromConfig();
+    if (name) {
+        const volume = await api.storageFiles(checkVolumeName(name));
+        for (const f of volume.files)
+            process.stdout.write(`${size(f.size).padStart(10)}  ${f.path}\n`);
+        if (volume.total_files > volume.files.length)
+            say(`… ${volume.total_files - volume.files.length} more files`);
+        say(`${volume.total_files} files, ${size(volume.bytes)}`);
+        return;
+    }
+    const { volumes, limit_bytes } = await api.storageList();
+    if (!volumes.length)
+        return say("No storage yet. Upload a folder with: nido storage put ./data my-data");
+    const width = Math.max(4, ...volumes.map((v) => v.name.length));
+    process.stdout.write(`${"NAME".padEnd(width)}  ${"FILES".padStart(7)}  ${"SIZE".padStart(10)}\n`);
+    for (const v of volumes)
+        process.stdout.write(`${v.name.padEnd(width)}  ${String(v.files).padStart(7)}  ${size(v.bytes).padStart(10)}\n`);
+    const used = volumes.reduce((sum, v) => sum + v.bytes, 0);
+    say(`${size(used)} of ${size(limit_bytes)} used`);
+}
+/** nido storage rm <name> */
+export async function storageRm(name) {
+    await Api.fromConfig().storageRemove(checkVolumeName(name));
+    say(`Removed storage ${name}`);
+}
+export function size(bytes) {
+    const units = ["B", "KB", "MB", "GB", "TB"];
+    let value = bytes;
+    let unit = 0;
+    while (value >= 1024 && unit < units.length - 1) {
+        value /= 1024;
+        unit++;
+    }
+    return `${unit === 0 ? value : value.toFixed(1)} ${units[unit]}`;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pirol/nido",
-  "version": "0.2.1",
+  "version": "0.3.0",
   "description": "Run agents in isolated sandboxes. Give it a folder, get results back.",
   "type": "module",
   "bin": {