@fieldwork-ai/cli 0.24.0 → 0.26.1

package/README.md

@@ -41,6 +41,8 @@ fieldwork connectors list | pull <key> | publish <dir> --message "…" | connect
 
 Mutations prompt for confirmation on a TTY; pass `--yes` to skip (required in non-interactive use). `--json` prints stable machine-readable output on every verb. Secret values never travel via argv or stdout: `secrets set` reads from `--stdin`, `--from-env-file`, or `--from-env`; `secrets pull` writes a chmod-600 file atomically and refuses to overwrite an existing file without `--force`.
 
+`packages pull <name>` and `packages publish <dir> --name <name> --message "…" --base-revision N` address the org's own code by package name, which is unique per organization, whether or not it runs on a system: a tree whose `connector.json` carries a `key` publishes onto that system, and one without is a package with no system, such as a sync that only reads and writes sheets. The `connectors pull`/`publish` verbs below are the system-addressed form of the same loop.
+
 The connector development loop end to end: `connectors pull acme_erp` → edit the adapter → `connectors publish acme_erp --message "…" --base-revision N` (a new key creates the definition from the tree's `connector.json`) → `connectors connect acme_erp --spec spec.json` → `connectors secrets set <id>` → `automations run-now <sync> --follow` to watch the logs.
 
 Publishing initializes a new connector from `connector.json`, but subsequent action revisions leave `connector_definitions` metadata unchanged. To edit its display name, description, branding, or REST secret-name hints, use `fieldwork connectors metadata <definition-id>`, then `fieldwork connectors update-metadata <definition-id> --changes changes.json --expected-updated-at '<exact updated_at from the read>'`. The update requires connector edit access and rejects stale reads; credential values are managed separately through `secrets`.

package/dist/commands/connectors.js

@@ -209,7 +209,7 @@ export function registerConnectorCommands(program) {
             // fall through: the server reports the real validation error
         }
         if (key === "?") {
-            fail('the tree\'s connector.json must carry a "key" field', EXIT.usage);
+            fail('the tree\'s connector.json must carry a "key" field — for a package with no system, use `packages publish`', EXIT.usage);
         }
         await confirm(`Publish ${Object.keys(files).length} files to connector "${key}"${opts.actionName ? `, action "${opts.actionName}"` : ""}?`, yes);
         const body = await apiJson(`/api/organizations/${requireOrgId()}/connectors/publish`, {

package/dist/commands/kb-upload.d.ts

@@ -0,0 +1,4 @@
+export declare function uploadLocal(local: string, folder: string | undefined, opts: {
+    shareWithOrg?: boolean;
+    onExisting: string;
+}): Promise<void>;

package/dist/commands/kb-upload.js

@@ -0,0 +1,263 @@
+import { createHash } from "node:crypto";
+import { createReadStream, readdirSync, statSync } from "node:fs";
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+import { api, getContext, requireOrgId } from "../http.js";
+import { EXIT, confirm, exitCodeFor, fail, note, printData } from "../output.js";
+/**
+ * `fieldwork kb upload <local> [folder]` — a folder (or one file) into the
+ * Knowledge Base, keeping its structure, through the same bulk upload the
+ * browser uses (/knowledge-base/uploads; ADR 20261003_kb_bulk_upload).
+ *
+ * Built for imports of tens of thousands of files: every file is hashed
+ * first, and anything already stored with the same contents is skipped, so
+ * running the same command again after an interruption resumes rather than
+ * starting over. A file that fails does not stop the rest; the summary lists
+ * them and the exit code says so.
+ */
+/** Kept in step with src/lib/kb/upload-ignore.ts. */
+const IGNORED_NAMES = new Set(["Thumbs.db", "desktop.ini", "Icon\r", "node_modules", "__MACOSX"]);
+const isIgnored = (segment) => segment.startsWith(".") || IGNORED_NAMES.has(segment) || segment.startsWith("~$");
+const BATCH = 200;
+const HASH_CONCURRENCY = 8;
+const SEND_CONCURRENCY = 4;
+const RETRY_MS = [1_000, 2_000, 5_000, 10_000, 30_000];
+export async function uploadLocal(local, folder, opts) {
+    const { json, yes } = getContext();
+    if (!["skip", "replace", "keep-both"].includes(opts.onExisting)) {
+        fail(`--on-existing must be skip, replace or keep-both`, EXIT.usage);
+    }
+    const onExisting = opts.onExisting;
+    const target = (folder ?? "").replace(/^\/+|\/+$/g, "");
+    const { files, ignored } = collect(path.resolve(local), target);
+    if (files.length === 0)
+        fail(`no files to upload in ${local}`, EXIT.usage);
+    const base = `/api/organizations/${requireOrgId()}/knowledge-base/uploads`;
+    const limits = await call(base);
+    const tally = { uploaded: 0, unchanged: 0, skipped: 0, failed: [] };
+    const sendable = files.filter((f) => {
+        if (f.size <= limits.maxFileBytes)
+            return true;
+        tally.failed.push({ path: f.path, reason: `larger than ${formatBytes(limits.maxFileBytes)}` });
+        return false;
+    });
+    const bytes = sendable.reduce((n, f) => n + f.size, 0);
+    await confirm(`Upload ${sendable.length.toLocaleString()} files (${formatBytes(bytes)}) to ${target ? `"${target}"` : "the top level"}` +
+        `${opts.shareWithOrg ? ", shared with the whole organization" : ""}?` +
+        (ignored ? ` ${ignored} hidden or system files are left out.` : "") +
+        (bytes > limits.limitBytes - limits.usedBytes
+            ? ` Only ${formatBytes(Math.max(0, limits.limitBytes - limits.usedBytes))} of space is left, so some will not fit.`
+            : ""), yes);
+    const { uploadId } = await call(base, { folder: target });
+    let done = 0;
+    for (let i = 0; i < sendable.length; i += BATCH) {
+        const batch = sendable.slice(i, i + BATCH);
+        await runBatch(`${base}/${uploadId}`, batch, onExisting, !!opts.shareWithOrg, tally);
+        done += batch.length;
+        if (!json)
+            note(`${done.toLocaleString()} of ${sendable.length.toLocaleString()} files`);
+    }
+    if (json) {
+        printData({ uploadId, ...tally }, true);
+    }
+    else {
+        note(`Uploaded ${tally.uploaded}, unchanged ${tally.unchanged}` +
+            (tally.skipped ? `, skipped ${tally.skipped} already there` : "") +
+            (tally.failed.length ? `, failed ${tally.failed.length}` : ""));
+        for (const f of tally.failed)
+            note(`  ${f.path}: ${f.reason}`);
+    }
+    if (tally.failed.length)
+        process.exit(EXIT.server);
+}
+/** The files under `local` (or `local` itself), each at `target/<dir name>/<relative path>`. */
+function collect(local, target) {
+    const files = [];
+    let ignored = 0;
+    const prefix = target ? `${target}/` : "";
+    const visit = (abs, rel) => {
+        if (isIgnored(path.basename(abs))) {
+            ignored += 1;
+            return;
+        }
+        let stat;
+        try {
+            stat = statSync(abs);
+        }
+        catch {
+            return;
+        }
+        if (stat.isDirectory()) {
+            for (const name of readdirSync(abs).sort())
+                visit(path.join(abs, name), `${rel}/${name}`);
+        }
+        else if (stat.isFile()) {
+            files.push({ abs, path: prefix + rel, size: stat.size });
+        }
+    };
+    let stat;
+    try {
+        stat = statSync(local);
+    }
+    catch {
+        fail(`no such file or folder: ${local}`, EXIT.usage);
+    }
+    if (!stat.isDirectory() && !stat.isFile())
+        fail(`not a file or folder: ${local}`, EXIT.usage);
+    visit(local, path.basename(local));
+    return { files, ignored };
+}
+async function runBatch(upload, batch, onExisting, share, tally) {
+    const readable = [];
+    await pool(batch, HASH_CONCURRENCY, async (f) => {
+        try {
+            f.sha256 = await hashFile(f.abs);
+            readable.push(f);
+        }
+        catch {
+            tally.failed.push({ path: f.path, reason: "could not be read" });
+        }
+    });
+    const plan = (files, mode) => call(`${upload}/plan`, {
+        entries: files.map((f) => ({ path: f.path, sizeBytes: f.size, sha256: f.sha256 })),
+        onExisting: mode,
+        shareWithOrganization: share,
+    }).then((r) => r.results);
+    let pending = batch.filter((f) => readable.includes(f));
+    let mode = "ask";
+    for (let round = 0; round < 3 && pending.length; round++) {
+        const results = await plan(pending, mode);
+        const toSend = [];
+        const clashed = [];
+        results.forEach((result, i) => {
+            const file = pending[i];
+            if (result.status === "upload")
+                toSend.push({ file, result });
+            else if (result.status === "unchanged")
+                tally.unchanged += 1;
+            else if (result.status === "invalid")
+                tally.failed.push({ path: file.path, reason: result.reason });
+            else if (result.status === "full")
+                tally.failed.push({ path: file.path, reason: "the Knowledge Base is full" });
+            else if (onExisting === "skip")
+                tally.skipped += 1;
+            else if (!result.replaceable && onExisting === "replace") {
+                tally.failed.push({ path: file.path, reason: "a file you can't replace is already there" });
+            }
+            else if (mode !== "ask") {
+                tally.failed.push({ path: file.path, reason: "a file is already there" });
+            }
+            else
+                clashed.push(file);
+        });
+        const sent = [];
+        await pool(toSend, SEND_CONCURRENCY, async (entry) => {
+            const error = await send(entry.file, entry.result);
+            if (error)
+                tally.failed.push({ path: entry.file.path, reason: error });
+            else
+                sent.push(entry);
+        });
+        const retry = [...clashed];
+        if (sent.length) {
+            const committed = await call(`${upload}/commit`, {
+                entries: sent.map((s) => ({ revisionId: s.result.revisionId, fileId: s.result.fileId })),
+            });
+            committed.results.forEach((result, i) => {
+                const file = sent[i].file;
+                if (result.status === "done")
+                    tally.uploaded += 1;
+                else if (result.status === "unchanged")
+                    tally.unchanged += 1;
+                else if (result.status === "exists") {
+                    if (onExisting === "skip")
+                        tally.skipped += 1;
+                    else
+                        retry.push(file);
+                }
+                else if (result.status === "missing")
+                    retry.push(file);
+                else
+                    tally.failed.push({ path: file.path, reason: result.reason });
+            });
+        }
+        pending = retry;
+        if (onExisting !== "skip")
+            mode = onExisting;
+    }
+    for (const f of pending)
+        tally.failed.push({ path: f.path, reason: "could not be uploaded after several attempts" });
+}
+/** PUT one file to its link; returns why it failed, or null. */
+async function send(file, result) {
+    const body = await readFile(file.abs);
+    for (let attempt = 0;; attempt++) {
+        let status = 0;
+        try {
+            // A request that never answers would hold the whole import; at 64 KB/s
+            // even the largest file finishes well inside this.
+            const timeout = AbortSignal.timeout(Math.max(120_000, (file.size / 65_536) * 1000));
+            status = (await fetch(result.url, { method: "PUT", headers: result.headers, body, signal: timeout })).status;
+        }
+        catch {
+            status = 0;
+        }
+        if (status >= 200 && status < 300)
+            return null;
+        if (status === 400)
+            return "the file changed while it was uploading";
+        const transient = status === 0 || status === 408 || status === 429 || status >= 500;
+        if (!transient || attempt >= RETRY_MS.length)
+            return `storage refused it (${status || "network error"})`;
+        await sleep(RETRY_MS[attempt]);
+    }
+}
+/** A JSON request, retried through network failures and 5xx; anything else ends the command. */
+async function call(pathname, body) {
+    for (let attempt = 0;; attempt++) {
+        let res = null;
+        try {
+            res = await api(pathname, body === undefined ? {} : { method: "POST", body: JSON.stringify(body) }, {
+                throwOnNetworkError: true,
+            });
+        }
+        catch {
+            res = null;
+        }
+        if (res && res.ok)
+            return (await res.json());
+        const transient = !res || res.status === 429 || res.status >= 500;
+        if (!transient || attempt >= RETRY_MS.length) {
+            const message = res ? (await res.json().catch(() => null))?.error : null;
+            fail(message ?? `request failed (${res?.status ?? "network error"})`, res ? exitCodeFor(res.status) : EXIT.server);
+        }
+        await sleep(RETRY_MS[attempt]);
+    }
+}
+function hashFile(abs) {
+    return new Promise((resolve, reject) => {
+        const hash = createHash("sha256");
+        createReadStream(abs)
+            .on("data", (chunk) => hash.update(chunk))
+            .on("end", () => resolve(hash.digest("hex")))
+            .on("error", reject);
+    });
+}
+async function pool(items, size, run) {
+    let next = 0;
+    await Promise.all(Array.from({ length: Math.min(size, items.length) }, async () => {
+        while (next < items.length)
+            await run(items[next++]);
+    }));
+}
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+function formatBytes(bytes) {
+    const units = ["B", "KB", "MB", "GB", "TB"];
+    let value = bytes;
+    let unit = 0;
+    while (value >= 1024 && unit < units.length - 1) {
+        value /= 1024;
+        unit += 1;
+    }
+    return `${unit === 0 ? value : value.toFixed(1)} ${units[unit]}`;
+}

package/dist/commands/kb.js

@@ -4,18 +4,34 @@ import { EXIT, confirm, fail, note, printData } from "../output.js";
 import { writeBundle } from "../bundle.js";
 import { assertSafeKbPath, lookupPage, putPage, readMarkdownTree } from "./kb-pages.js";
 import { downloadFile, uploadFile } from "./kb-blobs.js";
+import { uploadLocal } from "./kb-upload.js";
 import { isMarkdown } from "../content-kind.js";
 async function listTree() {
     const body = await apiJson(`/api/organizations/${requireOrgId()}/knowledge-base`);
     return body.files ?? [];
 }
+/**
+ * The row at exactly `kbPath`. Never through `listTree`: a command run once
+ * per file over an organization of 60,000 files listed the whole tree each
+ * time. The server follows move aliases; a command acts on the path it was
+ * given, so a row found under another path is not a match.
+ */
 async function rowAt(kbPath) {
-    return (await listTree()).find((r) => r.path === kbPath) ?? null;
+    const query = new URLSearchParams({ path: kbPath, content: "false" });
+    const body = await apiJson(`/api/organizations/${requireOrgId()}/knowledge-base?${query}`);
+    return body.files?.find((r) => r.path === kbPath) ?? null;
 }
 export function registerKbCommands(program) {
     const kb = program
         .command("kb")
         .description("The Knowledge Base: markdown pages and the org's other files, in one tree");
+    kb
+        .command("upload <local> [folder]")
+        .description("Upload a local folder (or file) into the Knowledge Base, keeping its structure. " +
+        "Files already there with the same contents are skipped, so running it again resumes.")
+        .option("--share-with-org", "let everyone in the organization view and edit the new files")
+        .option("--on-existing <action>", "when a different file is already at a path: skip, replace or keep-both", "skip")
+        .action(uploadLocal);
     kb
         .command("ls [prefix]")
         .description("List everything, optionally under a path prefix")

package/dist/commands/packages.d.ts

@@ -0,0 +1,2 @@
+import { Command } from "commander";
+export declare function registerPackageCommands(program: Command): void;

package/dist/commands/packages.js

@@ -0,0 +1,53 @@
+import { readBundle, writeBundle } from "../bundle.js";
+import { apiJson, getContext, requireOrgId } from "../http.js";
+import { confirm, note, printData } from "../output.js";
+/**
+ * Package family: the org's own code, addressed by the package's NAME (unique
+ * per organization), whether or not it runs on a system. A tree whose
+ * connector.json carries a `key` publishes onto that system; without one it is
+ * a package with no system. Platform adapters are not pullable.
+ */
+function defaultDir(name) {
+    return name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "") || "package";
+}
+export function registerPackageCommands(program) {
+    const packages = program.command("packages").description("The org's own code, by package name");
+    packages
+        .command("pull <name>")
+        .description("Download a package's source tree")
+        .option("--out <dir>", "target directory (defaults to a slug of the name)")
+        .option("--revision <n>", "a historical revision instead of head")
+        .action(async (name, opts) => {
+        const { json } = getContext();
+        const suffix = opts.revision ? `?${new URLSearchParams({ revision: opts.revision })}` : "";
+        const body = await apiJson(`/api/organizations/${requireOrgId()}/packages/${encodeURIComponent(name)}/source${suffix}`);
+        const dir = opts.out ?? defaultDir(name);
+        const pulled = writeBundle(dir, body.files);
+        if (json) {
+            printData({ name: body.name, key: body.key, revision: body.revision, head: body.head, dir, pulled }, true);
+            return;
+        }
+        note(`Pulled "${body.name}" revision ${body.revision} into ${dir}/ — pass --base-revision ${body.head} when publishing.`);
+    });
+    packages
+        .command("publish <dir>")
+        .description("Publish a source tree as the package's next revision (creates it on first publish)")
+        .requiredOption("--name <name>", "the package's name")
+        .requiredOption("--message <message>", "what changed and why")
+        .option("--base-revision <n>", "head revision your changes are based on (required once code is published)")
+        .action(async (dir, opts) => {
+        const { json, yes } = getContext();
+        const files = readBundle(dir);
+        await confirm(`Publish ${Object.keys(files).length} files to package "${opts.name}"?`, yes);
+        const body = await apiJson(`/api/organizations/${requireOrgId()}/packages/publish`, {
+            method: "POST",
+            body: JSON.stringify({
+                name: opts.name,
+                message: opts.message,
+                base_revision: opts.baseRevision ? Number(opts.baseRevision) : undefined,
+                files,
+            }),
+        });
+        printData(body, json);
+    });
+}

package/dist/commands/tool.js

@@ -68,8 +68,8 @@ export function registerToolCommands(program) {
             return;
         }
         for (const t of body.tools) {
-            // Skip leading bracketed router markers (Composio prefixes every
-            // "direct" tool with one) so the summary line is the real description.
+            // Skip leading bracketed router markers (some providers prefix a tool's
+            // description with one) so the summary line is the real description.
             const firstLine = t.description.split("\n").find((l) => l.trim() && !/^\[.*\]$/.test(l.trim())) ??
                 t.description.split("\n")[0] ??
                 "";

package/dist/http.d.ts

@@ -20,6 +20,10 @@ export declare function requireToken(): string;
 export declare function requireOrgId(): string;
 export declare function api(pathname: string, init?: RequestInit & {
     body?: string;
+}, 
+/** Throw a network failure to the caller, which retries it, instead of exiting. */
+options?: {
+    throwOnNetworkError?: boolean;
 }): Promise<Response>;
 /** api() + JSON body + fail-on-error, the shape most commands want. */
 export declare function apiJson<T = unknown>(pathname: string, init?: RequestInit & {

package/dist/http.js

@@ -77,7 +77,9 @@ async function refreshLoginToken() {
         return null;
     }
 }
-export async function api(pathname, init = {}) {
+export async function api(pathname, init = {}, 
+/** Throw a network failure to the caller, which retries it, instead of exiting. */
+options = {}) {
     const { config } = getContext();
     const token = requireToken();
     const headers = new Headers(init.headers);
@@ -93,7 +95,7 @@ export async function api(pathname, init = {}) {
         res = await fetch(`${config.apiUrl}${pathname}`, { ...init, headers });
     }
     catch (err) {
-        if (init.signal?.aborted)
+        if (init.signal?.aborted || options.throwOnNetworkError)
             throw err;
         fail(`could not reach ${config.apiUrl}: ${err instanceof Error ? err.message : "network error"}`, EXIT.server);
     }

package/dist/index.js

@@ -10,6 +10,7 @@ import { registerAppCommands } from "./commands/apps.js";
 import { registerAutomationCommands } from "./commands/automations.js";
 import { registerTaskCommands } from "./commands/tasks.js";
 import { registerConnectorCommands } from "./commands/connectors.js";
+import { registerPackageCommands } from "./commands/packages.js";
 import { registerAuthCommands } from "./commands/auth.js";
 /**
  * The Fieldwork CLI (ADR 20260804): a plain REST client of the org-scoped
@@ -41,6 +42,7 @@ registerAppCommands(program);
 registerAutomationCommands(program);
 registerTaskCommands(program);
 registerConnectorCommands(program);
+registerPackageCommands(program);
 program.addCommand(sheetsCommand());
 program.addCommand(sheetsCommand("tables"), { hidden: true });
 program.parseAsync(process.argv).catch((err) => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fieldwork-ai/cli",
-  "version": "0.24.0",
+  "version": "0.26.1",
   "description": "Fieldwork asset-management CLI",
   "license": "UNLICENSED",
   "type": "module",