@commercelayer/agent-cli 1.0.0-dev.8 → 1.0.0-dev.9

package/README.md

@@ -74,6 +74,14 @@ Destructive verbs (deletes, `orders cancel`, `payments void` and `refund`, `sand
 - `.c11r.json` holds only `{ "org": "<slug>" }`: no secret, safe to commit. `sandbox create`, `organizations create` and `auth switch` write it, and `organizations connect` when the folder has no link; `reset` follows the new slug; `destroy` removes it when it names the destroyed organization.
 - The project folder is the one with `.c11r.json`, else the nearest with `store.yml` or `loops.yml`, else the current one. `loops.yml` goes there.
 
+## The store's setup in files
+
+`c11r config export` writes the organization's setup into the project folder: `store.yml` (the Configure nouns, refs by key, no ids, no secrets; its first key, `meta`, names the organization and the time) and `price_lists/<code>.csv`. It records what it wrote in `.c11r/export.json`, and never overwrites a file changed since the last export (or one no export wrote) without `--yes`: the answer lists them. A price list file the organization no longer has, unchanged since, goes. `c11r config plan` sends the files and answers what `config apply` would change; `apply` makes the organization match them, and with `--prune` (confirmed) also deletes what the files lack. While imports run, `apply` answers `status: running` with each noun's state and `retry_after`; `--wait` applies again until done or failed.
+
+After a Configure write that went through (create, update, delete or a domain verb), in a folder whose `store.yml` names the same organization, the answer warns `config_drift` and `.c11r/drift` notes the write; `config plan` repeats the warning, and `config export` clears it.
+
+`c11r exports get <id> --out <file>` downloads a completed export while its link lasts (5 minutes), unzipped unless the name ends in `.gz`; `--wait` reads the job again until it completes.
+
 ## Output, errors and exit codes
 
 Every answer is one JSON object on stdout, the API's envelope:

package/dist/builtins/extensions.js

@@ -8,10 +8,14 @@
 // sales-calls book answers a cancel token once (section 20): stored the same way, by booking id,
 // and sent by sales-calls cancel <booking> without the agent ever holding it.
 import crypto from 'node:crypto';
+import fs from 'node:fs';
+import path from 'node:path';
+import zlib from 'node:zlib';
 import { credentialsFile, hintOf, readLink, storeBooking, storedBooking, storeOrg, writeLink } from '../config.js';
 import { confirm, terminal } from '../confirm.js';
 import { fail } from '../errors.js';
 import { apiUrl } from '../http.js';
+import { clearDrift, driftWarning, readDrift, readStoreFiles, storeFileOrg, writeStoreFiles } from '../project.js';
 import { resourceRequest } from '../resources.js';
 const NAME_MAX = 25;
 export const organizations = {
@@ -104,4 +108,137 @@ export const salesCalls = {
         return undefined;
     },
 };
-export const extensions = { organizations, 'sales-calls': salesCalls };
+// config (contract section 45.2): the API answers and applies documents; the files are the CLI's.
+//   export  writes store.yml and price_lists/<code>.csv, never over a file changed since the last
+//           export without --yes, and clears the drift note.
+//   plan    sends the folder's files and says what apply would change, and since when the
+//           folder is behind when Configure writes were made here.
+//   apply   sends them; while imports run (202, running) it shows each noun's state, and with
+//           --wait applies again until done or failed. prune is confirmed by the schema.
+const WAIT_MAX_MS = 15 * 60_000;
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+const waitMs = (data) => {
+    const s = Number(data?.retry_after);
+    const test = process.env.C11R_WAIT_MS;
+    if (test !== undefined)
+        return Number(test);
+    return (Number.isFinite(s) && s > 0 ? Math.min(s, 60) : 5) * 1000;
+};
+export const config = {
+    flags: {
+        apply: { wait: { type: 'boolean', to: 'local', description: 'While imports run, apply again every retry_after seconds until done or failed (15 minutes at most).' } },
+    },
+    prepare(verb, parsed) {
+        if (verb.name !== 'plan' && verb.name !== 'apply')
+            return;
+        if (parsed.values.store !== undefined)
+            return;
+        const files = readStoreFiles();
+        parsed.values.store = files.store;
+        parsed.values.price_lists = files.price_lists;
+    },
+    async run(run) {
+        const name = run.verb.name;
+        if (name === 'export') {
+            const r = await run.send(resourceRequest('config', run.verb, undefined, run.parsed.values));
+            if (run.globals.dryRun)
+                return r;
+            const data = r.data ?? {};
+            if (!data.store || typeof data.store !== 'object')
+                fail('invalid_response', 'config export answered without store.', { next_step: { action: 'none' } });
+            const out = writeStoreFiles({ store: data.store, price_lists: data.price_lists ?? {}, meta: { org: data.meta?.org ?? run.org.org ?? undefined, exported_at: data.meta?.exported_at } }, { overwrite: run.globals.yes });
+            const counts = Object.fromEntries(Object.entries(data.store).map(([k, v]) => [k, Array.isArray(v) ? v.length : 0]));
+            return {
+                ...r,
+                data: { org: data.meta?.org ?? run.org.org, exported_at: data.meta?.exported_at ?? null, folder: out.dir, files: out.written, removed: out.removed, records: counts, price_lists: Object.keys(data.price_lists ?? {}) },
+                next: [{ command: 'c11r config plan', why: 'what the files would change in an organization, after you edit them or for another one (--org)' }],
+            };
+        }
+        if (name === 'plan' || name === 'apply') {
+            const req = resourceRequest('config', run.verb, undefined, run.parsed.values);
+            let r = await run.send(req);
+            const started = Date.now();
+            while (name === 'apply' && run.parsed.values.wait && r.data?.status === 'running' && Date.now() - started < WAIT_MAX_MS) {
+                const ms = waitMs(r.data);
+                process.stderr.write(JSON.stringify({ running: { nouns: states(r.data), next_in_ms: ms } }) + '\n');
+                await sleep(ms);
+                r = await run.send(req);
+            }
+            const warnings = [...r.warnings];
+            const here = storeFileOrg();
+            const drift = readDrift();
+            if (name === 'plan' && drift.length && here === run.org.org)
+                warnings.push(driftWarning(drift));
+            // The organization now matches the files: the folder is no longer behind.
+            if (name === 'apply' && !run.globals.dryRun && r.data?.status === 'done' && here === run.org.org)
+                clearDrift();
+            const data = r.data ?? {};
+            const running = name === 'apply' && data.status === 'running';
+            return {
+                ...r,
+                data: running ? { ...data, retry_after: waitMs(data) / 1000, states: states(data) } : data,
+                warnings,
+                next: running && !run.parsed.values.wait ? [{ command: `${run.line} --wait`, why: 'carry on, and wait until every noun is done or failed' }, ...r.next] : r.next,
+            };
+        }
+        return undefined;
+    },
+};
+/** Each noun's state while apply runs: done, running (with its import), waiting, failed or unchanged. */
+function states(data) {
+    const out = {};
+    for (const n of Array.isArray(data?.nouns) ? data.nouns : [])
+        out[n.noun] = n.import?.id && n.status === 'running' ? `running (import ${n.import.id})` : String(n.status);
+    return out;
+}
+// exports (45.3): get --out downloads the file while its link is good (5 minutes), unzipped unless
+// the name ends in .gz; --wait polls a job not completed yet.
+export const exports = {
+    flags: {
+        get: {
+            out: { type: 'string', to: 'local', description: 'Download the export to this file once completed (its link lasts 5 minutes). Unzipped, unless the name ends in .gz.' },
+            wait: { type: 'boolean', to: 'local', description: 'Until the export is completed, failed or interrupted, read it again every few seconds (15 minutes at most).' },
+        },
+    },
+    async run(run) {
+        if (run.verb.name !== 'get' || (!run.parsed.values.out && !run.parsed.values.wait))
+            return undefined;
+        const req = resourceRequest('exports', run.verb, run.parsed.id, run.parsed.values);
+        let r = await run.send(req);
+        const started = Date.now();
+        const busy = (s) => s === 'pending' || s === 'in_progress';
+        while (run.parsed.values.wait && busy(r.data?.status) && Date.now() - started < WAIT_MAX_MS) {
+            const ms = waitMs(null);
+            process.stderr.write(JSON.stringify({ waiting: { export: run.parsed.id, status: r.data?.status, next_in_ms: ms } }) + '\n');
+            await sleep(ms);
+            r = await run.send(req);
+        }
+        const out = run.parsed.values.out;
+        if (!out)
+            return r;
+        const data = r.data ?? {};
+        if (data.status !== 'completed' || typeof data.url !== 'string') {
+            fail('conflict', `Export ${run.parsed.id} is ${data.status ?? 'not completed'}: there is no file to download yet.`, { path: 'out', details: { status: data.status ?? null }, next_step: { action: 'fix_request', command: `${run.line} --wait` } }, 'rejected');
+        }
+        const file = path.resolve(out);
+        let res;
+        try {
+            res = await fetch(data.url, { signal: AbortSignal.timeout(120_000) });
+        }
+        catch (e) {
+            fail('api_unreachable', `The export's file could not be downloaded: ${e.message}.`, { path: 'out', next_step: { action: 'fix_request', command: run.line } });
+        }
+        // An expired link: get the export again for a new one.
+        if (!res.ok)
+            fail('expired', `The export's link answered ${res.status}: it lasts 5 minutes. Get the export again for a new one.`, { path: 'out', next_step: { action: 'fix_request', command: run.line } }, 'rejected');
+        let bytes = Buffer.from(await res.arrayBuffer());
+        const zipped = bytes[0] === 0x1f && bytes[1] === 0x8b;
+        if (zipped && !file.endsWith('.gz'))
+            bytes = zlib.gunzipSync(bytes);
+        fs.mkdirSync(path.dirname(file), { recursive: true });
+        fs.writeFileSync(file, bytes);
+        const { url: _url, ...rest } = data;
+        return { ...r, data: { ...rest, file, bytes: bytes.length, unzipped: zipped && !file.endsWith('.gz') } };
+    },
+};
+export const extensions = { organizations, 'sales-calls': salesCalls, config, exports };

package/dist/engine.js

@@ -12,6 +12,7 @@ import { envelope, request } from './http.js';
 import { readInputs } from './inputs.js';
 import { hasLogin, loginToken } from './login.js';
 import { resourceRequest } from './resources.js';
+import { markDrift } from './project.js';
 import { describePlan, OUT_NOT_TRACKED, planSecret, removeRecordFiles, writeSecret } from './secrets.js';
 import { verbOf } from './schema.js';
 import { apiNouns, apiSchema, refreshSchemas, refusedBySchema } from './schemas.js';
@@ -53,7 +54,10 @@ async function executeOnce(argv, opts) {
     const schema = builtins[cmd.noun]?.schema ?? (await apiSchema(cmd.noun, Object.keys(builtins)));
     if (cmd.globals.help && !cmd.verb && !builtins[cmd.noun]?.defaultVerb)
         return envelope(nounHelp(schema));
-    const verb = verbOf(schema, cmd.verb ?? builtins[cmd.noun]?.defaultVerb);
+    const ext = opts.extensions?.[schema.noun];
+    const found = verbOf(schema, cmd.verb ?? builtins[cmd.noun]?.defaultVerb);
+    const local = ext?.flags?.[found.name];
+    const verb = local ? { ...found, flags: { ...found.flags, ...local } } : found;
     if (cmd.globals.help)
         return envelope(verbHelp(schema, verb));
     // A verb the schema lists as refused: no request, no confirmation for something that cannot happen.
@@ -61,7 +65,6 @@ async function executeOnce(argv, opts) {
         const by = verb.refused.by === 'commercelayer' ? 'Commerce Layer' : verb.refused.by;
         fail('not_supported', `c11r ${schema.noun} ${verb.name} is refused by ${by}: ${verb.description || 'it cannot be done through the API.'}`, { details: { refused_by: verb.refused.by }, next_step: { action: 'none' } });
     }
-    const ext = opts.extensions?.[schema.noun];
     const positional = ext?.positional?.[verb.name];
     const rest = positional && !verb.id && cmd.rest[0] !== undefined && !cmd.rest[0].startsWith('-') ? [`--${positional}`, ...cmd.rest] : cmd.rest;
     const parsed = parseFields(schema.noun, verb, rest);
@@ -140,7 +143,21 @@ async function executeOnce(argv, opts) {
     };
     const own = (await builtins[schema.noun]?.run?.(run)) ?? (await ext?.run?.(run));
     const r = own ?? (await run.send(resourceRequest(schema.noun, verb, parsed.id, parsed.values)));
-    return filesGone(secretOf(r, run, secretPlan), run);
+    return drifted(filesGone(secretOf(r, run, secretPlan), run), run);
+}
+/**
+ * A Configure write that went through, in a folder whose store.yml names this organization: the
+ * file is now behind it. Noted in .c11r/drift, with the warning config_drift.
+ */
+function drifted(r, run) {
+    if (run.globals.dryRun || !run.org.org || run.verb.method === 'GET')
+        return r;
+    if (run.schema.doc?.group !== 'configure')
+        return r;
+    const data = (r.data && typeof r.data === 'object' ? r.data : {});
+    const id = [run.parsed.id, data.code, data.name, data.id].find((k) => typeof k === 'string' && k !== '');
+    const warning = markDrift(run.org.org, { noun: run.schema.noun, verb: run.verb.name, id: id ?? null });
+    return warning ? { ...r, warnings: [...r.warnings, warning] } : r;
 }
 /**
  * A delete on a noun that answers secrets: the files c11r wrote for that record in the secrets

package/dist/project.js

@@ -0,0 +1,175 @@
+// The project folder's configuration files (contract section 45.2): store.yml and
+// price_lists/<code>.csv, written by config export and read by config plan and apply, and the
+// CLI's own notes about them in .c11r/:
+//
+//   .c11r/export.json  what the last export wrote: the organization, when, and each file's hash,
+//                      so the next export never overwrites a file changed since without --yes.
+//   .c11r/drift        the Configure writes made since, in a folder whose store.yml names the
+//                      same organization: config plan mentions them, config export clears them.
+//
+// store.yml carries the export's meta ({ org, exported_at }) as its first key; the CLI takes it
+// off before it sends the file, so the API sees { <noun>: [records] } only.
+import crypto from 'node:crypto';
+import fs from 'node:fs';
+import path from 'node:path';
+import YAML from 'yaml';
+import { projectDir } from './config.js';
+import { fail } from './errors.js';
+export const STORE_FILE = 'store.yml';
+export const PRICE_LISTS_DIR = 'price_lists';
+const NOTES_DIR = '.c11r';
+const EXPORT_FILE = 'export.json';
+const DRIFT_FILE = 'drift';
+const hashOf = (text) => `sha256-${crypto.createHash('sha256').update(text).digest('hex')}`;
+const notes = (dir) => path.join(dir, NOTES_DIR);
+/** store.yml and the price list CSVs of the project folder, or store_not_found. */
+export function readStoreFiles(dir = projectDir()) {
+    const file = path.join(dir, STORE_FILE);
+    let text;
+    try {
+        text = fs.readFileSync(file, 'utf8');
+    }
+    catch {
+        fail('not_found', `There is no ${STORE_FILE} in ${dir}. c11r config export writes one from the organization.`, { path: 'store', next_step: { action: 'fix_request', command: 'c11r config export' } });
+    }
+    let doc;
+    try {
+        doc = YAML.parse(text) ?? {};
+    }
+    catch (e) {
+        fail('invalid_config', `${file} is not YAML: ${e.message}`, { path: 'store' });
+    }
+    if (!doc || typeof doc !== 'object' || Array.isArray(doc))
+        fail('invalid_config', `${file} holds no nouns: it is { <noun>: [records] }.`, { path: 'store' });
+    const { meta, ...store } = doc;
+    const org = meta && typeof meta === 'object' && typeof meta.org === 'string' ? meta.org : null;
+    const price_lists = {};
+    const csvDir = path.join(dir, PRICE_LISTS_DIR);
+    let names = [];
+    try {
+        names = fs.readdirSync(csvDir);
+    }
+    catch {
+        // No price lists: fine.
+    }
+    for (const n of names.sort())
+        if (n.endsWith('.csv'))
+            price_lists[n.slice(0, -4)] = fs.readFileSync(path.join(csvDir, n), 'utf8');
+    return { store, price_lists, org };
+}
+/** The organization the folder's store.yml names, or null without one. */
+export function storeFileOrg(dir = projectDir()) {
+    try {
+        const doc = YAML.parse(fs.readFileSync(path.join(dir, STORE_FILE), 'utf8'));
+        return typeof doc?.meta?.org === 'string' ? doc.meta.org : null;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Writes an export: store.yml (meta first) and price_lists/<code>.csv. A file there that differs
+ * from what the last export wrote (or that no export wrote) is the user's: without overwrite,
+ * nothing is written and the answer lists them. Price list files the last export wrote, unchanged
+ * since, and no longer in the organization, go.
+ */
+export function writeStoreFiles(data, opts) {
+    const dir = opts.dir ?? projectDir();
+    const last = readRecord(dir);
+    const meta = { org: data.meta?.org ?? null, exported_at: data.meta?.exported_at ?? new Date().toISOString() };
+    const yaml = `# The organization's setup, written by c11r config export. Edit it, then c11r config plan and c11r config apply.\n${YAML.stringify({ meta, ...data.store }, { lineWidth: 0 })}`;
+    const files = { [STORE_FILE]: yaml };
+    for (const [code, csv] of Object.entries(data.price_lists ?? {}))
+        files[path.join(PRICE_LISTS_DIR, `${safeName(code)}.csv`)] = csv;
+    const changed = (rel) => {
+        const abs = path.join(dir, rel);
+        if (!fs.existsSync(abs))
+            return false;
+        const now = hashOf(fs.readFileSync(abs, 'utf8'));
+        return last.files[rel] !== now;
+    };
+    const stale = Object.keys(last.files).filter((rel) => rel.startsWith(`${PRICE_LISTS_DIR}${path.sep}`) && !(rel in files) && fs.existsSync(path.join(dir, rel)));
+    const mine = [...Object.keys(files).filter((rel) => files[rel] !== readOr(path.join(dir, rel))), ...stale].filter(changed);
+    if (mine.length && !opts.overwrite) {
+        fail('conflict', `${mine.join(', ')} changed since the last export${last.exported_at ? ` (${last.exported_at})` : ''}, or no export wrote ${mine.length === 1 ? 'it' : 'them'}. c11r replaces ${mine.length === 1 ? 'it' : 'them'} only with --yes; c11r config plan shows what the files would change in the organization.`, {
+            path: 'store',
+            details: { files: mine.map((rel) => path.join(dir, rel)) },
+            next_step: { action: 'fix_request', command: 'c11r config plan' },
+        }, 'rejected');
+    }
+    const written = [];
+    for (const [rel, text] of Object.entries(files)) {
+        const abs = path.join(dir, rel);
+        fs.mkdirSync(path.dirname(abs), { recursive: true });
+        if (readOr(abs) !== text)
+            fs.writeFileSync(abs, text);
+        written.push(abs);
+    }
+    const removed = [];
+    for (const rel of stale) {
+        fs.rmSync(path.join(dir, rel), { force: true });
+        removed.push(path.join(dir, rel));
+    }
+    writeRecord(dir, { org: meta.org, exported_at: meta.exported_at, files: Object.fromEntries(Object.entries(files).map(([rel, text]) => [rel, hashOf(text)])) });
+    clearDrift(dir);
+    return { written, removed, dir };
+}
+/**
+ * After a Configure write: in a folder whose store.yml names this organization, notes it in
+ * .c11r/drift and answers the warning. Elsewhere, nothing.
+ */
+export function markDrift(org, entry, dir = projectDir()) {
+    if (storeFileOrg(dir) !== org)
+        return null;
+    const list = readDrift(dir);
+    list.push({ ...entry, at: new Date().toISOString() });
+    fs.mkdirSync(notes(dir), { recursive: true });
+    // The first ones say since when; the last 50 say what.
+    const kept = list.length > 50 ? [list[0], ...list.slice(-49)] : list;
+    fs.writeFileSync(path.join(notes(dir), DRIFT_FILE), JSON.stringify(kept, null, 2) + '\n');
+    return driftWarning(kept);
+}
+/** The writes noted since the last export, oldest first. */
+export function readDrift(dir = projectDir()) {
+    try {
+        const v = JSON.parse(fs.readFileSync(path.join(notes(dir), DRIFT_FILE), 'utf8'));
+        return Array.isArray(v) ? v : [];
+    }
+    catch {
+        return [];
+    }
+}
+export function clearDrift(dir = projectDir()) {
+    fs.rmSync(path.join(notes(dir), DRIFT_FILE), { force: true });
+}
+export function driftWarning(list) {
+    const first = list[0];
+    const what = (e) => `${e.noun} ${e.verb}${e.id ? ` ${e.id}` : ''}`;
+    return {
+        code: 'config_drift',
+        message: `store.yml is now behind the organization, since ${first.at} (${what(first)}${list.length > 1 ? ` and ${list.length - 1} more` : ''}): c11r config export refreshes it.`,
+    };
+}
+function readRecord(dir) {
+    try {
+        const v = JSON.parse(fs.readFileSync(path.join(notes(dir), EXPORT_FILE), 'utf8'));
+        return { org: v.org ?? null, exported_at: v.exported_at ?? null, files: v.files && typeof v.files === 'object' ? v.files : {} };
+    }
+    catch {
+        return { org: null, exported_at: null, files: {} };
+    }
+}
+function writeRecord(dir, r) {
+    fs.mkdirSync(notes(dir), { recursive: true });
+    fs.writeFileSync(path.join(notes(dir), EXPORT_FILE), JSON.stringify(r, null, 2) + '\n');
+}
+function readOr(file) {
+    try {
+        return fs.readFileSync(file, 'utf8');
+    }
+    catch {
+        return null;
+    }
+}
+/** A price list code as a file name: letters, digits, dot, dash and underscore. */
+const safeName = (code) => code.replace(/[^A-Za-z0-9._-]+/g, '-');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@commercelayer/agent-cli",
-  "version": "1.0.0-dev.8",
+  "version": "1.0.0-dev.9",
   "description": "c11r: the Commerce Layer CLI for coding agents. One grammar, c11r <noun> <verb>, from a sandbox to a running organization.",
   "homepage": "https://commercelayer.ai",
   "keywords": [