@datumos/datum-cli 0.0.0 → 0.588.0

package/cli/spaceManifestDir.js

@@ -0,0 +1,481 @@
+/**
+ * The repository layout of a space manifest (#1809, docs/space-admin/space-repository.md):
+ * how one manifest document (`apiVersion: datum/v1`, docs/terraform.md)
+ * maps onto files a person reviews in a pull request, and back — for
+ * `datum space-manifest pull|push <dir>`.
+ *
+ *   <dir>/space.json                   apiVersion + every scalar kind
+ *   <dir>/<kind>.json                  a map kind (settings, github, jira, …)
+ *   <dir>/<kind>/<entry>.json          one entry of a list kind (agents, routines, …)
+ *   <dir>/<kind>/<entry>.<field>.md    a long text field of that entry (prompts)
+ *   <dir>/<kind>/<entry>.<field>.json  a large JSON field (schemas, flow definitions)
+ *
+ * An extracted field is referenced from its entry as `{ "$file": "<name>" }`.
+ * Pure functions over an in-memory file map (`{ "relative/path": "text" }`),
+ * so the round trip is testable without a disk or a server.
+ *
+ * The kind table below MIRRORS the server's manifest kinds
+ * (`server/src/manifest/kinds`): the published CLI package ships `cli/` and
+ * the spec alone and cannot import the engine, so `spaceManifestDir.test.ts`
+ * pins every entry (tier, shape, natural key) to the engine's kinds — a new
+ * kind fails that test until it is added here.
+ */
+export const SPACE_MANIFEST_KINDS = {
+    features: { tier: "org", shape: "map" },
+    monthlyBudgetUsd: { tier: "org", shape: "scalar" },
+    allocation: { tier: "org", shape: "map" },
+    members: { tier: "org", shape: "list", key: "user" },
+    systemGrants: { tier: "org", shape: "map" },
+    catalogBaseline: { tier: "org", shape: "scalar" },
+    customerGrants: { tier: "org", shape: "map" },
+    directoryGrants: { tier: "org", shape: "map" },
+    groupBindings: { tier: "org", shape: "map" },
+    teamGrants: { tier: "org", shape: "map" },
+    autonomyOptIn: { tier: "space", shape: "scalar" },
+    skillsRepoUrl: { tier: "space", shape: "scalar" },
+    settings: { tier: "space", shape: "map" },
+    credentials: { tier: "space", shape: "list", key: "label" },
+    credentialBindings: { tier: "space", shape: "map" },
+    accHosts: { tier: "space", shape: "list", key: "vhost" },
+    browserProfiles: { tier: "space", shape: "list", key: "name" },
+    observability: { tier: "space", shape: "list", key: "slug" },
+    github: { tier: "space", shape: "map" },
+    jira: { tier: "space", shape: "map" },
+    victorops: { tier: "space", shape: "map" },
+    grafanaAlerts: { tier: "space", shape: "map" },
+    teams: { tier: "space", shape: "map" },
+    slack: { tier: "space", shape: "map" },
+    queues: { tier: "space", shape: "list", key: "name" },
+    caseBoards: { tier: "space", shape: "list", key: "name" },
+    caseSteps: { tier: "space", shape: "list", key: "kind" },
+    caseScoreRules: { tier: "space", shape: "list", key: "name" },
+    chatEmbeds: { tier: "space", shape: "list", key: "slug" },
+    projects: { tier: "space", shape: "list", key: "key" },
+    accessPolicies: { tier: "space", shape: "list", key: "id" },
+    approvalPolicies: { tier: "space", shape: "list", key: "kind" },
+    agents: { tier: "space", shape: "list", key: "id", files: { prompt: "md" } },
+    routines: {
+        tier: "space",
+        shape: "list",
+        key: "name",
+        files: { prompt: "md", inputSchema: "json", outputSchema: "json" },
+    },
+    flows: { tier: "space", shape: "list", key: "name", files: { definition: "json" } },
+    flowReleases: { tier: "space", shape: "list", key: "flow" },
+    flowInstalls: { tier: "space", shape: "list", key: "name" },
+    spaceTools: { tier: "space", shape: "list", key: "name" },
+    toolInstalls: { tier: "space", shape: "list", key: "name" },
+    remoteMcpServers: { tier: "space", shape: "list", key: "label" },
+    spaceVariables: { tier: "space", shape: "list", key: "key" },
+    dataProducts: { tier: "space", shape: "list", key: "slug" },
+    artifacts: { tier: "space", shape: "list", key: "key" },
+    artifactCollections: { tier: "space", shape: "list", key: "name" },
+    knowledgeBases: { tier: "space", shape: "list", key: "name" },
+    eventSubscriptions: { tier: "space", shape: "list", key: "name" },
+    machinePools: { tier: "space", shape: "list", key: "name" },
+};
+export const SPACE_FILE = "space.json";
+const FILE_REF = "$file";
+const API_VERSION = "datum/v1";
+/** Response envelope fields of GET /api/space-manifest — not kinds. */
+const ENVELOPE_FIELDS = new Set(["spaceId"]);
+/** A problem with the files, naming the file. */
+export class SpaceManifestLayoutError extends Error {
+}
+/** A kind the CLI does not know yet (a newer server): handled by its
+ * runtime shape, keyed by "name" — `checkManifest` warns about it. */
+function layoutOf(kind, value) {
+    const known = SPACE_MANIFEST_KINDS[kind];
+    if (known)
+        return known;
+    const shape = Array.isArray(value) ? "list" : value !== null && typeof value === "object" ? "map" : "scalar";
+    return { tier: "space", shape, ...(shape === "list" ? { key: "name" } : {}) };
+}
+/**
+ * A file-system-safe, readable name for an entry key: lower-case ASCII
+ * (Polish letters fold), anything outside [a-z0-9_-] — the DOT included —
+ * becomes a dash. No dot in an entry's base name is what keeps the layout
+ * unambiguous: `<base>.json` is always an entry and `<base>.<field>.<ext>`
+ * always an extracted field, so a flow named `demo.definition` can never
+ * land on the definition file of the flow `demo` — on a case-insensitive
+ * file system neither (#1809 e2e).
+ */
+export function slugify(key) {
+    const folded = String(key)
+        .replace(/ł/g, "l")
+        .replace(/Ł/g, "L")
+        .normalize("NFKD")
+        .replace(/[̀-ͯ]/g, "");
+    const slug = folded
+        .toLowerCase()
+        .replace(/[^a-z0-9_-]+/g, "-")
+        .replace(/^-+|-+$/g, "")
+        .replace(/-{2,}/g, "-");
+    return slug || "entry";
+}
+/** An entry's own file: a base name without a dot, then `.json`. */
+const ENTRY_FILE_RE = /^[^.]+\.json$/;
+export function jsonFile(value) {
+    return `${JSON.stringify(value, null, 2)}\n`;
+}
+/** Text files end with exactly one added newline, which reading strips —
+ * lossless for any string, and an editor's final newline is not a change. */
+function textFile(value) {
+    return `${value}\n`;
+}
+function readText(content) {
+    return content.endsWith("\n") ? content.slice(0, -1) : content;
+}
+function isEmpty(value) {
+    if (Array.isArray(value))
+        return value.length === 0;
+    if (value !== null && typeof value === "object")
+        return Object.keys(value).length === 0;
+    return false;
+}
+function byKey(entries, key) {
+    return [...entries].sort((a, b) => String(a[key]).localeCompare(String(b[key]), "en"));
+}
+/** Splits a live manifest (the body of GET /api/space-manifest) into files. */
+export function splitManifest(manifest) {
+    const files = {};
+    const skipped = [];
+    const scalars = { apiVersion: manifest.apiVersion ?? API_VERSION };
+    for (const [kind, value] of Object.entries(manifest)) {
+        if (kind === "apiVersion" || ENVELOPE_FIELDS.has(kind))
+            continue;
+        const layout = SPACE_MANIFEST_KINDS[kind];
+        if (!layout) {
+            // Fail closed: a kind a newer server added may be organisation-tier
+            // or keyed differently — never export what this CLI cannot classify.
+            skipped.push({ kind, reason: "unknown to this CLI — run `datum update`, then pull again" });
+            continue;
+        }
+        if (layout.tier === "org") {
+            skipped.push({ kind, reason: "organisation tier — the org admin manages it" });
+            continue;
+        }
+        if (isEmpty(value)) {
+            // Left out = unmanaged; nothing exists to review.
+            skipped.push({ kind, reason: "empty in the space" });
+            continue;
+        }
+        if (Array.isArray(value)) {
+            Object.assign(files, splitList(kind, layout, value));
+        }
+        else if (value !== null && typeof value === "object") {
+            files[`${kind}.json`] = jsonFile(value);
+        }
+        else {
+            scalars[kind] = value;
+        }
+    }
+    files[SPACE_FILE] = jsonFile(scalars);
+    return { files, skipped };
+}
+function splitList(kind, layout, entries) {
+    const files = {};
+    const key = layout.key ?? "name";
+    const used = new Set();
+    // An entry whose key already IS its slug owns the bare name; the others
+    // take suffixes in key order — stable, so a second pull renames nothing.
+    const sorted = byKey(entries, key);
+    const exact = sorted.filter((e) => slugify(e[key]) === String(e[key]));
+    for (const entry of exact)
+        used.add(String(entry[key]));
+    for (const entry of sorted) {
+        let base = slugify(entry[key]);
+        if (!exact.includes(entry)) {
+            for (let n = 2; used.has(base); n += 1)
+                base = `${slugify(entry[key])}-${n}`;
+            used.add(base);
+        }
+        const body = { ...entry };
+        for (const [field, ext] of Object.entries(layout.files ?? {})) {
+            const value = body[field];
+            if (value === undefined || value === null)
+                continue;
+            if (ext === "md" && typeof value !== "string")
+                continue;
+            const name = `${base}.${field}.${ext}`;
+            files[`${kind}/${name}`] = ext === "md" ? textFile(value) : jsonFile(value);
+            body[field] = { [FILE_REF]: name };
+        }
+        files[`${kind}/${base}.json`] = jsonFile(body);
+    }
+    return files;
+}
+/**
+ * True for a path the layout owns — what `pull` may replace and `push`
+ * reads: `space.json`, `<kind>.json` of a known kind, and `.json`/`.md`
+ * files directly inside a known LIST kind's folder. Anything else in the
+ * directory (a README, scripts/, package.json) is never read or touched.
+ * `extraKinds`: kinds the live space reported that this CLI does not know.
+ */
+export function isLayoutPath(path, extraKinds = new Set()) {
+    const known = (kind) => kind in SPACE_MANIFEST_KINDS || extraKinds.has(kind);
+    const parts = path.split("/");
+    if (parts.length === 1) {
+        return path === SPACE_FILE || (path.endsWith(".json") && known(path.slice(0, -".json".length)));
+    }
+    if (parts.length !== 2)
+        return false;
+    const [kind, name] = parts;
+    const list = SPACE_MANIFEST_KINDS[kind] ? SPACE_MANIFEST_KINDS[kind].shape === "list" : extraKinds.has(kind);
+    return list && !name.startsWith(".") && (name.endsWith(".json") || name.endsWith(".md"));
+}
+/** Assembles the manifest back from files. Throws SpaceManifestLayoutError
+ * naming the file for anything that cannot be read unambiguously. */
+export function buildManifest(files) {
+    const spaceFile = files[SPACE_FILE];
+    if (spaceFile === undefined)
+        throw new SpaceManifestLayoutError(`${SPACE_FILE} is missing`);
+    const manifest = parseJson(SPACE_FILE, spaceFile);
+    if (manifest === null || typeof manifest !== "object" || Array.isArray(manifest)) {
+        throw new SpaceManifestLayoutError(`${SPACE_FILE}: must be a JSON object`);
+    }
+    const doc = manifest;
+    if (doc.apiVersion !== API_VERSION) {
+        throw new SpaceManifestLayoutError(`${SPACE_FILE}: apiVersion must be "${API_VERSION}"`);
+    }
+    const lists = {};
+    const referenced = new Set();
+    for (const path of Object.keys(files).sort()) {
+        if (path === SPACE_FILE)
+            continue;
+        const parts = path.split("/");
+        if (parts.length === 1) {
+            if (!path.endsWith(".json"))
+                continue;
+            const kind = path.slice(0, -".json".length);
+            if (kind in doc)
+                throw new SpaceManifestLayoutError(`${path}: kind "${kind}" is declared twice`);
+            doc[kind] = parseJson(path, files[path]);
+            continue;
+        }
+        if (parts.length !== 2) {
+            throw new SpaceManifestLayoutError(`${path}: list entries live directly under <kind>/`);
+        }
+        const [kind, name] = parts;
+        // `<base>.json` (no dot in the base) is an entry; every other file is a
+        // field an entry references through `$file` and is read there.
+        if (!ENTRY_FILE_RE.test(name))
+            continue;
+        const entry = parseJson(path, files[path]);
+        if (entry === null || typeof entry !== "object" || Array.isArray(entry)) {
+            throw new SpaceManifestLayoutError(`${path}: an entry must be a JSON object`);
+        }
+        resolveRefs(path, entry, files, referenced);
+        (lists[kind] ??= []).push({ path, entry: entry });
+    }
+    // A file in a kind's folder is either an entry or a field an entry
+    // references. Anything else (`reviewer.v2.json`, a leftover prompt) would
+    // silently drop out of the manifest — refuse it instead.
+    for (const path of Object.keys(files)) {
+        const parts = path.split("/");
+        if (parts.length === 2 && !ENTRY_FILE_RE.test(parts[1]) && !referenced.has(path)) {
+            throw new SpaceManifestLayoutError(`${path}: neither an entry (<name>.json, no dot in the name) nor a file an entry references with "$file" — rename or remove it`);
+        }
+    }
+    for (const [kind, entries] of Object.entries(lists)) {
+        if (kind in doc)
+            throw new SpaceManifestLayoutError(`${kind}/: kind "${kind}" is declared twice`);
+        const key = layoutOf(kind, entries).key ?? "name";
+        const seen = new Map();
+        for (const { path, entry } of entries) {
+            const value = entry[key];
+            if (value === undefined || value === null || value === "") {
+                throw new SpaceManifestLayoutError(`${path}: the entry has no "${key}" (its natural key)`);
+            }
+            const other = seen.get(value);
+            if (other !== undefined) {
+                throw new SpaceManifestLayoutError(`${path}: "${String(value)}" is declared twice (also in ${other})`);
+            }
+            seen.set(value, path);
+        }
+        doc[kind] = byKey(entries.map((e) => e.entry), key);
+    }
+    return doc;
+}
+function resolveRefs(path, entry, files, referenced) {
+    const dir = path.slice(0, path.lastIndexOf("/") + 1);
+    for (const [field, value] of Object.entries(entry)) {
+        if (value === null || typeof value !== "object" || Array.isArray(value) || !(FILE_REF in value))
+            continue;
+        const ref = value[FILE_REF];
+        if (typeof ref !== "string" || ref.includes("/") || ref.includes("\\") || ref.startsWith(".")) {
+            throw new SpaceManifestLayoutError(`${path}: ${field}.${FILE_REF} must name a file next to the entry`);
+        }
+        const target = `${dir}${ref}`;
+        const content = files[target];
+        if (content === undefined) {
+            throw new SpaceManifestLayoutError(`${path}: ${field} points at ${target}, which does not exist`);
+        }
+        referenced.add(target);
+        entry[field] = ref.endsWith(".json") ? parseJson(target, content) : readText(content);
+    }
+}
+function parseJson(path, content) {
+    try {
+        return JSON.parse(content);
+    }
+    catch (err) {
+        throw new SpaceManifestLayoutError(`${path}: not valid JSON (${err.message})`);
+    }
+}
+// ------------------------------------------------------------------ checks
+/** Shapes of key material that must never sit in a repository: a
+ * credential is declared by label + type only, its material is filled in
+ * the panel (docs/terraform.md "Secrets never travel in a manifest"). */
+export const SECRET_PATTERNS = [
+    { name: "Datum API key", re: /\bdatum_[A-Za-z0-9_-]{24,}/ },
+    { name: "Anthropic API key", re: /sk-ant-[A-Za-z0-9_-]{20,}/ },
+    { name: "OpenAI API key", re: /\bsk-(?:proj-)?[A-Za-z0-9_-]{32,}/ },
+    { name: "GitHub token", re: /\b(?:ghp|gho|ghs|ghu|github_pat)_[A-Za-z0-9_]{20,}/ },
+    { name: "AWS access key id", re: /\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/ },
+    { name: "Slack token", re: /\bxox[abprs]-[A-Za-z0-9-]{10,}/ },
+    { name: "private key block", re: /-----BEGIN [A-Z ]*PRIVATE KEY-----/ },
+    { name: "JWT", re: /\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}/ },
+    { name: "Grafana service-account token", re: /\bglsa_[A-Za-z0-9_]{20,}/ },
+    { name: "Slack webhook URL", re: /hooks\.slack\.com\/services\/T[A-Z0-9]+\/B[A-Z0-9]+\/[A-Za-z0-9]{10,}/ },
+    { name: "Atlassian API token", re: /\bATATT[A-Za-z0-9_=-]{20,}/ },
+    { name: "npm token", re: /\bnpm_[A-Za-z0-9]{30,}/ },
+    { name: "AWS secret access key", re: /aws_?secret_?access_?key["'\s:=]+[A-Za-z0-9/+]{40}/i },
+    { name: "password assignment", re: /\b(?:password|passwd|pwd)\s*[:=]\s*["']?[^\s"'`]{6,}/i },
+];
+/** Offline checks before anything is sent: organisation-tier kinds (the
+ * space surface refuses them anyway) and secret material (which must never
+ * even travel). The server's dry run stays the authority on the rest. */
+export function checkManifest(manifest) {
+    const errors = [];
+    const warnings = [];
+    for (const kind of Object.keys(manifest)) {
+        if (kind === "apiVersion")
+            continue;
+        const layout = SPACE_MANIFEST_KINDS[kind];
+        if (!layout) {
+            warnings.push(`${kind}: a kind this CLI does not know — run \`datum update\``);
+        }
+        else if (layout.tier === "org") {
+            errors.push(`${kind}: an organisation-tier kind — the space surface refuses it; the org admin manages it`);
+        }
+    }
+    for (const [kind, value] of Object.entries(manifest)) {
+        // A list entry is named by its natural key — what its file is named
+        // after — not by an index the person never sees.
+        const key = layoutOf(kind, value).key ?? "name";
+        const items = Array.isArray(value)
+            ? value.map((e, i) => [`${kind}[${JSON.stringify(e?.[key] ?? i)}]`, e])
+            : [[kind, value]];
+        for (const [label, item] of items) {
+            walkStrings(item, label, (path, text) => {
+                for (const { name, re } of SECRET_PATTERNS) {
+                    if (re.test(text))
+                        errors.push(`${path}: looks like a ${name} — secrets never go into a manifest`);
+                }
+            });
+        }
+    }
+    return { errors, warnings };
+}
+function walkStrings(value, path, onString) {
+    if (typeof value === "string")
+        onString(path || "(root)", value);
+    else if (Array.isArray(value))
+        value.forEach((item, i) => walkStrings(item, `${path}[${i}]`, onString));
+    else if (value !== null && typeof value === "object") {
+        for (const [key, item] of Object.entries(value))
+            walkStrings(item, path ? `${path}.${key}` : key, onString);
+    }
+}
+const WRITE_OPS = new Set(["create", "update", "delete"]);
+const OP_ORDER = ["create", "update", "delete", "orphan", "noop"];
+const OP_LABEL = {
+    create: "+ create",
+    update: "~ update",
+    delete: "- delete",
+    orphan: "? orphan",
+    noop: "= noop",
+};
+const MAX_VALUE = 160;
+const CONTEXT_LINES = 1;
+const MAX_DIFF_LINES = 40;
+/** The writes the plan makes (create/update/delete) — orphans and noops are not. */
+export function countWrites(answer) {
+    return (answer.plan ?? []).filter((op) => WRITE_OPS.has(op.op)).length;
+}
+/** The server's answer as the Markdown a pull-request comment carries. */
+export function renderPlan(answer, title) {
+    const plan = answer.plan ?? [];
+    const counts = {};
+    for (const op of plan)
+        counts[op.op] = (counts[op.op] ?? 0) + 1;
+    const lines = [`## ${title}`, ""];
+    lines.push(answer.applied ? "**Applied.**" : "**Dry run** — nothing was written.", "");
+    lines.push(OP_ORDER.filter((op) => (counts[op] ?? 0) > 0)
+        .map((op) => `${counts[op]} ${op}`)
+        .join(" · ") || "Empty plan.");
+    const changes = plan.filter((op) => op.op !== "noop");
+    if (changes.length > 0) {
+        lines.push("", "```diff");
+        for (const op of sortOps(changes)) {
+            lines.push(`${OP_LABEL[op.op] ?? op.op} ${op.kind}/${op.name}`);
+            for (const [field, change] of Object.entries(op.changes ?? {})) {
+                if (isMultiline(change.before) || isMultiline(change.after)) {
+                    lines.push(`    ${field}:`, ...lineDiff(String(change.before ?? ""), String(change.after ?? "")));
+                }
+                else {
+                    lines.push(`    ${field}: ${short(change.before)} → ${short(change.after)}`);
+                }
+            }
+        }
+        lines.push("```");
+    }
+    if ((counts.orphan ?? 0) > 0) {
+        lines.push("", "_Orphans exist in the space but not in the files. They are left alone; add them to the files, or delete them on purpose with `--prune`._");
+    }
+    const warnings = answer.warnings ?? [];
+    if (warnings.length > 0)
+        lines.push("", "**Warnings**", "", ...warnings.map((w) => `- ${w}`));
+    return `${lines.join("\n")}\n`;
+}
+function sortOps(ops) {
+    return [...ops].sort((a, b) => OP_ORDER.indexOf(a.op) - OP_ORDER.indexOf(b.op) ||
+        a.kind.localeCompare(b.kind) ||
+        String(a.name).localeCompare(String(b.name)));
+}
+function isMultiline(value) {
+    return typeof value === "string" && value.includes("\n");
+}
+/** A prompt edit as the changed lines (common head and tail trimmed, one
+ * line of context) — not the first characters of two near-identical prompts. */
+function lineDiff(before, after) {
+    const a = before.split("\n");
+    const b = after.split("\n");
+    let head = 0;
+    while (head < a.length && head < b.length && a[head] === b[head])
+        head += 1;
+    let tail = 0;
+    while (tail < a.length - head && tail < b.length - head && a[a.length - 1 - tail] === b[b.length - 1 - tail]) {
+        tail += 1;
+    }
+    const out = [];
+    for (let i = Math.max(0, head - CONTEXT_LINES); i < head; i += 1)
+        out.push(`      ${a[i]}`);
+    for (let i = head; i < a.length - tail; i += 1)
+        out.push(`-     ${a[i]}`);
+    for (let i = head; i < b.length - tail; i += 1)
+        out.push(`+     ${b[i]}`);
+    for (let i = a.length - tail; i < Math.min(a.length, a.length - tail + CONTEXT_LINES); i += 1) {
+        out.push(`      ${a[i]}`);
+    }
+    return out.length > MAX_DIFF_LINES
+        ? [...out.slice(0, MAX_DIFF_LINES), `      … ${out.length - MAX_DIFF_LINES} more lines`]
+        : out;
+}
+function short(value) {
+    const text = value === undefined ? "(absent)" : JSON.stringify(value);
+    const oneLine = text.replace(/\\n/g, " ").replace(/\s+/g, " ");
+    return oneLine.length > MAX_VALUE ? `${oneLine.slice(0, MAX_VALUE - 1)}…` : oneLine;
+}
+//# sourceMappingURL=spaceManifestDir.js.map