@tenonhq/dovetail-servicenow 0.0.37 → 0.0.39

package/README.md

@@ -555,6 +555,71 @@ Store `appLink`, and the publish's update-set sys_id where the instance reports
 one. Exit codes: `0` published or dry-run, `1` bad args/unconfirmed, `2`
 failed/timeout. Programmatic: `publishApp({ app, version, target, confirm })`.
 
+### Export an update set (or a whole app) to importable XML
+
+Produce the `<unload>` document the implementation team imports on a customer
+instance — with every secret value replaced by `__SET_DURING_INSTALL__`.
+
+```bash
+# An update set. assemble mode is READ-ONLY: nothing on the instance changes.
+npx dove-sn export-update-set --update-set 0123456789abcdef0123456789abcdef \
+  --out ./tenon-core.xml
+
+# A whole app: publish into a new set, then export it. Dry-run first (default).
+npx dove-sn export-app --app x_cadso_automate --out ./automate.xml
+npx dove-sn export-app --app x_cadso_automate --out ./automate.xml --confirm
+
+# A document exported some other way
+npx dove-sn strip-secrets --in ./exported.xml --out ./safe.xml
+npx dove-sn strip-secrets --in ./exported.xml --report      # what would be stripped
+```
+
+**Secret stripping is not optional.** There is no `--no-strip` flag on any of
+these verbs, and no field in the MCP schemas that disables it. This matters
+because an unload carries field *values*: on tenonworkstudio, completed update
+sets hold filled-in values for `password2` system properties and
+`oauth_entity.client_secret`, several of them in sets that shipped to a
+customer. A field is exempted only by a reviewed entry in the rules file.
+
+The rule is enumerable, in four layers:
+
+| Layer | Covers |
+|---|---|
+| L1 type | `password` / `password2` fields, restricted to tables an update set can capture (`update_synch=true`), resolved through `super_class`. Refreshed from the live dictionary per run, with a committed baseline as the fallback. |
+| L2 conditional / explicit | `sys_properties.value` when the property's `type` is a password type; named exceptions such as `x_cadso_core.google_translate_api_key`, which holds an API key in a *string* column. |
+| L3 JSON | secrets nested inside a JSON blob field, stripped in place. |
+| L4 heuristic | a field that merely *looks* secret. **Never stripped silently and never assumed safe** — the run fails and names it, until a human records it in the rules file as a strip rule or as `notSecret` with a reason. |
+
+Override the rules with `--rules <file>`; the JSON is merged over the built-ins,
+and the only subtractive key is `notSecret`, which requires a reason:
+
+```json
+{
+  "notSecret": [
+    { "table": "x_cadso_core_thing", "field": "webhook_token", "reason": "public identifier, not a credential" }
+  ],
+  "fieldRules": [
+    { "id": "thing-signing-key", "table": "x_cadso_core_thing", "field": "signing_key", "reason": "inbound webhook HMAC key" }
+  ]
+}
+```
+
+Two more things refuse to produce a file rather than produce a wrong one: a
+record count that does not match the set, and the documented **in-progress
+empty 200** from `export_update_set.do` (the servlet streams a document only for
+a *complete* set, and app-publish leaves the set in progress). After stripping,
+the output is re-read and verified; a secret that somehow survived fails the run.
+
+`export-update-set --mode complete` marks the set complete on the instance
+first — a real write, so it needs `--confirm`. `export-app` publishes ~1000+
+records into a new update set and is dry-run by default. Neither is the Store
+publish; that is `publish-app`, which is externally visible.
+
+Exit codes: `0` exported or dry-run, `1` bad args/unconfirmed, `2`
+failed/timeout. Programmatic: `exportUpdateSet({ updateSet, mode })`,
+`exportApp({ app, confirm })`, `stripSecrets(xml, rules)`. MCP:
+`update_set_export` (read-only in assemble mode) and `app_export`.
+
 `test-flow` defaults to **validate** — a safe pre-flight (published? inputs match
 declared variables?) that never runs the flow; `--execute --confirm` runs it via
 the server-side FlowAPI runner (deploy `resources/runFlow.md` first).

package/dist/cli.js

@@ -69,6 +69,10 @@ const formLayout_1 = require("./layout/formLayout");
 const relatedLists_1 = require("./layout/relatedLists");
 const formatter_2 = require("./layout/formatter");
 const server_1 = require("./mcp/server");
+const exportUpdateSet_1 = require("./exportUpdateSet");
+const exportApp_1 = require("./exportApp");
+const stripSecrets_1 = require("./secrets/stripSecrets");
+const secretRules_1 = require("./secrets/secretRules");
 const schemas_1 = require("./mcp/schemas");
 const buildFlowOrchestrator_1 = require("./flowDesigner/buildFlowOrchestrator");
 const flowDesigner_formatter_1 = require("./flowDesigner-formatter");
@@ -1048,10 +1052,29 @@ function printHelp() {
         "                      [--dry-run] [--json] [--confirm])\n" +
         "                     Store creds: SN_STORE_USERNAME/SN_STORE_PASSWORD in the --env file;\n" +
         "                     the password is never a flag. Repo publish needs the sn_cicd role.\n" +
+        "  export-update-set  Export an update set to importable <unload> XML, with secret\n" +
+        "                     values replaced by __SET_DURING_INSTALL__ (no opt-out).\n" +
+        "                     assemble mode is READ-ONLY; complete mode marks the set\n" +
+        "                     complete on the instance and needs --confirm.\n" +
+        "                     (--update-set <sys_id|name> --out <file>\n" +
+        "                      [--mode assemble|complete]\n" +
+        "                      [--rules <file>] [--page-size <n>] [--max-rows <n>]\n" +
+        "                      [--dry-run] [--json] [--confirm])\n" +
+        "  export-app         Publish a scoped app into a new update set and export it.\n" +
+        "                     PUBLISHING IS A REAL INSTANCE WRITE (~1000+ records).\n" +
+        "                     DRY-RUN BY DEFAULT — nothing is published without --confirm\n" +
+        "                     (--app <scope|sys_id|name> --out <file> [--version <v>]\n" +
+        "                      [--description <text>] [--include-data] [--rules <file>]\n" +
+        "                      [--timeout-ms <n>] [--dry-run] [--json] [--confirm])\n" +
+        "  strip-secrets      Strip secret values from an unload XML exported elsewhere\n" +
+        "                     (--in <file> [--out <file>] [--rules <file>] [--report] [--json])\n" +
         "  mcp                Run the MCP stdio server (--smoke lists tools and exits)\n" +
         "\nGlobal flags:\n" +
-        "  --env <path>       Load credentials from a specific .env file (also --env-file,\n" +
-        "                     or the DOVETAIL_ENV_FILE env var). Default: .env in the cwd.\n");
+        "  --env <name|path>  Load credentials from a specific env file (also --env-file,\n" +
+        "                     or the DOVETAIL_ENV_FILE env var). A bare name like 'prod'\n" +
+        "                     resolves to .env.prod in the cwd. The file's SN_* connection\n" +
+        "                     vars replace any already exported; a missing or incomplete\n" +
+        "                     file is an error (no fallback). Default: .env in the cwd.\n");
 }
 /** Parse inline `--columns "Label:type:max, Other:choice, ..."` into ColumnSpec[]. */
 function parseColumnsInline(input) {
@@ -1439,7 +1462,9 @@ async function runSetTable(flags, bare) {
     var stringFlags = ["table", "update-set", "updateSetSysId"];
     for (var f = 0; f < stringFlags.length; f += 1) {
         if (bare[stringFlags[f]]) {
-            process.stderr.write("set-table: --" + stringFlags[f] + " needs a value (it was given none).\n");
+            process.stderr.write("set-table: --" +
+                stringFlags[f] +
+                " needs a value (it was given none).\n");
             return 1;
         }
     }
@@ -1902,6 +1927,232 @@ async function runPublishApp(flags) {
     }
     return exitCode;
 }
+/**
+ * dove-sn export-update-set:
+ *   --update-set <sys_id|name>  Required. The set to export.
+ *   [--mode assemble|complete]  assemble (default) is READ-ONLY; complete marks
+ *                               the set complete on the instance first, which is
+ *                               a real write and needs --confirm.
+ *   [--out <file>]              Write the XML here (default: stdout is NOT used —
+ *                               a document this size belongs in a file).
+ *   [--rules <file>]            JSON overrides for the secret rules.
+ *   [--page-size <n>] [--max-rows <n>]
+ *   [--dry-run] [--json] [--confirm]
+ *
+ * Secret values are ALWAYS replaced with __SET_DURING_INSTALL__; there is no
+ * opt-out flag. A field that looks secret and is covered by no rule fails the
+ * run, and nothing is written.
+ * Exit codes: 0 exported/dry-run, 1 bad args/unconfirmed, 2 failed.
+ */
+async function runExportUpdateSet(flags) {
+    var selector = flags["update-set"];
+    if (!selector) {
+        process.stderr.write("export-update-set: --update-set <sys_id|name> is required\n");
+        return 1;
+    }
+    var mode = flags.mode || "assemble";
+    if (mode !== "assemble" && mode !== "complete") {
+        process.stderr.write("export-update-set: --mode must be assemble or complete\n");
+        return 1;
+    }
+    var outPath = flags.out;
+    if (!outPath && flags["dry-run"] !== "true") {
+        process.stderr.write("export-update-set: --out <file> is required for a real export\n");
+        return 1;
+    }
+    var pageSize = undefined;
+    if (flags["page-size"]) {
+        pageSize = Number(flags["page-size"]);
+        if (!Number.isInteger(pageSize) || pageSize < 1) {
+            process.stderr.write("export-update-set: --page-size must be a positive integer\n");
+            return 1;
+        }
+    }
+    var maxRows = undefined;
+    if (flags["max-rows"]) {
+        maxRows = Number(flags["max-rows"]);
+        if (!Number.isInteger(maxRows) || maxRows < 1) {
+            process.stderr.write("export-update-set: --max-rows must be a positive integer\n");
+            return 1;
+        }
+    }
+    var result = await (0, exportUpdateSet_1.exportUpdateSet)({
+        updateSet: selector,
+        mode: mode,
+        confirm: flags.confirm === "true",
+        dryRun: flags["dry-run"] === "true",
+        rulesPath: flags.rules,
+        pageSize: pageSize,
+        maxRows: maxRows,
+    });
+    if (result.status === "exported" && result.xml && outPath) {
+        fs.writeFileSync(path.resolve(outPath), result.xml, "utf8");
+    }
+    writeExportReceipt(flags, result, outPath);
+    if (result.status === "failed") {
+        return 2;
+    }
+    if (result.status === "dry-run" && flags["dry-run"] !== "true") {
+        return 1;
+    }
+    return 0;
+}
+/** Shared receipt for both export verbs. */
+function writeExportReceipt(flags, result, outPath) {
+    if (flags.json === "true") {
+        process.stdout.write(JSON.stringify(result, null, 2) + "\n");
+        return;
+    }
+    process.stdout.write("[" +
+        result.status +
+        "]" +
+        (outPath && result.status === "exported"
+            ? " " + path.resolve(outPath)
+            : "") +
+        "\n" +
+        result.note +
+        "\n");
+    if (result.secretFields.length > 0) {
+        process.stdout.write("Set these after loading the package:\n");
+        for (var i = 0; i < result.secretFields.length; i += 1) {
+            process.stdout.write("  " +
+                result.secretFields[i].table +
+                "." +
+                result.secretFields[i].field +
+                "\n");
+        }
+    }
+}
+/**
+ * dove-sn export-app:
+ *   --app <scope|sys_id|name>   Required. The sys_app to publish and export.
+ *   [--version <v>]             Publish version (default: the app's current one).
+ *   [--description <text>]      Recorded on the update set.
+ *   [--include-data]            Ship table DATA as well as schema (off by default).
+ *   --out <file>                Where to write the XML.
+ *   [--rules <file>] [--timeout-ms <n>]
+ *   [--dry-run] [--json] [--confirm]
+ *
+ * PUBLISHING IS A REAL INSTANCE WRITE — a new update set and ~1000+ records.
+ * DRY-RUN BY DEFAULT. Secret values are always stripped before the file lands.
+ * Exit codes: 0 exported/dry-run, 1 bad args/unconfirmed, 2 failed/timeout.
+ */
+async function runExportApp(flags) {
+    var app = flags.app;
+    if (!app) {
+        process.stderr.write("export-app: --app <scope|sys_id|name> is required\n");
+        return 1;
+    }
+    var outPath = flags.out;
+    if (!outPath && flags["dry-run"] !== "true") {
+        process.stderr.write("export-app: --out <file> is required for a real export\n");
+        return 1;
+    }
+    var timeoutMs = undefined;
+    if (flags["timeout-ms"]) {
+        timeoutMs = Number(flags["timeout-ms"]);
+        if (!Number.isInteger(timeoutMs) || timeoutMs <= 0) {
+            process.stderr.write("export-app: --timeout-ms must be a positive integer\n");
+            return 1;
+        }
+    }
+    var result = await (0, exportApp_1.exportApp)({
+        app: app,
+        version: flags.version,
+        description: flags.description,
+        includeData: flags["include-data"] === "true",
+        keepSet: flags["keep-set"] !== "false",
+        confirm: flags.confirm === "true",
+        dryRun: flags["dry-run"] === "true",
+        rulesPath: flags.rules,
+        timeoutMs: timeoutMs,
+    });
+    if (result.status === "exported" && result.xml && outPath) {
+        fs.writeFileSync(path.resolve(outPath), result.xml, "utf8");
+    }
+    writeExportReceipt(flags, result, outPath);
+    if (result.status === "failed" || result.status === "timeout") {
+        return 2;
+    }
+    if (result.status === "dry-run" && flags["dry-run"] !== "true") {
+        return 1;
+    }
+    return 0;
+}
+/**
+ * dove-sn strip-secrets:
+ *   --in <file>                 Required. An unload XML exported earlier.
+ *   --out <file>                Required unless --report.
+ *   [--rules <file>]            JSON overrides for the secret rules.
+ *   [--report]                  List what WOULD be stripped and what needs review;
+ *                               writes nothing.
+ *   [--json]
+ *
+ * Exists for documents produced outside these verbs. Exit codes: 0 clean,
+ * 1 bad args, 2 blocked (a field needs review, or a secret survived).
+ */
+async function runStripSecrets(flags) {
+    var inPath = flags.in;
+    if (!inPath) {
+        process.stderr.write("strip-secrets: --in <file> is required\n");
+        return 1;
+    }
+    var report = flags.report === "true";
+    var outPath = flags.out;
+    if (!report && !outPath) {
+        process.stderr.write("strip-secrets: --out <file> is required (or pass --report)\n");
+        return 1;
+    }
+    var xml = "";
+    try {
+        xml = fs.readFileSync(path.resolve(inPath), "utf8");
+    }
+    catch (e) {
+        process.stderr.write("strip-secrets: cannot read " + inPath + "\n");
+        return 1;
+    }
+    var rules = (0, secretRules_1.loadSecretRules)(flags.rules);
+    var result;
+    try {
+        result = (0, stripSecrets_1.stripSecrets)(xml, rules, { allowUnreviewed: report });
+    }
+    catch (e) {
+        process.stderr.write((e instanceof Error ? e.message : String(e)) + "\n");
+        return 2;
+    }
+    if (!report && outPath) {
+        fs.writeFileSync(path.resolve(outPath), result.xml, "utf8");
+    }
+    if (flags.json === "true") {
+        process.stdout.write(JSON.stringify({
+            recordsScanned: result.recordsScanned,
+            secretFields: result.secretFields,
+            reviewFindings: result.reviewFindings,
+            written: report ? null : path.resolve(outPath),
+        }, null, 2) + "\n");
+    }
+    else {
+        process.stdout.write("[" +
+            (report ? "report" : "stripped") +
+            "] " +
+            result.recordsScanned +
+            " record(s), " +
+            result.secretFields.length +
+            " secret value(s)" +
+            (report ? "" : " → " + path.resolve(outPath)) +
+            "\n");
+        for (var i = 0; i < result.reviewFindings.length; i += 1) {
+            process.stdout.write("  NEEDS REVIEW " +
+                result.reviewFindings[i].table +
+                "." +
+                result.reviewFindings[i].field +
+                " (matched '" +
+                result.reviewFindings[i].matched +
+                "')\n");
+        }
+    }
+    return report && result.reviewFindings.length > 0 ? 2 : 0;
+}
 async function main() {
     var parsed = parseArgs(process.argv.slice(2));
     // Load credentials before any command runs. `--env`/`--env-file` (or the
@@ -1987,6 +2238,15 @@ async function main() {
     if (parsed.command === "publish-app") {
         return await runPublishApp(parsed.flags);
     }
+    if (parsed.command === "export-update-set") {
+        return await runExportUpdateSet(parsed.flags);
+    }
+    if (parsed.command === "export-app") {
+        return await runExportApp(parsed.flags);
+    }
+    if (parsed.command === "strip-secrets") {
+        return await runStripSecrets(parsed.flags);
+    }
     if (parsed.command === "mcp") {
         return await runMcp(parsed.flags);
     }

package/dist/exportApp.d.ts

@@ -0,0 +1,102 @@
+/**
+ * Export a scoped application to an importable `<unload>` XML document — the
+ * headless equivalent of the classic UI's **Publish to Update Set → Export to
+ * XML**, with secret values stripped before the document is returned.
+ *
+ * Ground truth: `brainstorms/sn-export-app-flow.md` (HAR capture, tenonworkshop
+ * 2026-06-08) and the live run recorded there. "Export an app" is two operations
+ * chained, with no single button behind it:
+ *
+ *   1. `POST /xmlhttp.do` com.snc.apps.AppsAjaxProcessor `createUpdateSet` →
+ *      the new set's sys_id arrives in the answer attribute.
+ *   2. `POST /xmlhttp.do` AppsAjaxProcessor `publishToUpdateSet` → a worker id;
+ *      poll AJAXProgressStatusChecker/getStatus until "Successfully published"
+ *      (the HAR took 48 polls / ~44s for 1,088 files).
+ *   3. export that update set — delegated to exportUpdateSet, which owns the
+ *      in-progress empty-200 gotcha and the secret stripping.
+ *
+ * PUBLISHING IS A REAL SHARED-INSTANCE WRITE: it creates an update set and
+ * ~1000+ sys_update_xml rows. DRY-RUN BY DEFAULT — without confirm:true this
+ * resolves the app and returns the plan, having written nothing.
+ *
+ * This is NOT the Store publish. That is publishApp, which is externally
+ * visible; this one stays inside the instance.
+ *
+ * ES6 only, no optional chaining.
+ */
+import type { ServiceNowClient } from "./client";
+import type { FormAuth, FormSession, PostResult } from "./table";
+import type { ExportTransport } from "./exportUpdateSet";
+import type { SecretField } from "./secrets/stripSecrets";
+/** Injectable transport so tests never touch the network. */
+export interface ExportAppTransport extends ExportTransport {
+    post?: (auth: FormAuth, session: FormSession, path: string, fields: Record<string, string>) => Promise<PostResult>;
+    sleep?: (ms: number) => Promise<void>;
+}
+/** Inputs for exportApp. */
+export interface ExportAppParams {
+    /** App sys_id, scope, or name. */
+    app: string;
+    /** Publish version. Defaults to the app's current version. */
+    version?: string;
+    /** Description recorded on the update set. */
+    description?: string;
+    /** Include table DATA as well as schema. Off by default. */
+    includeData?: boolean;
+    /** Leave the published update set behind. Default true — deleting is not ours to do. */
+    keepSet?: boolean;
+    client?: ServiceNowClient;
+    instance?: string;
+    user?: string;
+    password?: string;
+    /** Required: publishing writes to the instance. */
+    confirm?: boolean;
+    /** Force a plan-only run. Wins over confirm. */
+    dryRun?: boolean;
+    /** Optional JSON secret-rules file. */
+    rulesPath?: string;
+    /** Milliseconds to wait for the publish worker. Default 300000. */
+    timeoutMs?: number;
+    transport?: ExportAppTransport;
+}
+/** Outcome of an app export. */
+export interface ExportAppResult {
+    status: "dry-run" | "exported" | "failed" | "timeout";
+    appName: string;
+    appScope: string;
+    appSysId: string;
+    version: string;
+    /** The update set the publish produced. */
+    updateSetSysId: string;
+    /** getStatus polls performed. */
+    polls: number;
+    recordCount: number;
+    xml?: string;
+    secretFields: Array<SecretField>;
+    message?: string;
+    note: string;
+}
+/** Publishing a large app is slow — the HAR run took ~44s for 1,088 files. */
+export declare var DEFAULT_EXPORT_APP_TIMEOUT_MS: number;
+interface ResolvedApp {
+    sysId: string;
+    scope: string;
+    name: string;
+    version: string;
+}
+/**
+ * Fields for the AppsAjaxProcessor createUpdateSet call. The UI sends a single
+ * space when the description box is empty — mirrored here so the processor sees
+ * what it sees from a browser.
+ */
+export declare function buildCreateSetFields(app: ResolvedApp, description: string): Record<string, string>;
+/** Fields for the AppsAjaxProcessor publishToUpdateSet call. */
+export declare function buildPublishFields(app: ResolvedApp, updateSetSysId: string, version: string, description: string, includeData: boolean): Record<string, string>;
+/**
+ * Publish an application into a fresh update set and export that set.
+ *
+ * Remote failures are RETURNED as a failed/timeout result; only caller errors
+ * (bad selector, bad timeout) throw.
+ */
+export declare function exportApp(params: ExportAppParams): Promise<ExportAppResult>;
+export {};