datagrok-tools 6.5.6 → 6.5.8

package/bin/commands/server-migrate.js

@@ -0,0 +1,392 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.handleMigrate = handleMigrate;
+exports.loadCreds = loadCreds;
+var fs = _interopRequireWildcard(require("fs"));
+var os = _interopRequireWildcard(require("os"));
+var path = _interopRequireWildcard(require("path"));
+var yaml = _interopRequireWildcard(require("js-yaml"));
+var _nodeDapi = require("../utils/node-dapi");
+var _serverClient = require("../utils/server-client");
+var _serverOutput = require("../utils/server-output");
+var _registry = require("../utils/migrate/registry");
+var bundle = _interopRequireWildcard(require("../utils/migrate/bundle"));
+var _walker = require("../utils/migrate/walker");
+var _pusher = require("../utils/migrate/pusher");
+var _parts = require("../utils/migrate/parts");
+function _interopRequireWildcard(e, t) { if ("function" == typeof WeakMap) var r = new WeakMap(), n = new WeakMap(); return (_interopRequireWildcard = function (e, t) { if (!t && e && e.__esModule) return e; var o, i, f = { __proto__: null, default: e }; if (null === e || "object" != typeof e && "function" != typeof e) return f; if (o = t ? n : r) { if (o.has(e)) return o.get(e); o.set(e, f); } for (const t in e) "default" !== t && {}.hasOwnProperty.call(e, t) && ((i = (o = Object.defineProperty) && Object.getOwnPropertyDescriptor(e, t)) && (i.get || i.set) ? o(f, t, i) : f[t] = e[t]); return f; })(e, t); }
+/// Docs: [Entity export / import](/docs/features/grok-tool/export-import/DESIGN.md)
+
+const SELECTION_USAGE = '  [<nqName|id>...] [--type t,t] [--namespace ns] [--space s] [--name n] [--author login]\n' + '  [--tag t] [--since 2w] [--filter expr] [--no-deps] [--no-include-data] [--include-files]';
+const PULL_USAGE = `Usage: grok s pull --out <dir> [--replace] [--admin] [--host <alias>]\n${SELECTION_USAGE}`;
+const MIGRATE_USAGE = 'Usage: grok s migrate --from <alias> --to <alias> [--dry-run] [--keep] [--admin]\n' + '  [--on-conflict fail|skip|duplicate|adopt] [--creds <file.yaml>]\n' + '  --by-namespace [--only a,b] [--skip c] [--state <file>] [--force] [--no-sweep]\n' + '    moves the instance one space at a time, checking users and packages first\n' + SELECTION_USAGE;
+async function handleMigrate(dapi, verb, rest, argv, output) {
+  if (verb === 'pull') return await handlePull(dapi, rest, argv, output);
+  if (verb === 'push') return await handlePush(dapi, rest, argv, output);
+  if (verb === 'migrate') return await handleTransfer(rest, argv, output);
+  if (verb === 'diff') return await handleDiff(dapi, rest, argv, output);
+  if (verb === 'bundle') return handleBundle(rest, output);
+  (0, _serverOutput.printError)(new Error(`Unknown migrate verb '${verb}'. Valid: pull, push, migrate, diff, bundle ls`));
+  return false;
+}
+const hasSelection = (rest, argv) => !!rest.length || !!argv.type || ['name', 'namespace', 'space', 'author', 'tag', 'since', 'filter', 'f'].some(f => argv[f]);
+async function handlePull(dapi, rest, argv, output, print = true, report) {
+  const out = argv.out ?? '';
+  if (!out || !hasSelection(rest, argv)) {
+    (0, _serverOutput.printError)(new Error(out ? `Refusing to pull the whole server — pass entity names, --type, or a filter flag.\n${PULL_USAGE}` : PULL_USAGE));
+    return false;
+  }
+  const sel = {
+    ...(0, _registry.resolveTypes)(argv.type ? String(argv.type).split(',') : _registry.DEFAULT_TYPES),
+    names: rest,
+    name: argv.name,
+    namespace: argv.namespace,
+    space: argv.space,
+    author: argv.author,
+    tag: argv.tag,
+    since: (0, _walker.normalizeSince)(argv.since),
+    filter: argv.filter ?? argv.f
+  };
+  const notes = [];
+  const note = row => notes.push(row);
+  const leftBehind = [];
+  const progress = (0, _serverOutput.progressReporter)(output === 'quiet');
+  const selected = await (0, _walker.select)(dapi, sel, note, progress);
+  const entities = argv.deps !== false ? await (0, _walker.expand)(dapi, selected, note, leftBehind, progress) : selected;
+  // Everything the bundle points at but leaves behind, so the push can re-find it by name.
+  const outside = await (0, _walker.collectExternals)(dapi, entities, note, progress);
+  const externals = [...leftBehind, ...outside.externals];
+  const kinds = [];
+  if (argv['include-data'] !== false) kinds.push('tables');
+  if (argv['include-files']) kinds.push('files');
+  const bytes = await (0, _walker.pullBytes)(dapi, entities, note, kinds, progress);
+  // A datasync table rebuilds itself from a share on open, so the file has to travel with it.
+  const shares = argv['include-files'] ? await (0, _walker.pullShares)(dapi, entities, note, progress) : new Map();
+  const info = await dapi.serverInfo();
+  const user = await dapi.client.get('/users/current');
+  bundle.write(out, entities, {
+    source: {
+      url: dapi.client.baseUrl,
+      version: info.version,
+      commit: info.commit,
+      userNamespace: user?.project?.name ? `${user.project.name}:` : ''
+    },
+    args: process.argv.slice(3),
+    packages: notes.filter(n => n.reason === 'package_entity').map(n => n.detail).filter(Boolean),
+    externals,
+    dangling: outside.dangling
+  }, {
+    replace: !!argv.replace
+  }, bytes);
+  bundle.writeShares(out, shares);
+  progress(`wrote ${entities.size} entities to ${out}`);
+  const rows = [...notes];
+  if (outside.dangling.length) rows.push({
+    name: dapi.client.baseUrl,
+    entityType: 'Bundle',
+    action: 'warn',
+    reason: 'source_dangling_refs',
+    detail: `${outside.dangling.length} reference(s) point at entities the source itself no longer has`
+  });
+  for (const [, {
+    type,
+    json
+  }] of entities) rows.push({
+    name: (0, _registry.nqNameOf)(json),
+    entityType: type,
+    action: 'info',
+    reason: 'pulled'
+  });
+  // An entity the server would not hand over is missing from the bundle, and pushing it would
+  // quietly promote less than was asked for. Absent bytes are not that: a datasync table that was
+  // never materialised has no data file to give, travels fine and refreshes on the target — so
+  // `no_data` is reported per table and left to the operator rather than blocking the push.
+  const dropped = notes.filter(n => n.reason === 'fetch_failed').length;
+  if (dropped) {
+    rows.push({
+      name: out,
+      entityType: 'Bundle',
+      action: 'failed',
+      reason: 'incomplete',
+      detail: `${dropped} entities could not be read in full`
+    });
+    process.exitCode = 1;
+    if (report) report.dropped = dropped;
+  }
+  // `migrate` silences the pull's own report, but a refusal has to say why.
+  if (print || dropped) (0, _serverOutput.printOutput)(rows, output);
+  return true;
+}
+const ENV_RE = /\$\{(\w*)\}/g;
+
+/**
+ * Target-side secrets, keyed by connection nqName. `${VAR}` is resolved from the
+ * environment, as `grok publish` does for `connections/*.json` — after the YAML is parsed,
+ * so a broken file is reported without a secret in the message.
+ */
+function loadCreds(file) {
+  if (!file) return undefined;
+  const loaded = yaml.load(fs.readFileSync(file, 'utf8')) ?? {};
+  const missing = [];
+  const creds = {};
+  for (const [key, params] of Object.entries(loaded)) {
+    if (!params || typeof params !== 'object' || Array.isArray(params)) throw new Error(`${file}: "${key}" must be a map of connection parameters, e.g. '${key}: {password: \${VAR}}'`);
+    creds[key] = {};
+    for (const [name, value] of Object.entries(params)) creds[key][name] = typeof value !== 'string' ? value : value.replace(ENV_RE, (whole, env) => {
+      const resolved = process.env[env];
+      if (resolved !== undefined) return resolved;
+      missing.push(env);
+      return whole;
+    });
+  }
+  if (missing.length) throw new Error(`${file}: cannot find environment variable "${[...new Set(missing)].join('", "')}"`);
+  return creds;
+}
+const POLICIES = ['fail', 'skip', 'duplicate', 'adopt'];
+const SWEEP_TYPES = 'layout,view';
+function conflictPolicy(argv) {
+  const policy = argv['on-conflict'] ?? 'fail';
+  if (!POLICIES.includes(policy)) throw new Error(`Unsupported conflict policy '${policy}'. Valid: ${POLICIES.join(', ')}`);
+  return policy;
+}
+async function handlePush(dapi, rest, argv, output) {
+  const dir = rest[0];
+  if (!dir) {
+    (0, _serverOutput.printError)(new Error('Usage: grok s push <bundle-dir> [--dry-run] [--on-conflict fail|skip|duplicate|adopt] [--creds <file.yaml>] [--admin] [--host <alias>]'));
+    return false;
+  }
+  const onConflict = conflictPolicy(argv);
+  const creds = loadCreds(argv.creds);
+  const dryRun = !!argv['dry-run'];
+  const result = await (0, _pusher.push)(dapi, bundle.read(dir), {
+    dryRun,
+    onConflict,
+    creds,
+    progress: (0, _serverOutput.progressReporter)(output === 'quiet')
+  }, rows => {
+    if (output === 'table' && !dryRun) {
+      console.log('Plan:');
+      (0, _serverOutput.printOutput)(rows, output);
+      console.log('');
+    }
+  });
+  if (result.items.some(r => r.action === 'failed')) process.exitCode = 1;
+  if (output === 'json') {
+    (0, _serverOutput.printOutput)({
+      ...result,
+      items: result.items.map(r => ({
+        ...r,
+        detail: r.detail ?? ''
+      }))
+    }, 'json');
+    return true;
+  }
+  if (!dryRun) console.log('Result:');
+  (0, _serverOutput.printOutput)(result.items, output);
+  return true;
+}
+
+/**
+ * A bundle pulled from one instance and pushed into another, with a temporary bundle
+ * directory in between — the same two verbs, so nothing behaves differently.
+ */
+async function handleTransfer(rest, argv, output) {
+  if (!argv.from || !argv.to || !argv['by-namespace'] && !hasSelection(rest, argv)) {
+    (0, _serverOutput.printError)(new Error(MIGRATE_USAGE));
+    return false;
+  }
+  const from = new _nodeDapi.NodeDapi(await (0, _serverClient.createClient)(String(argv.from), !!argv.admin));
+  const to = new _nodeDapi.NodeDapi(await (0, _serverClient.createClient)(String(argv.to), !!argv.admin));
+  if (from.client.baseUrl === to.client.baseUrl) throw new Error(`--from and --to are the same server (${to.client.baseUrl}) — nothing to migrate`);
+  if (argv['by-namespace']) {
+    if (rest.length) throw new Error('--by-namespace migrates whole spaces; it takes no entity selection');
+    return await transferByNamespace(from, to, argv, output);
+  }
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'grok-migrate-'));
+  try {
+    const report = {
+      dropped: 0
+    };
+    if (!(await handlePull(from, rest, {
+      ...argv,
+      out: dir
+    }, output, output !== 'json', report))) return false;
+    if (report.dropped) {
+      (0, _serverOutput.printError)(new Error(`Refusing to push a partial bundle: ${report.dropped} entities could not be read. ` + 'Re-run, or pull with --keep and push the bundle yourself.'));
+      return true;
+    }
+    return await handlePush(to, [dir], argv, output);
+  } finally {
+    // stderr: `--output json` must stay one parseable document.
+    if (argv.keep) console.error(`Bundle kept at ${dir}`);else fs.rmSync(dir, {
+      recursive: true,
+      force: true
+    });
+  }
+}
+
+/**
+ * A whole instance, one space at a time. Placement is exclusive on the server, so a single bundle
+ * holding every space has its projects taking entities from one another; scoped to one space that
+ * contention stays inside it. A part that fails does not stop the rest, and what finished is
+ * recorded so a re-run picks up where it stopped.
+ */
+async function transferByNamespace(from, to, argv, output) {
+  const only = argv.only ? String(argv.only).split(',').map(s => s.trim()) : [];
+  const skip = new Set(argv.skip ? String(argv.skip).split(',').map(s => s.trim()) : []);
+  // `--from` and `--to` can be full URLs, which are not file names.
+  const tag = v => String(v).replace(/[^\w.-]+/g, '_');
+  const stateFile = argv.state ?? path.join(os.tmpdir(), `grok-migrate-${tag(argv.from)}-${tag(argv.to)}.json`);
+  const state = (0, _parts.readState)(stateFile);
+
+  // Without an admin session the source lists only what this account can see, so the run would
+  // enumerate a subset of the instance and report a whole-instance migration.
+  if (!argv.admin && !argv.force) throw new Error('--by-namespace needs --admin, or it sees only the spaces this account can ' + 'reach and migrates part of the instance; pass --force to accept that.');
+
+  // Both are prerequisites of the instance, not of any bundle, and both are cheaper to fix now
+  // than to discover space by space: content of a user the target lacks lands under the pusher.
+  const [users, packages] = await Promise.all([(0, _parts.missingUsers)(from, to), (0, _parts.missingPackages)(from, to)]);
+  // A source can have hundreds of each; the count is what decides, and a few names say which kind.
+  const few = all => all.slice(0, 5).join(', ') + (all.length > 5 ? `, +${all.length - 5} more` : '');
+  const notes = [];
+  if (users.length) notes.push({
+    name: `${users.length} user(s)`,
+    entityType: 'User',
+    action: 'warn',
+    reason: 'user_missing',
+    detail: `${few(users)} — create them on the target first, or their content lands under the pushing account`
+  });
+  if (packages.length) notes.push({
+    name: `${packages.length} package(s)`,
+    entityType: 'Package',
+    action: 'warn',
+    reason: 'package_not_installed',
+    detail: `${few(packages)} — publish them on the target, or what they own cannot resolve`
+  });
+  (0, _serverOutput.printOutput)(notes, output);
+  // A whole-instance run stops: content of a user the target lacks lands under the pushing account,
+  // and that is not worth discovering space by space. A run the operator has already scoped with
+  // `--only` is their call, so it is reported and allowed.
+  if (notes.length && !argv.force && !only.length) {
+    (0, _serverOutput.printError)(new Error('Refusing to start: fix the above, or pass --force to migrate anyway.'));
+    process.exitCode = 1;
+    return true;
+  }
+
+  // Resolved once: a bad policy or a broken creds file is the run's problem, not each part's.
+  const onConflict = conflictPolicy(argv);
+  const creds = loadCreds(argv.creds);
+  const dryRun = !!argv['dry-run'];
+
+  // Not everything belongs to a space: a layout can sit under no namespace at all, and would
+  // otherwise never travel, so a full run ends with a sweep for what no space owns.
+  const spaces = (0, _parts.plannedParts)(await (0, _parts.namespacesOf)(from), {
+    only,
+    skip: [...skip],
+    state,
+    sweep: !only.length && argv.sweep !== false
+  });
+  const parts = [];
+  for (const [i, name] of spaces.entries()) {
+    console.error(`[${i + 1}/${spaces.length}] ${name === _parts.SWEEP ? 'everything a space does not own' : name}`);
+    const started = Date.now();
+    const part = {
+      name
+    };
+    const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'grok-migrate-'));
+    try {
+      const report = {
+        dropped: 0
+      };
+      // What a space can fail to own is a leaf — a layout or a view under no namespace. Sweeping
+      // every type instead would list the whole instance, including the loose tables a stand
+      // accumulates in the millions, which is the shape this command exists to avoid.
+      const scope = name === _parts.SWEEP ? {
+        type: argv.type ?? SWEEP_TYPES,
+        namespace: undefined
+      } : {
+        namespace: name
+      };
+      const pulled = await handlePull(from, [], {
+        ...argv,
+        ...scope,
+        'include-files': true,
+        out: dir
+      }, 'quiet', false, report);
+      const read = pulled ? bundle.read(dir) : null;
+      part.entities = read ? read.entities.size : 0;
+      // Pushing a bundle the pull could not fill promotes less than the space holds, and recording
+      // it as done would hide that for good: the part stays failed so a re-run takes it again.
+      if (report.dropped) part.error = `${report.dropped} entities could not be read — not pushed`;else if (read && part.entities) {
+        const result = await (0, _pusher.push)(to, read, {
+          onConflict,
+          creds,
+          dryRun,
+          progress: (0, _serverOutput.progressReporter)(output === 'quiet' || output === 'json')
+        }, () => {});
+        part.failed = result.items.filter(r => r.action === 'failed').length;
+      }
+    } catch (err) {
+      part.error = err?.message ?? String(err);
+    } finally {
+      fs.rmSync(dir, {
+        recursive: true,
+        force: true
+      });
+    }
+    part.seconds = Math.round((Date.now() - started) / 1000);
+    if (!dryRun) {
+      state[name] = part;
+      (0, _parts.writeState)(stateFile, state);
+    }
+    parts.push(part);
+  }
+
+  // What a resume skipped belongs in the table too, or the run reports less than it has done.
+  const all = [...Object.values(state).filter(p => !parts.some(q => q.name === p.name)), ...parts];
+  (0, _serverOutput.printOutput)(all.map(p => ({
+    space: p.name,
+    entities: p.entities ?? 0,
+    failed: p.failed ?? 0,
+    seconds: p.seconds ?? 0,
+    error: p.error ?? ''
+  })), output);
+  console.error(`state: ${stateFile}`);
+  if (all.some(p => p.error || (p.failed ?? 0) > 0)) process.exitCode = 1;
+  return true;
+}
+
+/** What a push would do, read-only: the plan plus the top-level keys that differ. */
+async function handleDiff(dapi, rest, argv, output) {
+  const dir = rest[0];
+  if (!dir) {
+    (0, _serverOutput.printError)(new Error('Usage: grok s diff <bundle-dir> [--host <alias>]'));
+    return false;
+  }
+  const read = bundle.read(dir);
+  const {
+    rows
+  } = await (0, _pusher.plan)(dapi, read, {
+    onConflict: argv['on-conflict'] ? conflictPolicy(argv) : 'skip',
+    idmap: {
+      ...read.idmap
+    }
+  });
+  const items = rows.map(r => ({
+    ...r,
+    detail: r.detail ?? ''
+  }));
+  (0, _serverOutput.printOutput)(output === 'json' ? (0, _pusher.summarize)(items, dapi, 'dry-run') : items, output);
+  return true;
+}
+function handleBundle(rest, output) {
+  if (rest[0] !== 'ls' || !rest[1]) {
+    (0, _serverOutput.printError)(new Error('Usage: grok s bundle ls <bundle-dir>'));
+    return false;
+  }
+  (0, _serverOutput.printOutput)(bundle.list(rest[1]), output);
+  return true;
+}