datagrok-tools 6.5.6 → 6.5.8

package/bin/grok.js

@@ -2,6 +2,8 @@
 const argv = require('minimist')(process.argv.slice(2), {
   alias: {k: 'key', h: 'help', r: 'recursive'},
   boolean: ['dartium'],
+  // keep versions verbatim — minimist would coerce '1.10' to the number 1.1
+  string: ['version'],
 });
 // minimist maps `--no-retry` to `{retry: false}`, so the `args['no-retry']` checks in
 // test.ts / playwright-runner.ts never fired and `--no-retry` was silently ignored
@@ -34,6 +36,14 @@ const commands = {
 
 const onPackageCommandNames = ['api', 'check', 'link', 'publish', 'test'];
 
+// A machine-readable run prints its error as JSON on stderr (server.ts) and nothing else:
+// a usage dump on stdout would corrupt the output the caller parses.
+const outputFormat = argv.output ?? argv.o;
+function printUsage(command) {
+  if (outputFormat !== 'json')
+    process.stderr.write(`${help[command]}\n`);
+}
+
 const command = argv['_'][0];
 if (command !== 'test' && command !== 'stresstest')
   delete argv.dartium;
@@ -50,23 +60,23 @@ if (command in commands) {
       if (result && typeof result.then === 'function') {
         result.then((ok) => {
           if (!ok) {
-            console.log(help[command]);
+            printUsage(command);
             exitWithCode(1);
           }
         }).catch((err) => {
           console.error(err);
-          console.log(help[command]);
+          printUsage(command);
           exitWithCode(255);
         });
       }
       else if (!result) {
-        console.log(help[command]);
+        printUsage(command);
         exitWithCode(1);
       }
     }
   } catch (err) {
     console.error(err);
-    console.log(help[command]);
+    printUsage(command);
     exitWithCode(255);
   }
 } else
@@ -74,6 +84,7 @@ if (command in commands) {
 
 
 function exitWithCode(code) {
-  console.log(`Exiting with code ${code}`);
+  if (outputFormat !== 'json')
+    console.log(`Exiting with code ${code}`);
   process.exit(code);
 }

package/bin/utils/migrate/bundle.js

@@ -0,0 +1,223 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.bytesPath = bytesPath;
+exports.hashOf = hashOf;
+exports.hashView = hashView;
+exports.list = list;
+exports.listShares = listShares;
+exports.normalize = normalize;
+exports.read = read;
+exports.sharePath = sharePath;
+exports.stripPrivate = stripPrivate;
+exports.write = write;
+exports.writeIdmap = writeIdmap;
+exports.writeShares = writeShares;
+var fs = _interopRequireWildcard(require("fs"));
+var path = _interopRequireWildcard(require("path"));
+var _crypto = require("crypto");
+var _registry = require("./registry");
+function _interopRequireWildcard(e, t) { if ("function" == typeof WeakMap) var r = new WeakMap(), n = new WeakMap(); return (_interopRequireWildcard = function (e, t) { if (!t && e && e.__esModule) return e; var o, i, f = { __proto__: null, default: e }; if (null === e || "object" != typeof e && "function" != typeof e) return f; if (o = t ? n : r) { if (o.has(e)) return o.get(e); o.set(e, f); } for (const t in e) "default" !== t && {}.hasOwnProperty.call(e, t) && ((i = (o = Object.defineProperty) && Object.getOwnPropertyDescriptor(e, t)) && (i.get || i.set) ? o(f, t, i) : f[t] = e[t]); return f; })(e, t); }
+/// Docs: [Entity export / import](/docs/features/grok-tool/export-import/DESIGN.md)
+
+// `entityTags` rows carry their own primary keys and come back in an unstable order:
+// pushing them makes the hash flap AND re-points the source's tag rows at the copy.
+const VOLATILE = ['createdOn', 'updatedOn', 'author', 'pictureId', 'encryptedParametersId', 'keyKid', 'isAvailable', 'entityTags'];
+function normalize(type, json) {
+  const copy = JSON.parse(JSON.stringify(json));
+  // Tag rows are volatile, but the tag names are worth migrating: keep them as a
+  // bundle-only list the pusher replays through `POST /entities/tag`.
+  const tags = [...new Set((copy.entityTags ?? []).map(t => t?.tag).filter(Boolean))].sort();
+  if (tags.length) copy._tags = tags;
+  for (const k of VOLATILE) delete copy[k];
+  if (copy.package) copy.package = {
+    id: copy.package.id
+  };
+  // The stamp is written by the pusher, so it must not make an unchanged entity look different.
+  if (copy.metaParams) delete copy.metaParams.sync_id;
+  _registry.TYPES[type]?.strip?.(copy);
+  return sortKeys(copy);
+}
+function sortKeys(v) {
+  if (Array.isArray(v)) return v.map(sortKeys);
+  if (v === null || typeof v !== 'object') return v;
+  const out = {};
+  for (const k of Object.keys(v).sort()) out[k] = sortKeys(v[k]);
+  return out;
+}
+
+/** Bundle-only keys (`_credentials`, `_grants`, `_members`) never travel to the server. */
+function stripPrivate(json) {
+  for (const k of Object.keys(json)) if (k.startsWith('_')) delete json[k];
+  return json;
+}
+
+/**
+ * What a comparison sees: the payload without the bundle-only keys and without the
+ * namespace, which the server computes from ownership (`Askalkin:` on the source,
+ * `Admin:` on the target) and would otherwise make every cross-instance push a rewrite.
+ * The bundle file keeps it — `findByNqName` resolves the twin by it.
+ */
+function hashView(type, json) {
+  const view = stripPrivate(normalize(type, json));
+  delete view.namespace;
+  // Relations have a pass of their own, which only ever adds: their row ids belong to the
+  // target, and a target that links more than the bundle is not stale. The pusher compares
+  // them by coverage instead.
+  delete view.relations;
+  return view;
+}
+function hashOf(type, json) {
+  return (0, _crypto.createHash)('sha256').update(JSON.stringify(hashView(type, json))).digest('hex');
+}
+function bytesPath(dir, kind, id) {
+  return path.join(dir, kind, kind === 'tables' ? `${id}.d42` : id);
+}
+
+/**
+ * A file a datasync table reads is stored under its full share path, flattened into one
+ * file name so the bundle stays a flat directory per kind and the path survives a round trip.
+ */
+function sharePath(dir, remotePath) {
+  return path.join(dir, 'shares', encodeURIComponent(remotePath));
+}
+function writeShares(dir, files) {
+  if (!files.size) return;
+  fs.mkdirSync(path.join(dir, 'shares'), {
+    recursive: true
+  });
+  for (const [remote, buf] of files) fs.writeFileSync(sharePath(dir, remote), buf);
+}
+function listShares(dir) {
+  const at = path.join(dir, 'shares');
+  return fs.existsSync(at) ? fs.readdirSync(at).map(decodeURIComponent) : [];
+}
+const EMPTY = {
+  formatVersion: 1,
+  source: {
+    url: '',
+    version: '',
+    userNamespace: ''
+  },
+  pulls: [],
+  order: [],
+  packages: []
+};
+function readManifest(dir) {
+  const file = path.join(dir, 'manifest.json');
+  if (!fs.existsSync(file)) return JSON.parse(JSON.stringify(EMPTY));
+  return JSON.parse(fs.readFileSync(file, 'utf8'));
+}
+function write(dir, entities, meta, opts, bytes = new Map()) {
+  if (opts.replace) fs.rmSync(dir, {
+    recursive: true,
+    force: true
+  });
+  const manifest = readManifest(dir);
+  if (!manifest.source.url) manifest.source = meta.source;else if (manifest.source.url !== meta.source.url) throw new Error(`${dir} was pulled from ${manifest.source.url}; ` + `use --replace to start a new bundle from ${meta.source.url}`);
+  fs.mkdirSync(dir, {
+    recursive: true
+  });
+  const byId = new Map(manifest.order.map(e => [e.id, e]));
+  const takenBy = new Map([...byId.values()].map(e => [e.file, e.id]));
+  const ids = [];
+  for (const [id, {
+    type,
+    json
+  }] of entities) {
+    const base = `${type}/${(0, _registry.fileNameFor)(json)}`;
+    // Names are not unique across a bundle (two files of the same name in different shares);
+    // whoever claimed the plain name keeps it, the rest are suffixed by their id.
+    const owner = takenBy.get(`${base}.json`);
+    const file = !owner || owner === id ? `${base}.json` : `${base}-${id.slice(0, 8)}.json`;
+    const previous = byId.get(id);
+    if (previous && previous.file !== file) {
+      fs.rmSync(path.join(dir, previous.file), {
+        force: true
+      });
+      takenBy.delete(previous.file);
+    }
+    takenBy.set(file, id);
+    fs.mkdirSync(path.join(dir, type), {
+      recursive: true
+    });
+    fs.writeFileSync(path.join(dir, file), JSON.stringify(normalize(type, json), null, 2));
+    byId.set(id, {
+      type,
+      id,
+      file
+    });
+    ids.push(id);
+  }
+  for (const [id, buf] of bytes) {
+    const kind = _registry.TYPES[entities.get(id).type].bytes.kind;
+    fs.mkdirSync(path.join(dir, kind), {
+      recursive: true
+    });
+    fs.writeFileSync(bytesPath(dir, kind, id), buf);
+  }
+  manifest.formatVersion = 1;
+  manifest.pulls.push({
+    at: new Date().toISOString(),
+    args: meta.args,
+    ids
+  });
+  manifest.packages = [...new Set([...manifest.packages, ...meta.packages])].sort();
+  if (meta.externals?.length) {
+    const byId = new Map((manifest.externals ?? []).map(e => [e.id, e]));
+    for (const e of meta.externals) byId.set(e.id, e);
+    manifest.externals = [...byId.values()].sort((a, b) => a.nqName.localeCompare(b.nqName));
+  }
+  if (meta.dangling?.length) manifest.dangling = [...new Set([...(manifest.dangling ?? []), ...meta.dangling])].sort();
+  const pullOf = new Map();
+  manifest.pulls.forEach((p, i) => p.ids.forEach(id => pullOf.has(id) || pullOf.set(id, i)));
+  manifest.order = [...byId.values()].sort((a, b) => (0, _registry.rankOf)(a.type) - (0, _registry.rankOf)(b.type) || pullOf.get(a.id) - pullOf.get(b.id) || a.file.localeCompare(b.file));
+  fs.writeFileSync(path.join(dir, 'manifest.json'), JSON.stringify(manifest, null, 2));
+  return manifest;
+}
+function read(dir) {
+  if (!fs.existsSync(path.join(dir, 'manifest.json'))) throw new Error(`Not a bundle directory (no manifest.json): ${dir}`);
+  const manifest = readManifest(dir);
+  const entities = new Map();
+  for (const e of manifest.order) {
+    const file = path.join(dir, e.file);
+    if (!fs.existsSync(file)) throw new Error(`Bundle is missing ${e.file} listed in manifest.json`);
+    entities.set(e.id, {
+      type: e.type,
+      json: JSON.parse(fs.readFileSync(file, 'utf8')),
+      file: e.file
+    });
+  }
+  const idmapFile = path.join(dir, 'idmap.json');
+  const idmap = fs.existsSync(idmapFile) ? JSON.parse(fs.readFileSync(idmapFile, 'utf8')) : {};
+  return {
+    dir,
+    manifest,
+    entities,
+    idmap
+  };
+}
+
+/** Adopted `sourceId → targetId` pairs, so the next push of this bundle is stable. */
+function writeIdmap(dir, idmap) {
+  fs.writeFileSync(path.join(dir, 'idmap.json'), JSON.stringify(idmap, null, 2));
+}
+function list(dir) {
+  const {
+    manifest,
+    entities
+  } = read(dir);
+  return manifest.order.map(e => {
+    const pull = manifest.pulls.findIndex(p => p.ids.includes(e.id));
+    const json = entities.get(e.id).json;
+    return {
+      type: e.type,
+      nqName: (0, _registry.nqNameOf)(json),
+      file: e.file,
+      pulledOn: manifest.pulls[pull]?.at ?? '',
+      pull: pull + 1
+    };
+  });
+}

package/bin/utils/migrate/bundle.ts

@@ -0,0 +1,222 @@
+/// Docs: [Entity export / import](/docs/features/grok-tool/export-import/DESIGN.md)
+import * as fs from 'fs';
+import * as path from 'path';
+import {createHash} from 'crypto';
+import {BytesKind, TYPES, fileNameFor, nqNameOf, rankOf} from './registry';
+
+export interface ManifestEntry {type: string; id: string; file: string}
+
+export interface Manifest {
+  formatVersion: 1;
+  source: {url: string; version: string; commit?: string; userNamespace: string};
+  pulls: {at: string; args: string[]; ids: string[]}[];
+  order: ManifestEntry[];
+  packages: string[];
+  /** Referenced entities the source keeps; the push re-finds them by name. */
+  externals?: {id: string; type: string; nqName: string}[];
+  /** Ids the bundle references that nothing on the source answers to — dead there, unfixable here. */
+  dangling?: string[];
+}
+
+export interface BundleEntity {type: string; json: any; file?: string}
+
+export interface Bundle {
+  dir: string;
+  manifest: Manifest;
+  entities: Map<string, BundleEntity>;
+  idmap: Record<string, string>;
+}
+
+// `entityTags` rows carry their own primary keys and come back in an unstable order:
+// pushing them makes the hash flap AND re-points the source's tag rows at the copy.
+const VOLATILE = ['createdOn', 'updatedOn', 'author', 'pictureId', 'encryptedParametersId',
+  'keyKid', 'isAvailable', 'entityTags'];
+
+export function normalize(type: string, json: any): any {
+  const copy = JSON.parse(JSON.stringify(json));
+  // Tag rows are volatile, but the tag names are worth migrating: keep them as a
+  // bundle-only list the pusher replays through `POST /entities/tag`.
+  const tags = [...new Set((copy.entityTags ?? []).map((t: any) => t?.tag).filter(Boolean))].sort();
+  if (tags.length)
+    copy._tags = tags;
+  for (const k of VOLATILE)
+    delete copy[k];
+  if (copy.package)
+    copy.package = {id: copy.package.id};
+  // The stamp is written by the pusher, so it must not make an unchanged entity look different.
+  if (copy.metaParams)
+    delete copy.metaParams.sync_id;
+  TYPES[type]?.strip?.(copy);
+  return sortKeys(copy);
+}
+
+function sortKeys(v: any): any {
+  if (Array.isArray(v)) return v.map(sortKeys);
+  if (v === null || typeof v !== 'object') return v;
+  const out: any = {};
+  for (const k of Object.keys(v).sort())
+    out[k] = sortKeys(v[k]);
+  return out;
+}
+
+/** Bundle-only keys (`_credentials`, `_grants`, `_members`) never travel to the server. */
+export function stripPrivate(json: any): any {
+  for (const k of Object.keys(json))
+    if (k.startsWith('_'))
+      delete json[k];
+  return json;
+}
+
+/**
+ * What a comparison sees: the payload without the bundle-only keys and without the
+ * namespace, which the server computes from ownership (`Askalkin:` on the source,
+ * `Admin:` on the target) and would otherwise make every cross-instance push a rewrite.
+ * The bundle file keeps it — `findByNqName` resolves the twin by it.
+ */
+export function hashView(type: string, json: any): any {
+  const view = stripPrivate(normalize(type, json));
+  delete view.namespace;
+  // Relations have a pass of their own, which only ever adds: their row ids belong to the
+  // target, and a target that links more than the bundle is not stale. The pusher compares
+  // them by coverage instead.
+  delete view.relations;
+  return view;
+}
+
+export function hashOf(type: string, json: any): string {
+  return createHash('sha256').update(JSON.stringify(hashView(type, json))).digest('hex');
+}
+
+export function bytesPath(dir: string, kind: BytesKind, id: string): string {
+  return path.join(dir, kind, kind === 'tables' ? `${id}.d42` : id);
+}
+
+/**
+ * A file a datasync table reads is stored under its full share path, flattened into one
+ * file name so the bundle stays a flat directory per kind and the path survives a round trip.
+ */
+export function sharePath(dir: string, remotePath: string): string {
+  return path.join(dir, 'shares', encodeURIComponent(remotePath));
+}
+
+export function writeShares(dir: string, files: Map<string, Buffer>): void {
+  if (!files.size) return;
+  fs.mkdirSync(path.join(dir, 'shares'), {recursive: true});
+  for (const [remote, buf] of files)
+    fs.writeFileSync(sharePath(dir, remote), buf);
+}
+
+export function listShares(dir: string): string[] {
+  const at = path.join(dir, 'shares');
+  return fs.existsSync(at) ? fs.readdirSync(at).map(decodeURIComponent) : [];
+}
+
+const EMPTY: Manifest = {
+  formatVersion: 1,
+  source: {url: '', version: '', userNamespace: ''},
+  pulls: [],
+  order: [],
+  packages: [],
+};
+
+function readManifest(dir: string): Manifest {
+  const file = path.join(dir, 'manifest.json');
+  if (!fs.existsSync(file)) return JSON.parse(JSON.stringify(EMPTY));
+  return JSON.parse(fs.readFileSync(file, 'utf8'));
+}
+
+export function write(dir: string, entities: Map<string, BundleEntity>,
+                      meta: {source: Manifest['source']; args: string[]; packages: string[];
+                             externals?: {id: string; type: string; nqName: string}[]; dangling?: string[]},
+                      opts: {replace?: boolean},
+                      bytes: Map<string, Buffer> = new Map()): Manifest {
+  if (opts.replace)
+    fs.rmSync(dir, {recursive: true, force: true});
+  const manifest = readManifest(dir);
+  if (!manifest.source.url)
+    manifest.source = meta.source;
+  else if (manifest.source.url !== meta.source.url)
+    throw new Error(`${dir} was pulled from ${manifest.source.url}; ` +
+      `use --replace to start a new bundle from ${meta.source.url}`);
+  fs.mkdirSync(dir, {recursive: true});
+
+  const byId = new Map<string, ManifestEntry>(manifest.order.map((e) => [e.id, e]));
+  const takenBy = new Map<string, string>([...byId.values()].map((e) => [e.file, e.id]));
+  const ids: string[] = [];
+  for (const [id, {type, json}] of entities) {
+    const base = `${type}/${fileNameFor(json)}`;
+    // Names are not unique across a bundle (two files of the same name in different shares);
+    // whoever claimed the plain name keeps it, the rest are suffixed by their id.
+    const owner = takenBy.get(`${base}.json`);
+    const file = !owner || owner === id ? `${base}.json` : `${base}-${id.slice(0, 8)}.json`;
+    const previous = byId.get(id);
+    if (previous && previous.file !== file) {
+      fs.rmSync(path.join(dir, previous.file), {force: true});
+      takenBy.delete(previous.file);
+    }
+    takenBy.set(file, id);
+    fs.mkdirSync(path.join(dir, type), {recursive: true});
+    fs.writeFileSync(path.join(dir, file), JSON.stringify(normalize(type, json), null, 2));
+    byId.set(id, {type, id, file});
+    ids.push(id);
+  }
+
+  for (const [id, buf] of bytes) {
+    const kind = TYPES[entities.get(id)!.type].bytes!.kind;
+    fs.mkdirSync(path.join(dir, kind), {recursive: true});
+    fs.writeFileSync(bytesPath(dir, kind, id), buf);
+  }
+
+  manifest.formatVersion = 1;
+  manifest.pulls.push({at: new Date().toISOString(), args: meta.args, ids});
+  manifest.packages = [...new Set([...manifest.packages, ...meta.packages])].sort();
+  if (meta.externals?.length) {
+    const byId = new Map((manifest.externals ?? []).map((e) => [e.id, e]));
+    for (const e of meta.externals) byId.set(e.id, e);
+    manifest.externals = [...byId.values()].sort((a, b) => a.nqName.localeCompare(b.nqName));
+  }
+  if (meta.dangling?.length)
+    manifest.dangling = [...new Set([...(manifest.dangling ?? []), ...meta.dangling])].sort();
+  const pullOf = new Map<string, number>();
+  manifest.pulls.forEach((p, i) => p.ids.forEach((id) => pullOf.has(id) || pullOf.set(id, i)));
+  manifest.order = [...byId.values()].sort((a, b) =>
+    rankOf(a.type) - rankOf(b.type) || pullOf.get(a.id)! - pullOf.get(b.id)! || a.file.localeCompare(b.file));
+  fs.writeFileSync(path.join(dir, 'manifest.json'), JSON.stringify(manifest, null, 2));
+  return manifest;
+}
+
+export function read(dir: string): Bundle {
+  if (!fs.existsSync(path.join(dir, 'manifest.json')))
+    throw new Error(`Not a bundle directory (no manifest.json): ${dir}`);
+  const manifest = readManifest(dir);
+  const entities = new Map<string, BundleEntity>();
+  for (const e of manifest.order) {
+    const file = path.join(dir, e.file);
+    if (!fs.existsSync(file))
+      throw new Error(`Bundle is missing ${e.file} listed in manifest.json`);
+    entities.set(e.id, {type: e.type, json: JSON.parse(fs.readFileSync(file, 'utf8')), file: e.file});
+  }
+  const idmapFile = path.join(dir, 'idmap.json');
+  const idmap = fs.existsSync(idmapFile) ? JSON.parse(fs.readFileSync(idmapFile, 'utf8')) : {};
+  return {dir, manifest, entities, idmap};
+}
+
+/** Adopted `sourceId → targetId` pairs, so the next push of this bundle is stable. */
+export function writeIdmap(dir: string, idmap: Record<string, string>): void {
+  fs.writeFileSync(path.join(dir, 'idmap.json'), JSON.stringify(idmap, null, 2));
+}
+
+export function list(dir: string): any[] {
+  const {manifest, entities} = read(dir);
+  return manifest.order.map((e) => {
+    const pull = manifest.pulls.findIndex((p) => p.ids.includes(e.id));
+    const json = entities.get(e.id)!.json;
+    return {
+      type: e.type,
+      nqName: nqNameOf(json),
+      file: e.file,
+      pulledOn: manifest.pulls[pull]?.at ?? '',
+      pull: pull + 1,
+    };
+  });
+}

package/bin/utils/migrate/parts.js

@@ -0,0 +1,83 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.SWEEP = void 0;
+exports.missingPackages = missingPackages;
+exports.missingUsers = missingUsers;
+exports.namespacesOf = namespacesOf;
+exports.plannedParts = plannedParts;
+exports.readState = readState;
+exports.writeState = writeState;
+var fs = _interopRequireWildcard(require("fs"));
+var path = _interopRequireWildcard(require("path"));
+function _interopRequireWildcard(e, t) { if ("function" == typeof WeakMap) var r = new WeakMap(), n = new WeakMap(); return (_interopRequireWildcard = function (e, t) { if (!t && e && e.__esModule) return e; var o, i, f = { __proto__: null, default: e }; if (null === e || "object" != typeof e && "function" != typeof e) return f; if (o = t ? n : r) { if (o.has(e)) return o.get(e); o.set(e, f); } for (const t in e) "default" !== t && {}.hasOwnProperty.call(e, t) && ((i = (o = Object.defineProperty) && Object.getOwnPropertyDescriptor(e, t)) && (i.get || i.set) ? o(f, t, i) : f[t] = e[t]); return f; })(e, t); }
+/// Docs: [Entity export / import](/docs/features/grok-tool/export-import/DESIGN.md)
+
+/**
+ * Every space worth migrating on its own: the personal space of each user, and each team space.
+ * A whole instance moved in one bundle has every project competing for the same entities, and
+ * placement is exclusive on the server — scoping a run to one space keeps that contention inside
+ * it, which is the difference between a run that finishes and one that does not.
+ */
+async function namespacesOf(dapi) {
+  const names = new Set();
+  for (const p of await dapi.internal('/projects').listAll({
+    includeRoot: 'true'
+  })) {
+    if (p?.isEntity || p?.isPackage) continue;
+    // The platform's own space: its connections are already refused as `platform_connection`, the
+    // target builds its own, and listing what is under it does not answer.
+    if (String(p.namespace ?? '').startsWith('System:') || p.name === 'System') continue;
+    const own = String(p.namespace ?? '').split(':')[0];
+    if (own) names.add(own);else if (p.isRoot && p.name) names.add(String(p.name));
+  }
+  return [...names].sort();
+}
+
+/** Users the source has and the target does not: their content lands under the pusher instead. */
+async function missingUsers(from, to) {
+  const here = new Set((await to.internal('/users').listAll({})).map(u => u.login));
+  return (await from.internal('/users').listAll({})).map(u => u.login).filter(login => login && !here.has(login)).sort();
+}
+
+/** Packages the source has and the target does not: their functions and connections cannot resolve. */
+async function missingPackages(from, to) {
+  const here = new Set((await to.internal('/packages').listAll({})).map(p => p.name));
+  return (await from.internal('/packages').listAll({})).map(p => p.name).filter(name => name && !here.has(name)).sort();
+}
+
+/** A run is resumable: what finished is remembered, so a repeat does not redo it. */
+function readState(file) {
+  try {
+    return JSON.parse(fs.readFileSync(file, 'utf8'));
+  } catch {
+    return {};
+  }
+}
+function writeState(file, state) {
+  fs.mkdirSync(path.dirname(file), {
+    recursive: true
+  });
+  fs.writeFileSync(file, JSON.stringify(state, null, 2));
+}
+
+/** The last part: whatever no space owns, which would otherwise never travel. */
+const SWEEP = exports.SWEEP = '(unowned)';
+
+/**
+ * Which parts a run still has to do. A name that matches no space is a typo, not an empty
+ * selection — silently migrating nothing is the one outcome worse than refusing.
+ */
+function plannedParts(all, opts) {
+  const only = opts.only ?? [];
+  const skip = new Set(opts.skip ?? []);
+  const unknown = [...only, ...(opts.skip ?? [])].filter(n => !all.includes(n));
+  if (unknown.length) throw new Error(`No such space: ${unknown.join(', ')}`);
+  const chosen = all.filter(n => (!only.length || only.includes(n)) && !skip.has(n));
+  if (opts.sweep) chosen.push(SWEEP);
+  // A part that failed, or that reported failures, is not finished.
+  const done = opts.state ?? {};
+  return chosen.filter(n => !done[n] || done[n].error || done[n].failed);
+}

package/bin/utils/migrate/parts.ts

@@ -0,0 +1,72 @@
+/// Docs: [Entity export / import](/docs/features/grok-tool/export-import/DESIGN.md)
+import * as fs from 'fs';
+import * as path from 'path';
+import {NodeDapi} from '../node-dapi';
+
+export interface Part {name: string; entities?: number; failed?: number; seconds?: number; error?: string}
+
+/**
+ * Every space worth migrating on its own: the personal space of each user, and each team space.
+ * A whole instance moved in one bundle has every project competing for the same entities, and
+ * placement is exclusive on the server — scoping a run to one space keeps that contention inside
+ * it, which is the difference between a run that finishes and one that does not.
+ */
+export async function namespacesOf(dapi: NodeDapi): Promise<string[]> {
+  const names = new Set<string>();
+  for (const p of await dapi.internal('/projects').listAll({includeRoot: 'true'})) {
+    if (p?.isEntity || p?.isPackage) continue;
+    // The platform's own space: its connections are already refused as `platform_connection`, the
+    // target builds its own, and listing what is under it does not answer.
+    if (String(p.namespace ?? '').startsWith('System:') || p.name === 'System') continue;
+    const own = String(p.namespace ?? '').split(':')[0];
+    if (own) names.add(own);
+    else if (p.isRoot && p.name) names.add(String(p.name));
+  }
+  return [...names].sort();
+}
+
+/** Users the source has and the target does not: their content lands under the pusher instead. */
+export async function missingUsers(from: NodeDapi, to: NodeDapi): Promise<string[]> {
+  const here = new Set((await to.internal('/users').listAll({})).map((u: any) => u.login));
+  return (await from.internal('/users').listAll({}))
+    .map((u: any) => u.login).filter((login: string) => login && !here.has(login)).sort();
+}
+
+/** Packages the source has and the target does not: their functions and connections cannot resolve. */
+export async function missingPackages(from: NodeDapi, to: NodeDapi): Promise<string[]> {
+  const here = new Set((await to.internal('/packages').listAll({})).map((p: any) => p.name));
+  return (await from.internal('/packages').listAll({}))
+    .map((p: any) => p.name).filter((name: string) => name && !here.has(name)).sort();
+}
+
+/** A run is resumable: what finished is remembered, so a repeat does not redo it. */
+export function readState(file: string): Record<string, Part> {
+  try { return JSON.parse(fs.readFileSync(file, 'utf8')); }
+  catch { return {}; }
+}
+
+export function writeState(file: string, state: Record<string, Part>): void {
+  fs.mkdirSync(path.dirname(file), {recursive: true});
+  fs.writeFileSync(file, JSON.stringify(state, null, 2));
+}
+
+/** The last part: whatever no space owns, which would otherwise never travel. */
+export const SWEEP = '(unowned)';
+
+/**
+ * Which parts a run still has to do. A name that matches no space is a typo, not an empty
+ * selection — silently migrating nothing is the one outcome worse than refusing.
+ */
+export function plannedParts(all: string[], opts: {only?: string[]; skip?: string[];
+                             state?: Record<string, Part>; sweep?: boolean}): string[] {
+  const only = opts.only ?? [];
+  const skip = new Set(opts.skip ?? []);
+  const unknown = [...only, ...(opts.skip ?? [])].filter((n) => !all.includes(n));
+  if (unknown.length)
+    throw new Error(`No such space: ${unknown.join(', ')}`);
+  const chosen = all.filter((n) => (!only.length || only.includes(n)) && !skip.has(n));
+  if (opts.sweep) chosen.push(SWEEP);
+  // A part that failed, or that reported failures, is not finished.
+  const done = opts.state ?? {};
+  return chosen.filter((n) => !done[n] || done[n].error || done[n].failed);
+}

package/bin/utils/migrate/pool.js

@@ -0,0 +1,17 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.pool = pool;
+/// Docs: [Entity export / import](/docs/features/grok-tool/export-import/DESIGN.md)
+
+/** Runs `work` over `items` with at most `concurrency` in flight, in the order given. */
+async function pool(items, concurrency, work) {
+  const queue = items.slice();
+  const workers = [];
+  for (let i = 0; i < Math.min(concurrency, queue.length); i++) workers.push((async () => {
+    while (queue.length) await work(queue.shift());
+  })());
+  await Promise.all(workers);
+}