datagrok-tools 6.7.8 → 6.7.10

package/bin/commands/connect-folder.js

@@ -0,0 +1,411 @@
+"use strict";
+
+var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.checkFolderSafety = checkFolderSafety;
+exports.connectFolder = connectFolder;
+exports.matches = matches;
+exports.withKeyFile = withKeyFile;
+var _fs = _interopRequireDefault(require("fs"));
+var _os = _interopRequireDefault(require("os"));
+var _path = _interopRequireDefault(require("path"));
+var color = _interopRequireWildcard(require("../utils/color-utils"));
+var _keypair = require("../utils/keypair");
+var _serverClient = require("../utils/server-client");
+var _nodeDapi = require("../utils/node-dapi");
+var _serverOutput = require("../utils/server-output");
+var dk = _interopRequireWildcard(require("../utils/connect-folder/docker"));
+function _interopRequireWildcard(e, t) { if ("function" == typeof WeakMap) var r = new WeakMap(), n = new WeakMap(); return (_interopRequireWildcard = function (e, t) { if (!t && e && e.__esModule) return e; var o, i, f = { __proto__: null, default: e }; if (null === e || "object" != typeof e && "function" != typeof e) return f; if (o = t ? n : r) { if (o.has(e)) return o.get(e); o.set(e, f); } for (const t in e) "default" !== t && {}.hasOwnProperty.call(e, t) && ((i = (o = Object.defineProperty) && Object.getOwnPropertyDescriptor(e, t)) && (i.get || i.set) ? o(f, t, i) : f[t] = e[t]); return f; })(e, t); }
+/**
+ * `grok connect-folder <folder>` - makes a local folder a Datagrok connection. The connection is created over REST
+ * with the keypair session, and served by the proxy-storage agent in a Docker container that dials out
+ * to the server; `list`, `start`, `stop`, `rm` and `logs` manage the containers of this machine.
+ */
+async function connectFolder(args) {
+  const follow = flag(args, 'f') || args.follow === true;
+  const output = args.output ?? args.o ?? 'table';
+  (0, _serverOutput.setOutputFormat)(output);
+  const [, sub, ref, ...extra] = args._.map(a => `${a}`);
+  if (sub == null || extra.length > 0) return false;
+  try {
+    switch (sub) {
+      case 'list':
+        return ref == null && (await list(args.host, output));
+      case 'start':
+      case 'stop':
+      case 'logs':
+        return ref != null && (await manage(sub, ref, args.host, follow));
+      case 'rm':
+        return ref != null && (await remove(ref, args.host));
+      default:
+        return ref == null && (await create(sub, {
+          host: args.host,
+          image: args.image,
+          readOnly: args['read-only'] === true,
+          foreground: args.foreground === true,
+          force: args.force === true
+        }));
+    }
+  } catch (e) {
+    (0, _serverOutput.printError)(e);
+    return false;
+  }
+}
+async function create(folderArg, opts) {
+  const cred = shareCredentials(opts.host);
+  await dk.dockerAvailable();
+  let folder;
+  let note;
+  try {
+    ({
+      path: folder,
+      note
+    } = dk.hostFolder(folderArg));
+  } catch (e) {
+    throw e.code === 'ENOENT' ? new Error(`${folderArg} does not exist`) : e;
+  }
+  if (!_fs.default.statSync(folder).isDirectory()) throw new Error(`${folder} is not a folder`);
+  checkFolderSafety(folder, opts.force, _os.default.homedir(), process.platform, cred.privateKeySource && cred.privateKeySource !== 'GROK_PRIVATE_KEY' ? [cred.privateKeySource] : []);
+  if (note) color.warn(note);
+  const alias = cred.alias ?? cred.url;
+  const client = await (0, _serverClient.createClient)(alias);
+  const before = await client.get('/local_shares');
+  const created = await color.step(`Connecting ${folder} to ${cred.url}`, () => client.post('/local_shares', {
+    host: dk.canonicalHost(),
+    hostLabel: _os.default.hostname(),
+    folder: dk.canonicalFolder(folder),
+    folderLabel: folder,
+    readOnly: opts.readOnly
+  })).catch(e => {
+    throw e?.apiError?.errorCode === 403 && `${e.apiError.error}`.includes('ConnectLocalFolders') ? new Error(`You don't have permission to connect folders on ${new URL(cred.url).host}. Ask an administrator.`) : e;
+  });
+  const conn = created.connection;
+  const name = conn.friendlyName ?? conn.name;
+  if (!opts.image && !dk.isAgentImage(created.agentImage)) throw new Error(`${new URL(cred.url).host} asks to run the folder agent from ${created.agentImage}, not ${dk.AGENT_REPOSITORY}. ` + `Pass --image ${created.agentImage} to run it anyway.`);
+  const spec = {
+    image: opts.image ?? created.agentImage,
+    connectionId: conn.id,
+    alias,
+    apiRoot: cred.url,
+    name,
+    folder,
+    hostLabel: _os.default.hostname(),
+    readOnly: opts.readOnly,
+    envFile: '',
+    foreground: opts.foreground,
+    user: dk.linuxUser()
+  };
+  const containers = await dk.ps({
+    [`${dk.LABEL}.connection`]: conn.id,
+    [`${dk.LABEL}.url`]: cred.url
+  });
+  const existing = containers.find(c => c.name === dk.containerName(cred.url, conn.id));
+  const keys = await shareKeys(client, conn.id);
+  // A key is registered only once the image is here: a Ctrl+C during a long pull then leaves nothing behind.
+  const image = await color.step(`Getting the folder agent image ${spec.image}`, () => dk.ensureImage(spec.image));
+  if (image.pullError) color.warn(`Could not pull ${spec.image}, using the local copy: ${image.pullError}`);
+  spec.imageId = image.id;
+  if (existing && !opts.foreground && dk.sameSettings(existing, spec) && keys.some(k => k.id === existing.labels[`${dk.LABEL}.key`])) {
+    // A share deleted on the server leaves its agent idle (4410), so the recreated one needs a fresh start.
+    const wasDeleted = !before.some(s => s.id === conn.id);
+    if (existing.state === 'running' && !wasDeleted) color.info(`"${name}" is already connected from this machine (container ${existing.name})`);else await color.step(`Starting container ${existing.name}`, () => dk.dockerOk([wasDeleted ? 'restart' : 'start', existing.name]));
+    return await reportOnline(client, conn.id, name, existing.name);
+  }
+  for (const c of containers) await color.step(`Replacing container ${c.name}`, () => dk.rm(c.name));
+  const key = await issueShareKey(client, conn.id);
+  const revokeMine = () => client.del(`/users/keys/${key.id}`).catch(() => {});
+  spec.keyId = key.id;
+  const container = dk.containerName(cred.url, conn.id, opts.foreground);
+  if (opts.foreground) return await runForeground(client, spec, key.privateKey, container, name, revokeMine);
+  try {
+    await withKeyFile(key.privateKey, envFile => color.step('Starting the folder agent', () => dk.dockerOk(dk.runArgs({
+      ...spec,
+      envFile
+    }))), revokeMine);
+  } catch (e) {
+    await revokeMine();
+    throw e;
+  }
+  if (await dk.isRunning(container)) await revokeShareKeys(client, conn.id, key.id);
+  if (process.platform === 'win32' || process.platform === 'darwin') color.warn('The folder comes back after a reboot only when Docker does: let Docker Desktop start when you sign in.');
+  return await reportOnline(client, conn.id, name, container);
+}
+
+/** `docker create` reads the key file, which is deleted before the agent starts; `start -a` forwards Ctrl+C. */
+async function runForeground(client, spec, key, container, name, revokeMine) {
+  try {
+    await withKeyFile(key, envFile => dk.dockerOk(['create', ...dk.runArgs({
+      ...spec,
+      envFile
+    }).slice(1)]), revokeMine);
+    color.info(`Serving "${name}" until Ctrl+C`);
+    process.on('SIGINT', () => {
+      dk.docker(['stop', container]).catch(() => {});
+    });
+    const attached = dk.docker(['start', '-a', container], true);
+    if (await dk.isRunning(container, 10)) await revokeShareKeys(client, spec.connectionId, spec.keyId).catch(() => {});
+    const r = await attached;
+    return r.code === 0 || r.code === 130 || r.code === 143;
+  } finally {
+    await revokeMine();
+  }
+}
+
+/**
+ * [key], base64 in a 0600 `--env-file` that exists only while [action] runs; Ctrl+C meanwhile deletes it and
+ * runs [onAbort] before exiting.
+ */
+async function withKeyFile(key, action, onAbort = async () => {}) {
+  const dir = _fs.default.mkdtempSync(_path.default.join(_os.default.tmpdir(), 'grok-folder-'));
+  const file = _path.default.join(dir, 'key.env');
+  const abort = signal => {
+    _fs.default.rmSync(dir, {
+      recursive: true,
+      force: true
+    });
+    onAbort().finally(() => process.exit(signal === 'SIGINT' ? 130 : 143));
+  };
+  process.once('SIGINT', abort);
+  process.once('SIGTERM', abort);
+  try {
+    _fs.default.writeFileSync(file, `GROK_SHARE_KEY=${Buffer.from(JSON.stringify(key)).toString('base64')}\n`, {
+      mode: 0o600
+    });
+    return await action(file);
+  } finally {
+    process.off('SIGINT', abort);
+    process.off('SIGTERM', abort);
+    _fs.default.rmSync(dir, {
+      recursive: true,
+      force: true
+    });
+  }
+}
+
+/** The container never gets the user's own key: the server issues each share one that can only serve it. */
+async function issueShareKey(client, connectionId) {
+  const {
+    publicKey,
+    privateKey
+  } = (0, _keypair.generateKeyPair)();
+  const key = await client.post(`/local_shares/${connectionId}/agent_key`, {
+    publicKey: JSON.stringify(publicKey)
+  });
+  return {
+    id: key.id,
+    privateKey
+  };
+}
+const folderScope = connectionId => `connected-folder:${connectionId}`;
+async function shareKeys(client, connectionId) {
+  return ((await client.get('/users/current/keys')) ?? []).filter(k => k.scope === folderScope(connectionId));
+}
+async function revokeShareKeys(client, connectionId, keep) {
+  for (const k of await shareKeys(client, connectionId)) {
+    if (k.id !== keep) await client.del(`/users/keys/${k.id}`);
+  }
+}
+async function reportOnline(client, id, name, container) {
+  const online = await color.step('Waiting for the agent to connect', async () => {
+    for (let i = 0; i < 30; i++) {
+      const s = (await client.get('/local_shares')).find(s => s.id === id);
+      if (s != null && s.isAvailable !== false) return true;
+      await new Promise(r => setTimeout(r, 1000));
+    }
+    return false;
+  });
+  if (!online) {
+    color.warn(`"${name}" is connected, but its agent has not dialed in yet. See: grok connect-folder logs ${id.slice(0, 8)}`);
+    return true;
+  }
+  color.success(`"${name}" is online (connection ${id}, container ${container})`);
+  return true;
+}
+async function list(host, output) {
+  const containers = await dockerContainersOrEmpty();
+  const own = (0, _keypair.getServerCredentials)(host ?? '');
+  const aliases = new Set([own.alias ?? own.url]);
+  if (host == null) {
+    for (const c of containers) aliases.add(c.labels[`${dk.LABEL}.alias`]);
+  }
+  const rows = [];
+  const seen = new Set();
+  for (const alias of aliases) {
+    let shares = null;
+    let keys = [];
+    let url;
+    try {
+      const cred = (0, _keypair.getServerCredentials)(alias);
+      url = cred.url;
+      const client = await (0, _serverClient.createClient)(alias);
+      shares = await client.get('/local_shares');
+      keys = (await client.get('/users/current/keys')) ?? [];
+    } catch (e) {
+      color.warn(`${alias}: ${e.message ?? e}`);
+      url = '';
+    }
+    const ofServer = containers.filter(c => c.labels[`${dk.LABEL}.url`] === url || c.labels[`${dk.LABEL}.alias`] === alias);
+    for (const s of shares ?? []) {
+      const c = ofServer.find(c => c.labels[`${dk.LABEL}.connection`] === s.id);
+      seen.add(`${alias}|${s.id}`);
+      // The server refuses an agent whose key is not limited to its folder: its container runs, but serves nothing.
+      const outdated = c != null && !keys.some(k => k.id === c.labels[`${dk.LABEL}.key`] && k.scope === folderScope(s.id));
+      rows.push(row(s.friendlyName ?? s.name, s.parameters?.localShareFolder, s.parameters?.localShareReadOnly === 'true', c?.state, outdated ? 'outdated key: re-run connect-folder' : s.isAvailable === false ? 'offline' : 'online', s.parameters?.localShareHost, s.id, alias));
+    }
+    for (const c of ofServer) {
+      const id = c.labels[`${dk.LABEL}.connection`];
+      if (seen.has(`${alias}|${id}`)) continue;
+      seen.add(`${alias}|${id}`);
+      rows.push(row(c.labels[`${dk.LABEL}.name`], c.labels[`${dk.LABEL}.folder`], c.labels[`${dk.LABEL}.readonly`] === 'true', c.state, shares == null ? 'unknown' : 'revoked', _os.default.hostname(), id, alias));
+    }
+  }
+  (0, _serverOutput.printOutput)(rows, output);
+  return true;
+}
+function row(name, folder, readOnly, container, server, host, id, alias) {
+  return {
+    name,
+    folder,
+    access: readOnly ? 'read-only' : 'read-write',
+    container: container ?? '-',
+    server,
+    host,
+    id,
+    alias
+  };
+}
+async function dockerContainersOrEmpty() {
+  try {
+    return await dk.ps();
+  } catch (e) {
+    color.warn(`${e.message ?? e} Only the server side is listed.`);
+    return [];
+  }
+}
+async function manage(command, ref, host, follow) {
+  await dk.dockerAvailable();
+  const c = findContainer(inScope(await dk.ps(), host), ref);
+  if (command === 'logs') return (await dk.logs(c.name, follow)).code === 0;
+  const name = c.labels[`${dk.LABEL}.name`];
+  await color.step(`${command === 'start' ? 'Starting' : 'Stopping'} "${name}"`, () => command === 'start' ? dk.start(c.name) : dk.stop(c.name));
+  if (command === 'stop') return true;
+  let client;
+  try {
+    client = await (0, _serverClient.createClient)(c.labels[`${dk.LABEL}.alias`]);
+  } catch (e) {
+    color.warn(`"${name}" is started, but ${c.labels[`${dk.LABEL}.url`]} cannot be reached to check it: ${e.message ?? e}`);
+    return true;
+  }
+  return await reportOnline(client, c.labels[`${dk.LABEL}.connection`], name, c.name);
+}
+
+/** The containers of the server [host] names (all of them without one), by alias or by URL. */
+function inScope(containers, host) {
+  if (host == null) return containers;
+  let url;
+  try {
+    url = (0, _keypair.getServerCredentials)(host).url;
+  } catch {/* an alias that is gone can still name its containers */}
+  return containers.filter(c => c.labels[`${dk.LABEL}.alias`] === host || c.labels[`${dk.LABEL}.url`] === url);
+}
+
+/**
+ * Removes the share's containers on this machine, then its connection and keys on the server. The containers go
+ * even when the server cannot be reached.
+ */
+async function remove(ref, host) {
+  const containers = await dockerContainersOrEmpty();
+  const found = matchContainers(inScope(containers, host), ref);
+  const ids = [...new Set(found.map(c => c.labels[`${dk.LABEL}.connection`]))];
+  if (new Set(found.map(c => `${c.labels[`${dk.LABEL}.alias`]}|${c.labels[`${dk.LABEL}.connection`]}`)).size > 1) throw ambiguous(ref, found.map(c => `${c.labels[`${dk.LABEL}.name`]} (${c.labels[`${dk.LABEL}.alias`]})`));
+  const alias = host ?? found[0]?.labels[`${dk.LABEL}.alias`] ?? '';
+  let id = ids[0];
+  let client;
+  let serverError;
+  try {
+    client = await (0, _serverClient.createClient)(alias);
+    if (id == null) {
+      const shares = (await client.get('/local_shares')).filter(s => matches(ref, s.id, s.friendlyName ?? s.name, s.parameters?.localShareFolder));
+      if (shares.length > 1) throw ambiguous(ref, shares.map(s => s.friendlyName ?? s.name));
+      id = shares[0]?.id;
+    }
+  } catch (e) {
+    serverError = new Error(`${alias || 'the default server'}: ${e?.message ?? e}`);
+  }
+  if (id == null) throw serverError ?? new Error(`No connected folder matches "${ref}". See: grok connect-folder list`);
+  for (const c of inScope(containers, alias).filter(c => c.labels[`${dk.LABEL}.connection`] === id)) await color.step(`Removing container ${c.name}`, () => dk.rm(c.name));
+  if (client && !serverError) {
+    let doing = 'revoke the folder key';
+    try {
+      await color.step('Revoking the folder key', () => revokeShareKeys(client, id));
+      doing = 'delete the connection';
+      await color.step(`Deleting connection ${id}`, () => new _nodeDapi.NodeDapi(client).connections.delete({
+        id: id
+      }));
+    } catch (e) {
+      serverError = new Error(`could not ${doing}: ${e?.message ?? e}`);
+    }
+  }
+  if (serverError) {
+    color.error(`Removed on this machine, but the connection is still on the server: ${serverError.message ?? serverError}. ` + `Run grok connect-folder rm ${id.slice(0, 8)} again, or delete the connection in Datagrok`);
+    return false;
+  }
+  return true;
+}
+function findContainer(containers, ref) {
+  const found = matchContainers(containers, ref);
+  if (found.length === 0) throw new Error(`No folder container on this machine matches "${ref}". See: grok connect-folder list`);
+  if (found.length > 1) throw ambiguous(ref, found.map(c => `${c.labels[`${dk.LABEL}.name`]} (${c.labels[`${dk.LABEL}.alias`]})`));
+  return found[0];
+}
+function matchContainers(containers, ref) {
+  return containers.filter(c => matches(ref, c.labels[`${dk.LABEL}.connection`], c.labels[`${dk.LABEL}.name`], c.labels[`${dk.LABEL}.folder`]));
+}
+
+/** A share's `<ref>`: its name, a connection id prefix of 8 or more characters, or its folder. */
+function matches(ref, id, name, folder) {
+  if (name != null && name.toLowerCase() === ref.toLowerCase()) return true;
+  if (ref.length >= 8 && id?.toLowerCase().startsWith(ref.toLowerCase())) return true;
+  if (folder == null) return false;
+  let abs = _path.default.resolve(ref);
+  try {
+    abs = _fs.default.realpathSync.native(abs);
+  } catch {/* a folder that is gone can still name its share */}
+  return dk.canonicalFolder(abs) === dk.canonicalFolder(folder);
+}
+function ambiguous(ref, names) {
+  return new Error(`"${ref}" matches several connected folders: ${names.join(', ')}. Use the connection id, or --host`);
+}
+
+/** The server alias with the keypair `grok connect-folder` signs in with (the agent gets a key of its own). */
+function shareCredentials(host) {
+  const cred = (0, _keypair.getServerCredentials)(host ?? '');
+  if (cred.privateKey == null) throw new Error(`grok connect-folder signs in with a keypair, and ${cred.alias ?? cred.url} has none: run grok login ${cred.url}`);
+  return cred;
+}
+
+/**
+ * Refuses a folder that is, holds or is inside `~/.grok` or one of [keyFiles] (private keys); a filesystem or
+ * drive root, or the home folder, needs [force].
+ */
+function checkFolderSafety(folder, force, home = _os.default.homedir(), platform = process.platform, keyFiles = []) {
+  const f = dk.canonicalFolder(folder, platform);
+  const within = (child, parent) => child === parent || child.startsWith(parent.endsWith('/') ? parent : `${parent}/`);
+  for (const secret of [_path.default.join(home, '.grok'), ...keyFiles]) {
+    const s = dk.canonicalFolder(secret, platform);
+    if (within(f, s) || within(s, f)) throw new Error(`${folder} is or holds ${secret}, where your private keys are; it cannot be connected`);
+  }
+  if (!force && (f === '/' || /^[a-z]:\/?$/i.test(f) || f === dk.canonicalFolder(home, platform))) throw new Error(`${folder} is a root or your home folder; pass --force to connect it anyway`);
+}
+
+/** minimist reads `-f <connection>` as a value of the flag, so the value goes back to the positionals. */
+function flag(args, name) {
+  const v = args[name];
+  if (typeof v === 'string' && v !== '') {
+    args._.push(v);
+    return true;
+  }
+  return v === true;
+}

package/bin/commands/help.js

@@ -31,6 +31,8 @@ Commands:
     testall     Run packages tests
     migrate     Migrate legacy tags to meta.role
     server (s)  Manage a Datagrok server (list/get/delete entities, run functions)
+    src         Work across the repos of an umbrella checkout (git submodules)
+    connect-folder  Connect a local folder to Datagrok as a connection (runs a Docker container)
 
 To get help on a particular command, use:
     grok <command> --help
@@ -96,7 +98,7 @@ grok add tests
 Please note that entity names may only include letters and numbers
 
 --domain scaffolds a working browse/CRUD app over an entity-mapped domain table
-(\`grok.dapi.domains\`) from the u2 defaults (\`@datagrok-libraries/u2\`): a three-line
+(\`grok.dapi.domains\`) from the EMS defaults (\`@datagrok-libraries/ems\`): a three-line
 app function, a \`src/app.spec.json\` the designer edits, and — for a schema the package
 does not declare yet — a one-table starter \`databases/<schema>/schema.json\`. Give it
 one table (\`--domain tracker.issue\`), a whole schema the package declares in
@@ -140,7 +142,7 @@ Options:
 --verbose         Print detailed output
 --ui              Also generate \`src/generated/db-ui.ts\` — typed u2 handles
                   (\`get<Schema>Db()\` → a \`DomainTable<Row>\` per table, opened in
-                  parallel behind one await) over \`@datagrok-libraries/u2\`. Once the
+                  parallel behind one await) over \`@datagrok-libraries/ems\`. Once the
                   file exists, plain \`grok api\` keeps it up to date; delete it to
                   opt out again
 `;
@@ -167,6 +169,36 @@ Options:
 For CI, set GROK_PRIVATE_KEY to the private key JWK (raw or base64) instead of a
 config file. Read more: https://datagrok.ai/help/govern/access-control/keypair-authentication
 `;
+const HELP_CONNECT_FOLDER = `
+Usage: grok connect-folder <folder> [--host <alias>] [--read-only] [--foreground] [--image <ref>] [--force]
+       grok connect-folder list [--host <alias>] [--output table|json|csv]
+       grok connect-folder start|stop|logs <connection> [-f]
+       grok connect-folder rm <connection> [--host <alias>]
+
+Connect a local folder to Datagrok as a connection. The folder agent runs in
+Docker, survives closing the terminal, and dials out to the server: no inbound
+port is opened. Connecting the same folder again resumes the same connection.
+
+grok connect-folder ~/data                Read-write, on the default server
+grok connect-folder D:\\data --read-only   Others with access can only read
+grok connect-folder list                  Your connected folders and their containers
+grok connect-folder stop "laptop data"    Stop serving; stays stopped after a reboot
+grok connect-folder rm 7ff0714e           Remove the container and delete the connection
+
+<connection> is the connection name, a connection id prefix (8+ characters), or
+the folder.
+
+Options:
+--host        Server alias from ~/.grok/config.yaml (default: the default server)
+--read-only   Mount the folder read-only; writes are refused
+--foreground  Run attached, removed on Ctrl+C (CI, debugging)
+--image       Agent image (default: the one the server names)
+--force       Allow a drive root or the home folder
+-f            With logs: follow
+
+Requires Docker and a keypair (grok login <server>), which never leaves this
+machine: the container holds a key of its own that can only serve this folder.
+`;
 const HELP_CONFIG = `
 Usage: grok config
 
@@ -469,6 +501,38 @@ Examples:
 
 The instance name must match a server alias in ~/.grok/config.yaml.
 `;
+const HELP_SRC = `
+Usage: grok src <command> [options]
+
+Git recipes across the repos of the datagrok umbrella (git submodules). Runs from anywhere
+in the tree; paths are relative to the umbrella root. Every git command that changes
+something is printed; a repo that fails is listed and the exit code is 1.
+
+Commands:
+    clone [profile | path...]   Clone the umbrella and the given children (profiles: .gitprofiles;
+                                none: profile "default", else every child)
+    add <profile | path>...     Clone more children, each on its branch
+    sparse <path> <glob>...     Clone a child with only the matching files
+    switch                      Put every child on its branch from .gitmodules
+    status                      Branch, unpushed commits and changes of every repo
+    pull                        git pull --ff-only in every repo
+    at <ref>                    Check out an umbrella ref with its pinned children
+    bump [path...]              Pin children to their branch tip on origin and move clean checkouts
+                                there; commit in each umbrella (never pushes)
+    wt add <name> <path>...     Worktrees of the umbrella and the given children on branch <name>
+    doctor [--fix]              Detached HEADs, unpushed commits, stale URLs; --fix resets those URLs
+
+Options:
+    --explain     Print the git commands without running them
+    --url <url>   clone: the umbrella (default git@github.com:datagrok-ai/datagrok.git)
+    --dir <dir>   clone: target directory; wt add: worktree root (default <umbrella>-worktrees/<name>)
+
+Examples:
+    grok src clone platform
+    grok src sparse help '*.md'
+    grok src bump core public
+    grok src wt add fix-login core public
+`;
 const help = exports.help = {
   add: HELP_ADD,
   api: HELP_API,
@@ -491,5 +555,7 @@ const help = exports.help = {
   server: _server.HELP_SERVER,
   s: _server.HELP_SERVER,
   setup: HELP_SETUP,
+  src: HELP_SRC,
+  'connect-folder': HELP_CONNECT_FOLDER,
   help: HELP
 };

package/bin/commands/publish.js

@@ -196,6 +196,12 @@ function dockerBuild(imageName, dockerfilePath, context, usePlatform = true) {
     });
     return true;
   } catch (e) {
+    // A local build targets the host platform, which fails on Apple Silicon for a base
+    // published only for linux/amd64 (datagrok/demo_db_*).
+    if (!usePlatform && process.arch !== 'x64') {
+      color.warn(`Building ${imageName} for the host platform failed. Retrying for linux/amd64...`);
+      return dockerBuild(imageName, dockerfilePath, context, true);
+    }
     color.error(`Failed to build ${imageName}: ${e.message || e}`);
     return false;
   }

package/bin/commands/server-domains.js

@@ -21,7 +21,9 @@ function _interopRequireWildcard(e, t) { if ("function" == typeof WeakMap) var r
 /// `grok s domains ...` — entity-mapped domain schemas and their rows (DomainsRouter, `/domains/`).
 /// Every verb addresses a schema as `<schema>` or a table as `<schema>.<table>`.
 
-const APPLY_KEYS = ['tables', 'extend', 'propertySchemas', 'dropTables', 'ifVersion', 'confirmDestructive'];
+const APPLY_KEYS = ['tables', 'extend', 'propertySchemas', 'dropTables', 'access', 'ifVersion', 'ifIncarnation', 'confirmDestructive'];
+/** Accepted by a user-managed schema's apply only; a package schema refuses them as unknown keys. */
+const USER_APPLY_KEYS = ['friendlyName', 'description', 'storage'];
 const PERMISSIONS = ['View', 'Edit', 'Delete', 'Share', 'Extend'];
 const USAGE = `Usage: grok s domains <verb> [args]
   list [<schema>] [--filter <text>]                      Schemas, or the tables of one schema
@@ -210,11 +212,23 @@ async function handleDomains(dapi, verb, rest, argv, output) {
       }
     case 'apply':
       {
-        if (!args[0] || !argv.json) return usage('apply <schema> --json manifest.json [--dry-run] [--confirm-destructive] [--if-version <v>]');
+        if (!args[0] || !argv.json) return usage('apply <schema> --json manifest.json [--dry-run] [--confirm-destructive] [--if-version <v>|current]');
         const a = (0, _nodeDapi.parseDomainAddress)(args[0], {
           table: false
         });
-        const body = applyBody(readJson(argv.json), argv);
+        const json = readJson(argv.json);
+        // One registry read, only when the token or the metadata keys depend on who manages the schema.
+        const s = argv['if-version'] === undefined || argv['if-version'] === 'current' || USER_APPLY_KEYS.some(k => json?.[k] !== undefined) ? await domains.schema(a.schema) : null;
+        const body = applyBody(json, argv, {
+          userManaged: s?.managedBy === 'user'
+        });
+        if (body.ifVersion === 'current') {
+          // Both halves of the token as registered now: a name deleted and re-created
+          // since the file was written must conflict, not pass as a fresh `1`.
+          const m = await domains.manifest(a.schema);
+          body.ifVersion = String(m?.version ?? s.version ?? '1');
+          if (m?.incarnation !== undefined) body.ifIncarnation = String(m.incarnation);
+        } else if (body.ifVersion === undefined && s.managedBy === 'user') return usage(`apply ${a.schema} --json manifest.json --if-version <v>|current — a user-managed schema ` + 'requires the version the edit was made against (the "version" of the manifest JSON is used when present)');
         printJson(await domains.applySchema(a.schema, body, argv['dry-run'] === true), output, r => r?.version);
         return true;
       }
@@ -305,12 +319,15 @@ function aggregateSpec(argv, filter, limit) {
   return spec;
 }
 
-/** The apply payload from a schema.json (or a partial body): only the keys the server accepts, flags folded in. */
-function applyBody(json, argv) {
+/** The apply payload from a schema.json (or a partial body): only the keys the server accepts
+ * (the schema's metadata and storage keys for a user-managed schema only), flags folded in. */
+function applyBody(json, argv, options = {}) {
   const body = {};
-  for (const k of APPLY_KEYS) if (json?.[k] !== undefined) body[k] = json[k];
+  for (const k of options.userManaged ? [...APPLY_KEYS, ...USER_APPLY_KEYS] : APPLY_KEYS) if (json?.[k] !== undefined) body[k] = json[k];
   if (argv['confirm-destructive'] === true) body.confirmDestructive = true;
-  if (argv['if-version'] !== undefined) body.ifVersion = String(argv['if-version']);
+  if (argv['if-version'] !== undefined) body.ifVersion = String(argv['if-version']);else if (json?.version !== undefined) body.ifVersion = String(json.version);
+  // The manifest's `incarnation` rides with its `version` (the server keeps it optional).
+  if (json?.incarnation !== undefined) body.ifIncarnation = String(json.incarnation);
   return body;
 }