datagrok-tools 6.7.9 → 6.7.10

package/bin/utils/connect-folder/docker.js

@@ -0,0 +1,244 @@
+"use strict";
+
+var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.SHARE_ROOT = exports.LABEL = exports.DockerError = exports.AGENT_REPOSITORY = void 0;
+exports.canonicalFolder = canonicalFolder;
+exports.canonicalHost = canonicalHost;
+exports.containerName = containerName;
+exports.containerUrl = containerUrl;
+exports.docker = docker;
+exports.dockerAvailable = dockerAvailable;
+exports.dockerOk = dockerOk;
+exports.ensureImage = ensureImage;
+exports.hostFolder = hostFolder;
+exports.isAgentImage = isAgentImage;
+exports.isRunning = isRunning;
+exports.linuxUser = linuxUser;
+exports.proc = exports.logs = void 0;
+exports.ps = ps;
+exports.rm = void 0;
+exports.runArgs = runArgs;
+exports.sameSettings = sameSettings;
+exports.stop = exports.start = void 0;
+var _child_process = _interopRequireDefault(require("child_process"));
+var _crypto = _interopRequireDefault(require("crypto"));
+var _fs = _interopRequireDefault(require("fs"));
+var _os = _interopRequireDefault(require("os"));
+var _path = _interopRequireDefault(require("path"));
+/** The `docker` CLI for `grok connect-folder`: always an argument vector, never a shell, so no quoting or MSYS path mangling. */
+
+const LABEL = exports.LABEL = 'ai.datagrok.folder';
+const SHARE_ROOT = exports.SHARE_ROOT = '/share';
+/** The only image run with the user's folder mounted unless `--image` names another. */
+const AGENT_REPOSITORY = exports.AGENT_REPOSITORY = 'datagrok/proxy_storage';
+
+/** Whether [image] is [AGENT_REPOSITORY] on Docker Hub, by tag or digest: a server cannot name any other. */
+function isAgentImage(image) {
+  return /^((docker\.io|index\.docker\.io)\/)?datagrok\/proxy_storage(:\w[\w.-]{0,127})?(@sha256:[0-9a-f]{64})?$/.test(image);
+}
+const DOCKER_HOST = 'host.docker.internal';
+class DockerError extends Error {
+  constructor(message, stderr = '') {
+    super(message);
+    this.stderr = stderr;
+    this.name = 'DockerError';
+  }
+}
+
+/** Where `docker` is started; the tests replace it, since a module mock cannot reach the transpiled require. */
+exports.DockerError = DockerError;
+const proc = exports.proc = {
+  spawn: _child_process.default.spawn
+};
+/** Runs `docker <args>`; [inherit] attaches the terminal (logs -f, a foreground share). */
+function docker(args, inherit = false) {
+  return new Promise((resolve, reject) => {
+    const child = proc.spawn('docker', args, {
+      stdio: inherit ? 'inherit' : ['ignore', 'pipe', 'pipe'],
+      windowsHide: true
+    });
+    let stdout = '';
+    let stderr = '';
+    child.stdout?.on('data', d => stdout += d);
+    child.stderr?.on('data', d => stderr += d);
+    child.on('error', e => reject(e?.code === 'ENOENT' ? new DockerError('grok connect-folder runs the folder agent in Docker, which is not installed. ' + 'Install Docker Desktop (Windows/macOS) or Docker Engine (Linux).') : e));
+    child.on('close', code => resolve({
+      code: code ?? 1,
+      stdout,
+      stderr
+    }));
+  });
+}
+async function dockerOk(args) {
+  const r = await docker(args);
+  if (r.code !== 0) throw new DockerError(r.stderr.trim() || `docker ${args[0]} failed with exit code ${r.code}`, r.stderr);
+  return r.stdout;
+}
+async function dockerAvailable() {
+  const version = await docker(['version', '--format', '{{.Server.Version}}']);
+  if (version.code !== 0) throw new DockerError('Docker is installed but not running: start Docker Desktop / the docker service.', version.stderr);
+  const osType = (await dockerOk(['info', '--format', '{{.OSType}}'])).trim();
+  if (osType !== 'linux') {
+    throw new DockerError(`Docker runs ${osType} containers; the folder agent needs Linux containers. ` + 'Switch Docker Desktop to Linux containers.');
+  }
+}
+
+/**
+ * The absolute path Docker mounts for [folder]. Refuses network and `\\wsl$` paths, which Docker Desktop
+ * cannot bind; [note] is set for a Windows drive seen from WSL, which works but is slow.
+ */
+function hostFolder(folder, platform = process.platform, resolve = p => _fs.default.realpathSync.native(p)) {
+  const raw = folder.trim();
+  if (/^[\\/]{2}(wsl\$|wsl\.localhost)([\\/]|$)/i.test(raw)) throw new Error(`${raw} is inside WSL: run grok connect-folder inside WSL for WSL paths`);
+  if (/^[\\/]{2}[^\\/]/.test(raw)) throw new Error(`${raw} is a network path; Docker cannot mount it. Connect a local folder instead`);
+  const p = platform === 'win32' ? _path.default.win32 : _path.default.posix;
+  const resolved = resolve(p.resolve(raw));
+  if (/^[\\/]{2}/.test(resolved)) throw new Error(`${raw} resolves to the network path ${resolved}; Docker cannot mount it`);
+  const note = platform === 'linux' && /^\/mnt\/[a-z](\/|$)/i.test(resolved) ? 'Windows drives seen from WSL (/mnt/...) are slow to read through Docker; consider a folder in the WSL filesystem' : undefined;
+  return {
+    path: resolved,
+    note
+  };
+}
+
+/**
+ * The folder the server derives the share id from: `/` separators, no trailing slash, lower case where paths are
+ * case-insensitive, and a Windows drive seen from WSL (`/mnt/c/x`) the same as on Windows (`c:/x`).
+ */
+function canonicalFolder(abs, platform = process.platform) {
+  let f = abs.replace(/\\/g, '/');
+  const wslDrive = platform === 'linux' ? /^\/mnt\/([a-z])(\/.*)?$/i.exec(f) : null;
+  if (wslDrive) f = `${wslDrive[1]}:${wslDrive[2] ?? '/'}`;
+  while (f.length > 1 && f.endsWith('/') && !f.endsWith(':/')) f = f.slice(0, -1);
+  return platform === 'win32' || platform === 'darwin' || wslDrive ? f.toLowerCase() : f;
+}
+
+/** `Mac.local` and `Mac` are one machine: DHCP and macOS flip between them. */
+function canonicalHost(hostname = _os.default.hostname()) {
+  return hostname.trim().toLowerCase().split('.')[0];
+}
+
+/** `localhost` in a container is the container itself; Docker reaches the host as `host.docker.internal`. */
+function containerUrl(url) {
+  const u = new URL(url);
+  if (!['localhost', '127.0.0.1', '[::1]'].includes(u.hostname)) return {
+    url,
+    rewritten: false
+  };
+  u.hostname = DOCKER_HOST;
+  return {
+    url: u.href.replace(/\/$/, ''),
+    rewritten: true
+  };
+}
+
+/** The same login shares a folder under the same id on every server, so the name hashes the server in. */
+function containerName(apiRoot, connectionId, foreground = false) {
+  const hash = _crypto.default.createHash('sha256').update(`${apiRoot}|${connectionId}`).digest('hex').slice(0, 12);
+  return `grok-folder-${hash}${foreground ? '-fg' : ''}`;
+}
+
+/** A `--mount` field: the value is CSV, so a path with a comma or a quote must be quoted. */
+function mountField(field) {
+  return /[",]/.test(field) ? `"${field.replace(/"/g, '""')}"` : field;
+}
+/** Whether [c] runs [spec] as is: the mount, the mode, the image and the user are fixed at `docker run`. */
+function sameSettings(c, spec) {
+  const l = c.labels;
+  return c.image === spec.image && c.imageId === (spec.imageId ?? c.imageId) && l[`${LABEL}.folder`] === spec.folder && l[`${LABEL}.readonly`] === `${spec.readOnly}` && (l[`${LABEL}.user`] ?? '') === (runUser(spec) ?? '');
+}
+function runUser(spec) {
+  return (spec.platform ?? process.platform) === 'linux' ? spec.user : undefined;
+}
+function runArgs(spec) {
+  const platform = spec.platform ?? process.platform;
+  const server = containerUrl(spec.apiRoot);
+  const labels = {
+    [LABEL]: '1',
+    [`${LABEL}.connection`]: spec.connectionId,
+    [`${LABEL}.alias`]: spec.alias,
+    [`${LABEL}.url`]: spec.apiRoot,
+    [`${LABEL}.name`]: spec.name,
+    [`${LABEL}.folder`]: spec.folder,
+    [`${LABEL}.readonly`]: `${spec.readOnly}`,
+    [`${LABEL}.user`]: runUser(spec) ?? '',
+    [`${LABEL}.key`]: spec.keyId ?? ''
+  };
+  const args = ['run', '--name', containerName(spec.apiRoot, spec.connectionId, spec.foreground)];
+  args.push(...(spec.foreground ? ['--rm'] : ['-d', '--restart', 'unless-stopped']));
+  for (const [k, v] of Object.entries(labels)) args.push('--label', `${k}=${v}`);
+  args.push('--mount', ['type=bind', mountField(`source=${spec.folder}`), `target=${SHARE_ROOT}`, ...(spec.readOnly ? ['readonly'] : [])].join(','));
+  args.push('--env-file', spec.envFile);
+  args.push('--cap-drop', 'ALL', '--security-opt', 'no-new-privileges', '--memory', '256m', '-e', 'DEBUG_MODE=false');
+  if (runUser(spec)) args.push('--user', runUser(spec));
+  // Docker Desktop resolves host.docker.internal by itself; Docker Engine on Linux (and WSL) needs it mapped to the host.
+  if (platform === 'linux' && server.rewritten) args.push('--add-host', `${DOCKER_HOST}:host-gateway`);
+  args.push(spec.image, '--tunnel', '--server', server.url, '--root', SHARE_ROOT, '--connection', spec.connectionId, '--host-label', spec.hostLabel, '--folder-label', spec.folder);
+  if (spec.readOnly) args.push('--read-only');
+  return args;
+}
+function linuxUser(platform = process.platform) {
+  return platform === 'linux' && process.getuid && process.getgid ? `${process.getuid()}:${process.getgid()}` : undefined;
+}
+
+/** Share containers with all of [labels], stopped ones included. */
+async function ps(labels = {}) {
+  const filters = [`label=${LABEL}=1`, ...Object.entries(labels).map(([k, v]) => `label=${k}=${v}`)];
+  const ids = (await dockerOk(['ps', '-a', '-q', ...filters.flatMap(f => ['--filter', f])])).split(/\s+/).filter(s => s);
+  if (!ids.length) return [];
+  // `docker ps --format` joins labels with commas, which a folder name can contain; inspect keeps them apart.
+  return JSON.parse(await dockerOk(['inspect', ...ids])).map(c => ({
+    id: c.Id,
+    name: String(c.Name ?? '').replace(/^\//, ''),
+    state: c.State?.Status ?? '',
+    image: c.Config?.Image ?? '',
+    imageId: c.Image ?? '',
+    labels: c.Config?.Labels ?? {},
+    mounts: c.Mounts ?? []
+  }));
+}
+
+/**
+ * The local id of [image]. A tag is pulled every time, since the server moves it, and Docker's copy is used when the
+ * pull fails ([pullError]: offline, a local build); a digest is pulled only when missing.
+ */
+async function ensureImage(image) {
+  const local = async () => {
+    const r = await docker(['image', 'inspect', '--format', '{{.Id}}', image]);
+    return r.code === 0 ? r.stdout.trim() : '';
+  };
+  const pinned = image.includes('@sha256:') ? await local() : '';
+  if (pinned) return {
+    id: pinned
+  };
+  const pull = await docker(['pull', image]);
+  const id = await local();
+  if (!id) throw new DockerError(pull.stderr.trim() || `docker pull ${image} failed with exit code ${pull.code}`, pull.stderr);
+  return pull.code === 0 ? {
+    id
+  } : {
+    id,
+    pullError: pull.stderr.trim() || `exit code ${pull.code}`
+  };
+}
+
+/** Whether [container] is running, waiting up to [seconds] for it to get there. */
+async function isRunning(container, seconds = 0) {
+  for (let i = 0;; i++) {
+    const r = await docker(['inspect', '--format', '{{.State.Running}}', container]);
+    if (r.code === 0 && r.stdout.trim() === 'true') return true;
+    if (i >= seconds * 4) return false;
+    await new Promise(resolve => setTimeout(resolve, 250));
+  }
+}
+const start = container => dockerOk(['start', container]);
+exports.start = start;
+const stop = container => dockerOk(['stop', container]);
+exports.stop = stop;
+const rm = container => dockerOk(['rm', '-f', container]);
+exports.rm = rm;
+const logs = (container, follow = false) => docker(['logs', ...(follow ? ['-f'] : []), container], true);
+exports.logs = logs;

package/bin/utils/connect-folder/docker.ts

@@ -0,0 +1,259 @@
+import childProcess from 'child_process';
+import crypto from 'crypto';
+import fs from 'fs';
+import os from 'os';
+import path from 'path';
+
+/** The `docker` CLI for `grok connect-folder`: always an argument vector, never a shell, so no quoting or MSYS path mangling. */
+
+export const LABEL = 'ai.datagrok.folder';
+export const SHARE_ROOT = '/share';
+/** The only image run with the user's folder mounted unless `--image` names another. */
+export const AGENT_REPOSITORY = 'datagrok/proxy_storage';
+
+/** Whether [image] is [AGENT_REPOSITORY] on Docker Hub, by tag or digest: a server cannot name any other. */
+export function isAgentImage(image: string): boolean {
+  return /^((docker\.io|index\.docker\.io)\/)?datagrok\/proxy_storage(:\w[\w.-]{0,127})?(@sha256:[0-9a-f]{64})?$/.test(image);
+}
+
+const DOCKER_HOST = 'host.docker.internal';
+
+export class DockerError extends Error {
+  constructor(message: string, public readonly stderr: string = '') {
+    super(message);
+    this.name = 'DockerError';
+  }
+}
+
+/** Where `docker` is started; the tests replace it, since a module mock cannot reach the transpiled require. */
+export const proc = {spawn: childProcess.spawn};
+
+export interface DockerResult { code: number, stdout: string, stderr: string }
+
+export interface ShareContainer {
+  id: string,
+  name: string,
+  state: string,
+  image: string,
+  imageId: string,
+  labels: Record<string, string>,
+  mounts: {Source: string, Destination: string, RW: boolean}[],
+}
+
+/** Runs `docker <args>`; [inherit] attaches the terminal (logs -f, a foreground share). */
+export function docker(args: string[], inherit: boolean = false): Promise<DockerResult> {
+  return new Promise((resolve, reject) => {
+    const child = proc.spawn('docker', args, {stdio: inherit ? 'inherit' : ['ignore', 'pipe', 'pipe'], windowsHide: true});
+    let stdout = '';
+    let stderr = '';
+    child.stdout?.on('data', (d) => stdout += d);
+    child.stderr?.on('data', (d) => stderr += d);
+    child.on('error', (e: any) => reject(e?.code === 'ENOENT'
+      ? new DockerError('grok connect-folder runs the folder agent in Docker, which is not installed. ' +
+        'Install Docker Desktop (Windows/macOS) or Docker Engine (Linux).')
+      : e));
+    child.on('close', (code) => resolve({code: code ?? 1, stdout, stderr}));
+  });
+}
+
+export async function dockerOk(args: string[]): Promise<string> {
+  const r = await docker(args);
+  if (r.code !== 0)
+    throw new DockerError(r.stderr.trim() || `docker ${args[0]} failed with exit code ${r.code}`, r.stderr);
+  return r.stdout;
+}
+
+export async function dockerAvailable(): Promise<void> {
+  const version = await docker(['version', '--format', '{{.Server.Version}}']);
+  if (version.code !== 0)
+    throw new DockerError('Docker is installed but not running: start Docker Desktop / the docker service.', version.stderr);
+  const osType = (await dockerOk(['info', '--format', '{{.OSType}}'])).trim();
+  if (osType !== 'linux') {
+    throw new DockerError(`Docker runs ${osType} containers; the folder agent needs Linux containers. ` +
+      'Switch Docker Desktop to Linux containers.');
+  }
+}
+
+/**
+ * The absolute path Docker mounts for [folder]. Refuses network and `\\wsl$` paths, which Docker Desktop
+ * cannot bind; [note] is set for a Windows drive seen from WSL, which works but is slow.
+ */
+export function hostFolder(folder: string, platform: string = process.platform,
+  resolve: (p: string) => string = (p) => fs.realpathSync.native(p)): {path: string, note?: string} {
+  const raw = folder.trim();
+  if (/^[\\/]{2}(wsl\$|wsl\.localhost)([\\/]|$)/i.test(raw))
+    throw new Error(`${raw} is inside WSL: run grok connect-folder inside WSL for WSL paths`);
+  if (/^[\\/]{2}[^\\/]/.test(raw))
+    throw new Error(`${raw} is a network path; Docker cannot mount it. Connect a local folder instead`);
+  const p = platform === 'win32' ? path.win32 : path.posix;
+  const resolved = resolve(p.resolve(raw));
+  if (/^[\\/]{2}/.test(resolved))
+    throw new Error(`${raw} resolves to the network path ${resolved}; Docker cannot mount it`);
+  const note = platform === 'linux' && /^\/mnt\/[a-z](\/|$)/i.test(resolved)
+    ? 'Windows drives seen from WSL (/mnt/...) are slow to read through Docker; consider a folder in the WSL filesystem'
+    : undefined;
+  return {path: resolved, note};
+}
+
+/**
+ * The folder the server derives the share id from: `/` separators, no trailing slash, lower case where paths are
+ * case-insensitive, and a Windows drive seen from WSL (`/mnt/c/x`) the same as on Windows (`c:/x`).
+ */
+export function canonicalFolder(abs: string, platform: string = process.platform): string {
+  let f = abs.replace(/\\/g, '/');
+  const wslDrive = platform === 'linux' ? /^\/mnt\/([a-z])(\/.*)?$/i.exec(f) : null;
+  if (wslDrive)
+    f = `${wslDrive[1]}:${wslDrive[2] ?? '/'}`;
+  while (f.length > 1 && f.endsWith('/') && !f.endsWith(':/'))
+    f = f.slice(0, -1);
+  return platform === 'win32' || platform === 'darwin' || wslDrive ? f.toLowerCase() : f;
+}
+
+/** `Mac.local` and `Mac` are one machine: DHCP and macOS flip between them. */
+export function canonicalHost(hostname: string = os.hostname()): string {
+  return hostname.trim().toLowerCase().split('.')[0];
+}
+
+/** `localhost` in a container is the container itself; Docker reaches the host as `host.docker.internal`. */
+export function containerUrl(url: string): {url: string, rewritten: boolean} {
+  const u = new URL(url);
+  if (!['localhost', '127.0.0.1', '[::1]'].includes(u.hostname))
+    return {url, rewritten: false};
+  u.hostname = DOCKER_HOST;
+  return {url: u.href.replace(/\/$/, ''), rewritten: true};
+}
+
+/** The same login shares a folder under the same id on every server, so the name hashes the server in. */
+export function containerName(apiRoot: string, connectionId: string, foreground: boolean = false): string {
+  const hash = crypto.createHash('sha256').update(`${apiRoot}|${connectionId}`).digest('hex').slice(0, 12);
+  return `grok-folder-${hash}${foreground ? '-fg' : ''}`;
+}
+
+/** A `--mount` field: the value is CSV, so a path with a comma or a quote must be quoted. */
+function mountField(field: string): string {
+  return /[",]/.test(field) ? `"${field.replace(/"/g, '""')}"` : field;
+}
+
+export interface RunSpec {
+  image: string,
+  connectionId: string,
+  alias: string,
+  apiRoot: string,
+  name: string,
+  folder: string,
+  hostLabel: string,
+  readOnly: boolean,
+  envFile: string,
+  foreground?: boolean,
+  platform?: string,
+  /** `uid:gid` on Linux, so files the agent writes belong to the user. */
+  user?: string,
+  /** The server id of the share's own key, whose private half is in [envFile]. */
+  keyId?: string,
+  /** The local id [image] resolves to, which a tag the server moved changes. */
+  imageId?: string,
+}
+
+/** Whether [c] runs [spec] as is: the mount, the mode, the image and the user are fixed at `docker run`. */
+export function sameSettings(c: ShareContainer, spec: RunSpec): boolean {
+  const l = c.labels;
+  return c.image === spec.image && c.imageId === (spec.imageId ?? c.imageId) && l[`${LABEL}.folder`] === spec.folder && l[`${LABEL}.readonly`] === `${spec.readOnly}` &&
+    (l[`${LABEL}.user`] ?? '') === (runUser(spec) ?? '');
+}
+
+function runUser(spec: RunSpec): string | undefined {
+  return (spec.platform ?? process.platform) === 'linux' ? spec.user : undefined;
+}
+
+export function runArgs(spec: RunSpec): string[] {
+  const platform = spec.platform ?? process.platform;
+  const server = containerUrl(spec.apiRoot);
+  const labels: Record<string, string> = {
+    [LABEL]: '1',
+    [`${LABEL}.connection`]: spec.connectionId,
+    [`${LABEL}.alias`]: spec.alias,
+    [`${LABEL}.url`]: spec.apiRoot,
+    [`${LABEL}.name`]: spec.name,
+    [`${LABEL}.folder`]: spec.folder,
+    [`${LABEL}.readonly`]: `${spec.readOnly}`,
+    [`${LABEL}.user`]: runUser(spec) ?? '',
+    [`${LABEL}.key`]: spec.keyId ?? '',
+  };
+  const args = ['run', '--name', containerName(spec.apiRoot, spec.connectionId, spec.foreground)];
+  args.push(...(spec.foreground ? ['--rm'] : ['-d', '--restart', 'unless-stopped']));
+  for (const [k, v] of Object.entries(labels))
+    args.push('--label', `${k}=${v}`);
+  args.push('--mount', ['type=bind', mountField(`source=${spec.folder}`), `target=${SHARE_ROOT}`,
+    ...(spec.readOnly ? ['readonly'] : [])].join(','));
+  args.push('--env-file', spec.envFile);
+  args.push('--cap-drop', 'ALL', '--security-opt', 'no-new-privileges', '--memory', '256m', '-e', 'DEBUG_MODE=false');
+  if (runUser(spec))
+    args.push('--user', runUser(spec)!);
+  // Docker Desktop resolves host.docker.internal by itself; Docker Engine on Linux (and WSL) needs it mapped to the host.
+  if (platform === 'linux' && server.rewritten)
+    args.push('--add-host', `${DOCKER_HOST}:host-gateway`);
+  args.push(spec.image, '--tunnel', '--server', server.url, '--root', SHARE_ROOT, '--connection', spec.connectionId,
+    '--host-label', spec.hostLabel, '--folder-label', spec.folder);
+  if (spec.readOnly)
+    args.push('--read-only');
+  return args;
+}
+
+export function linuxUser(platform: string = process.platform): string | undefined {
+  return platform === 'linux' && process.getuid && process.getgid ? `${process.getuid()}:${process.getgid()}` : undefined;
+}
+
+/** Share containers with all of [labels], stopped ones included. */
+export async function ps(labels: Record<string, string> = {}): Promise<ShareContainer[]> {
+  const filters = [`label=${LABEL}=1`, ...Object.entries(labels).map(([k, v]) => `label=${k}=${v}`)];
+  const ids = (await dockerOk(['ps', '-a', '-q', ...filters.flatMap((f) => ['--filter', f])])).split(/\s+/).filter((s) => s);
+  if (!ids.length)
+    return [];
+  // `docker ps --format` joins labels with commas, which a folder name can contain; inspect keeps them apart.
+  return (JSON.parse(await dockerOk(['inspect', ...ids])) as any[]).map((c) => ({
+    id: c.Id,
+    name: String(c.Name ?? '').replace(/^\//, ''),
+    state: c.State?.Status ?? '',
+    image: c.Config?.Image ?? '',
+    imageId: c.Image ?? '',
+    labels: c.Config?.Labels ?? {},
+    mounts: c.Mounts ?? [],
+  }));
+}
+
+/**
+ * The local id of [image]. A tag is pulled every time, since the server moves it, and Docker's copy is used when the
+ * pull fails ([pullError]: offline, a local build); a digest is pulled only when missing.
+ */
+export async function ensureImage(image: string): Promise<{id: string, pullError?: string}> {
+  const local = async () => {
+    const r = await docker(['image', 'inspect', '--format', '{{.Id}}', image]);
+    return r.code === 0 ? r.stdout.trim() : '';
+  };
+  const pinned = image.includes('@sha256:') ? await local() : '';
+  if (pinned)
+    return {id: pinned};
+  const pull = await docker(['pull', image]);
+  const id = await local();
+  if (!id)
+    throw new DockerError(pull.stderr.trim() || `docker pull ${image} failed with exit code ${pull.code}`, pull.stderr);
+  return pull.code === 0 ? {id} : {id, pullError: pull.stderr.trim() || `exit code ${pull.code}`};
+}
+
+/** Whether [container] is running, waiting up to [seconds] for it to get there. */
+export async function isRunning(container: string, seconds: number = 0): Promise<boolean> {
+  for (let i = 0; ; i++) {
+    const r = await docker(['inspect', '--format', '{{.State.Running}}', container]);
+    if (r.code === 0 && r.stdout.trim() === 'true')
+      return true;
+    if (i >= seconds * 4)
+      return false;
+    await new Promise((resolve) => setTimeout(resolve, 250));
+  }
+}
+
+export const start = (container: string) => dockerOk(['start', container]);
+export const stop = (container: string) => dockerOk(['stop', container]);
+export const rm = (container: string) => dockerOk(['rm', '-f', container]);
+export const logs = (container: string, follow: boolean = false) =>
+  docker(['logs', ...(follow ? ['-f'] : []), container], true);

package/bin/utils/ent-helpers.js

@@ -19,7 +19,7 @@ The app is the u2 defaults alone — list, search, filters, entity page with chi
 and history, editing under one session, permissions and deep links. src/app.spec.json
 is the same app as a designer-editable spec. To go further, run \`grok api --ui\` for
 typed handles (get<Schema>Db()) and declare on a table: actions.add(...),
-validators.add(column, ...), renderer, and app({app: <a DomainApp subclass>}) for
+validators.add(column, ...), renderer, and app({ribbon, shortcuts, menu, setup}) for
 presets and shortcuts — the Grit package is the reference.
 Read more at https://datagrok.ai/help/develop/how-to/build-an-app`;
 const connection = connectionName => `

package/bin/utils/node-dapi.js

@@ -64,6 +64,17 @@ async function fetchOrRetry(url, opts, retriable, timeoutMs = setting('TIMEOUT',
 class NodeApiClient {
   /** Set by `createClient` when the run asked for an admin session, so a re-login restores it. */
   adminMode = false;
+  /**
+   * `--log`: one id for every request of the command, wherever the server runs it, and the logger settings
+   * its records are kept at until `GET /log/requests/<logId>` collects them.
+   */
+
+  get logHeaders() {
+    const headers = {};
+    if (this.logId) headers['datlas-log-id'] = this.logId;
+    if (this.logSpec) headers['datlas-log'] = this.logSpec;
+    return headers;
+  }
   constructor(baseUrl, token, devKey, privateKey) {
     this.baseUrl = baseUrl;
     this.token = token;
@@ -120,6 +131,7 @@ class NodeApiClient {
       headers: {
         'Authorization': this.token,
         'Content-Type': 'application/json',
+        ...this.logHeaders,
         ...headers
       }
     };
@@ -165,7 +177,8 @@ class NodeApiClient {
       method: 'POST',
       headers: {
         'Authorization': this.token,
-        'Content-Type': contentType
+        'Content-Type': contentType,
+        ...this.logHeaders
       },
       body: bytes
     }, false, BYTES_TIMEOUT_MS);
@@ -180,7 +193,8 @@ class NodeApiClient {
       method: 'POST',
       headers: {
         'Authorization': this.token,
-        'Content-Type': 'application/json'
+        'Content-Type': 'application/json',
+        ...this.logHeaders
       },
       body: JSON.stringify(body),
       signal: AbortSignal.timeout(BYTES_TIMEOUT_MS)
@@ -193,7 +207,8 @@ class NodeApiClient {
   async getBytes(path) {
     const res = await fetchOrRetry(`${this.baseUrl}${path}`, {
       headers: {
-        'Authorization': this.token
+        'Authorization': this.token,
+        ...this.logHeaders
       }
     }, true, BYTES_TIMEOUT_MS);
     if (!res.ok) await throwHttpError(res);
@@ -641,9 +656,11 @@ class NodeFuncsDataSource extends NodeHttpDataSource {
   constructor(client) {
     super(client, 'functions', '/log/funcs');
   }
-  async run(name, params) {
+
+  /** [debug] runs it as a debug call, as the query debugger does; its log is kept with the command's `--log`. */
+  async run(name, params, debug = false) {
     const normalizedName = name.replace(':', '.');
-    return this.client.post(`/public/v1/functions/${encodeURIComponent(normalizedName)}/call`, params ?? {});
+    return this.client.post(`/public/v1/functions/${encodeURIComponent(normalizedName)}/call${debug ? '?debug=true' : ''}`, params ?? {});
   }
 
   /** The public API has no DELETE for functions; scripts and queries go to their own routers. */
@@ -1170,9 +1187,14 @@ class NodeDapi {
   internal(route) {
     return new InternalDataSource(this.client, route);
   }
+
+  /** `/info/server`; [extended] adds the in-flight requests per endpoint (admins only). */
+  async serverStatus(extended = false) {
+    const raw = await this.client.get(extended ? '/info/server?extended=true' : '/info/server');
+    return typeof raw === 'string' ? JSON.parse(raw) : raw;
+  }
   async serverInfo() {
-    const raw = await this.client.get('/info/server');
-    const info = typeof raw === 'string' ? JSON.parse(raw) : raw;
+    const info = await this.serverStatus();
     return {
       version: info?.Version ?? '',
       commit: info?.Commit

package/bin/utils/server-client.js

@@ -4,6 +4,7 @@ Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.createClient = createClient;
+exports.logSpecOf = logSpecOf;
 var _nodeDapi = require("./node-dapi");
 var _keypair = require("./keypair");
 /**
@@ -13,11 +14,17 @@ var _keypair = require("./keypair");
  * flag into the token, and a dev key mints a fresh session per invocation, so it cannot outlive
  * the command or reach another session.
  */
-async function createClient(hostArg, admin = false) {
+/** `--log` alone is `verbose`. */
+function logSpecOf(argv) {
+  return argv.log === undefined || argv.log === false ? undefined : argv.log === true ? 'verbose' : String(argv.log);
+}
+async function createClient(hostArg, admin = false, log) {
   // Resolved from the alias the caller named, not from its URL: two aliases can point at the
   // same server, and only this side knows which one was asked for.
   const cred = (0, _keypair.getServerCredentials)(hostArg ?? '');
   const client = await _nodeDapi.NodeApiClient.login(cred.url, cred.key ?? '', cred.privateKey);
+  client.logId = log?.logId;
+  client.logSpec = log?.spec;
   if (!admin) return client;
   const token = (await client.post('/users/sessions/current/admin'))?.token;
   if (!token) throw new Error(`${cred.url} refused an admin session — the account behind this key cannot start one`);