datagrok-tools 6.5.6 → 6.5.8

package/bin/utils/node-dapi.js

@@ -3,8 +3,13 @@
 Object.defineProperty(exports, "__esModule", {
   value: true
 });
-exports.NodeUsersDataSource = exports.NodeTablesDataSource = exports.NodeSharesDataSource = exports.NodeHttpDataSource = exports.NodeGroupsDataSource = exports.NodeFuncsDataSource = exports.NodeFilesDataSource = exports.NodeDapi = exports.NodeConnectionsDataSource = exports.NodeApiClient = void 0;
+exports.NodeUsersDataSource = exports.NodeTablesDataSource = exports.NodeSharesDataSource = exports.NodePackagesDataSource = exports.NodeHttpDataSource = exports.NodeGroupsDataSource = exports.NodeFuncsDataSource = exports.NodeFilesDataSource = exports.NodeDomainsDataSource = exports.NodeDapi = exports.NodeConnectionsDataSource = exports.NodeApiClient = exports.InternalDataSource = void 0;
+exports.apiPath = apiPath;
+exports.buildQuery = buildQuery;
 exports.ensureBodyId = ensureBodyId;
+exports.mapPositionalParams = mapPositionalParams;
+exports.parseDomainAddress = parseDomainAddress;
+exports.throwIfApiError = throwIfApiError;
 var _crypto = require("crypto");
 /// Docs: [Grok Dapi](/docs/plans/grok-dapi/)
 
@@ -12,20 +17,80 @@ function ensureBodyId(body) {
   if (body && typeof body === 'object' && !body.id) body.id = (0, _crypto.randomUUID)();
   return body;
 }
+const setting = (name, fallback) => {
+  const value = Number(process.env[`GROK_HTTP_${name}`]);
+  return Number.isFinite(value) && value >= 0 ? value : fallback;
+};
+const BYTES_TIMEOUT_MS = setting('BYTES_TIMEOUT', 600000);
+
+/** Load shedding, not a verdict on the request: the same call succeeds once the queue drains. */
+const RETRIABLE_STATUS = new Set([429, 502, 503, 504]);
+
+/**
+ * undici keeps the socket checked out until the body is read, so a response that is answered but
+ * never consumed leaks a connection. Enough of them exhaust the pool, and the requests that
+ * follow queue forever — before the request starts, so no deadline ever fires.
+ */
+const discard = res => res.body?.cancel() ?? Promise.resolve();
+
+/**
+ * Without a deadline one unresponsive entity stalls a whole pull — `GET /projects/{id}` on a
+ * space holding tens of thousands of children never answers. A request that hung or dropped is
+ * retried, since a deadline is as often a server busy with this very pull as a dead one; a reply
+ * the server actually sent is not. The deadline covers the body too, so a transfer that is slow
+ * by nature rather than stuck (`.d42` table data) asks for a longer one.
+ */
+async function fetchOrRetry(url, opts, retriable, timeoutMs = setting('TIMEOUT', 60000)) {
+  const retries = setting('RETRIES', 5);
+  for (let attempt = 0;; attempt++) {
+    const last = !retriable || attempt >= retries;
+    try {
+      const res = await fetch(url, {
+        ...opts,
+        signal: AbortSignal.timeout(timeoutMs)
+      });
+      if (last || !RETRIABLE_STATUS.has(res.status)) return res;
+      await res.body?.cancel();
+    } catch (err) {
+      if (last) throw new Error(`${opts.method ?? 'GET'} ${url}: ` + (err?.name === 'TimeoutError' ? `no answer in ${timeoutMs}ms` : err?.message ?? err));
+    }
+    // Capped: uncapped doubling turns a long retry budget into minutes asleep on one request.
+    const backoff = Math.min(setting('BACKOFF', 1000) * Math.pow(2, attempt), setting('BACKOFF_MAX', 15000));
+    await new Promise(resolve => setTimeout(resolve, backoff));
+  }
+}
 class NodeApiClient {
-  constructor(baseUrl, token) {
+  /** Set by `createClient` when the run asked for an admin session, so a re-login restores it. */
+  adminMode = false;
+  constructor(baseUrl, token, devKey) {
     this.baseUrl = baseUrl;
     this.token = token;
+    this.devKey = devKey;
   }
   static async login(baseUrl, devKey) {
     const res = await fetch(`${baseUrl}/users/login/dev/${devKey}`, {
       method: 'POST'
     });
-    const json = await res.json();
-    if (!json.token) throw new Error('Login failed. Check your developer key.');
-    return new NodeApiClient(baseUrl, json.token);
+    const json = (res.headers.get('content-type') ?? '').includes('application/json') ? await res.json() : null;
+    if (!json) throw new Error(`Login failed at ${baseUrl} (HTTP ${res.status}): not a Datagrok API URL — it should end with /api`);
+    if (!json.token) throw new Error(`Login failed at ${baseUrl}: ${json.message ?? 'check your developer key'}`);
+    return new NodeApiClient(baseUrl, json.token, devKey);
   }
-  async request(method, path, body, headers) {
+
+  /**
+   * A stand serving several isolates can reject a session one of them does not know, and an
+   * hour-long walk has no way to ask the operator to log in again. The developer key is good
+   * for a new session, so one is taken rather than losing the run.
+   */
+  async reauthenticate() {
+    if (!this.devKey) return false;
+    const fresh = await NodeApiClient.login(this.baseUrl, this.devKey).catch(() => null);
+    if (!fresh) return false;
+    this.token = fresh.token;
+    if (this.adminMode) this.token = (await fresh.post('/users/sessions/current/admin'))?.token ?? this.token;
+    return true;
+  }
+  async request(method, path, body, headers, reauthed = false, timeoutMs) {
     const url = `${this.baseUrl}${path}`;
     const opts = {
       method,
@@ -36,38 +101,32 @@ class NodeApiClient {
       }
     };
     if (body !== undefined) opts.body = JSON.stringify(body);
-    const res = await fetch(url, opts);
-    if (!res.ok) {
-      // Read as text first to avoid "Body has already been read" when JSON.parse fails
-      const rawText = await res.text();
-      let errBody;
-      try {
-        errBody = JSON.parse(rawText);
-      } catch {
-        errBody = {
-          error: rawText || `HTTP ${res.status}`
-        };
-      }
-      const err = {
-        error: errBody?.message ?? errBody?.error ?? `HTTP ${res.status}`,
-        source: errBody?.source ?? 'Server',
-        errorCode: errBody?.errorCode ?? res.status,
-        stackTrace: errBody?.stackTrace
-      };
-      throw Object.assign(new Error(err.error), {
-        apiError: err
+    const res = await fetchOrRetry(url, opts, method === 'GET', timeoutMs);
+    if (res.status === 401 && !reauthed) {
+      const refusal = await res.text();
+      if (await this.reauthenticate()) return this.request(method, path, body, headers, true, timeoutMs);
+      const error = refusal || `HTTP ${res.status}`;
+      throw Object.assign(new Error(error), {
+        apiError: {
+          error,
+          source: 'Server',
+          errorCode: 401
+        }
       });
     }
-    if (res.status === 204 || res.headers.get('content-length') === '0') return null;
+    if (!res.ok) await throwHttpError(res);
+    if (res.status === 204 || res.headers.get('content-length') === '0') {
+      await discard(res);
+      return null;
+    }
     const ct = res.headers.get('content-type') ?? '';
-    if (ct.includes('application/json')) return res.json();
-    return res.text();
+    return throwIfApiError(ct.includes('application/json') ? await res.json() : await res.text());
   }
   get(path) {
     return this.request('GET', path);
   }
-  post(path, body) {
-    return this.request('POST', path, body);
+  post(path, body, timeoutMs) {
+    return this.request('POST', path, body, undefined, false, timeoutMs);
   }
   del(path) {
     return this.request('DELETE', path);
@@ -79,53 +138,129 @@ class NodeApiClient {
    * similar) when the server demands a specific content type.
    */
   async putBytes(path, bytes, contentType = 'application/octet-stream') {
-    const res = await fetch(`${this.baseUrl}${path}`, {
+    const res = await fetchOrRetry(`${this.baseUrl}${path}`, {
       method: 'POST',
       headers: {
         'Authorization': this.token,
         'Content-Type': contentType
       },
       body: bytes
+    }, false, BYTES_TIMEOUT_MS);
+    if (!res.ok) await throwHttpError(res);
+    const ct = res.headers.get('content-type') ?? '';
+    return throwIfApiError(ct.includes('application/json') ? await res.json() : await res.text());
+  }
+
+  /** POST a JSON body and read the response as raw bytes (d42 query results). */
+  async postForBytes(path, body) {
+    const res = await fetch(`${this.baseUrl}${path}`, {
+      method: 'POST',
+      headers: {
+        'Authorization': this.token,
+        'Content-Type': 'application/json'
+      },
+      body: JSON.stringify(body),
+      signal: AbortSignal.timeout(BYTES_TIMEOUT_MS)
     });
-    if (!res.ok) {
-      const rawText = await res.text();
-      let errBody;
-      try {
-        errBody = JSON.parse(rawText);
-      } catch {
-        errBody = {
-          error: rawText || `HTTP ${res.status}`
-        };
+    if (!res.ok) await throwHttpError(res);
+    return Buffer.from(await res.arrayBuffer());
+  }
+
+  /** GET raw bytes — d42 table data, file content, model blobs. */
+  async getBytes(path) {
+    const res = await fetchOrRetry(`${this.baseUrl}${path}`, {
+      headers: {
+        'Authorization': this.token
       }
-      const err = {
-        error: errBody?.message ?? errBody?.error ?? `HTTP ${res.status}`,
-        source: errBody?.source ?? 'Server',
-        errorCode: errBody?.errorCode ?? res.status,
-        stackTrace: errBody?.stackTrace
-      };
-      throw Object.assign(new Error(err.error), {
-        apiError: err
-      });
-    }
-    const ct = res.headers.get('content-type') ?? '';
-    if (ct.includes('application/json')) return res.json();
-    return res.text();
+    }, true, BYTES_TIMEOUT_MS);
+    if (!res.ok) await throwHttpError(res);
+    const bytes = Buffer.from(await res.arrayBuffer());
+    // A table whose data file is missing answers 200 — as text/plain — with an ApiError body.
+    // Writing that into the bundle ships an error message as a table and only fails on the far
+    // stand at push time; d42 never starts with `{`, so an envelope here is an error, not data.
+    if (bytes[0] === 0x7B) throwIfApiError(bytes.toString('utf8'));
+    return bytes;
   }
 }
+
+// Read as text first to avoid "Body has already been read" when JSON.parse fails
 exports.NodeApiClient = NodeApiClient;
+async function throwHttpError(res) {
+  const rawText = await res.text();
+  let errBody;
+  // A gateway answers overload with an HTML page, where the status is the only real information.
+  const markup = rawText.trimStart().startsWith('<') || rawText.length > 200;
+  try {
+    errBody = JSON.parse(rawText);
+  } catch {
+    errBody = {
+      error: markup || !rawText ? `HTTP ${res.status} ${res.statusText}`.trim() : rawText
+    };
+  }
+  const err = {
+    error: errBody?.message ?? errBody?.error ?? `HTTP ${res.status}`,
+    source: errBody?.source ?? 'Server',
+    errorCode: errBody?.errorCode ?? res.status,
+    stackTrace: errBody?.stackTrace,
+    body: errBody
+  };
+  throw Object.assign(new Error(err.error), {
+    apiError: err
+  });
+}
+
+/**
+ * The server answers many failures with HTTP 200 and an `ApiError` body — as an object, or as
+ * a JSON string when the handler returned text. Every response passes through here so a
+ * missing entity, a rejected save or an unknown function throws instead of printing as data.
+ */
+function throwIfApiError(payload) {
+  const parsed = typeof payload === 'string' ? tryParseJson(payload) : payload;
+  if (parsed?.['#type'] === 'ApiError') {
+    const err = {
+      error: parsed.message ?? 'Request failed',
+      source: 'Server',
+      errorCode: parsed.errorCode,
+      stackTrace: parsed.stackTrace,
+      body: parsed
+    };
+    throw Object.assign(new Error(err.error), {
+      apiError: err
+    });
+  }
+  return payload;
+}
+
+/**
+ * `grok s raw` paths are API-relative; a leading `/api` is accepted and dropped, so the same
+ * path works against an nginx-fronted `.../api` base and a bare Datlas root.
+ */
+function apiPath(path) {
+  const p = path.startsWith('/') ? path : `/${path}`;
+  return p === '/api' ? '/' : p.startsWith('/api/') ? p.slice(4) : p;
+}
+
+/** Empty values are sent verbatim — an empty `namespace` selects the root namespace. */
 function buildQuery(params) {
-  const entries = Object.entries(params).filter(([, v]) => v !== undefined && v !== null && v !== '');
+  const entries = Object.entries(params).filter(([, v]) => v !== undefined && v !== null);
   if (!entries.length) return '';
   return '?' + entries.map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(String(v))}`).join('&');
 }
+
+/**
+ * A public-API entity. `find`/`save`/`delete` go to `/public/v1/<path>`; `list` and `count` go to
+ * [listRoute] when given — the internal router that pages (`limit`, 1-based `page`, `order`) and
+ * has `/count`, which the public list routes of users, groups, connections and functions do not.
+ */
 class NodeHttpDataSource {
   _filter = '';
   _limit = 50;
   _page = 0;
   _order = '';
-  constructor(client, path) {
+  constructor(client, path, listRoute) {
     this.client = client;
     this.path = path;
+    this.listRoute = listRoute;
   }
   filter(w) {
     this._filter = w;
@@ -135,22 +270,24 @@ class NodeHttpDataSource {
     this._limit = n;
     return this;
   }
+  /** Zero-based page of `by(n)` rows. */
   page(n) {
     this._page = n;
     return this;
   }
+  /** Smart-order syntax: `!field` is descending. */
   order(field, desc = false) {
-    this._order = desc ? `-${field}` : field;
+    this._order = desc ? `!${field}` : field;
     return this;
   }
   async list() {
     const q = buildQuery({
       text: this._filter || undefined,
       limit: this._limit,
-      page: this._page || undefined,
+      page: this._page + 1,
       order: this._order || undefined
     });
-    return this.client.get(`/public/v1/${this.path}${q}`);
+    return this.client.get(`${this.listRoute ?? `/public/v1/${this.path}`}${q}`);
   }
   async find(id) {
     return this.client.get(`/public/v1/${this.path}/${encodeURIComponent(id.replace(':', '.'))}`);
@@ -159,19 +296,80 @@ class NodeHttpDataSource {
     const q = buildQuery({
       text: this._filter || undefined
     });
-    const res = await this.client.get(`/public/v1/${this.path}/count${q}`);
-    return typeof res === 'number' ? res : res?.count ?? 0;
+    const res = await this.client.get(`${this.listRoute ?? `/public/v1/${this.path}`}/count${q}`);
+    return typeof res === 'number' ? res : Number(res?.count ?? res ?? 0);
   }
   async delete(idOrEntity) {
     const id = typeof idOrEntity === 'string' ? idOrEntity : idOrEntity?.id ?? '';
     await this.client.del(`/public/v1/${this.path}/${encodeURIComponent(id)}`);
   }
 }
+
+/**
+ * Generic client for the internal entity routers (`/projects`, `/scripts`,
+ * `/connectors/queries`, ...) the browser itself uses. Unlike `NodeDapi.raw` it goes
+ * through `client.request`, so a non-2xx response throws instead of returning the
+ * error body as data.
+ */
 exports.NodeHttpDataSource = NodeHttpDataSource;
+class InternalDataSource {
+  constructor(client, route) {
+    this.client = client;
+    this.route = route;
+  }
+
+  /**
+   * The internal routers answer a missing or rejected entity with HTTP 200 and an
+   * `ApiError` body, so success has to be decided from the payload, not the status.
+   * Only a router that says so is a 404 — everything else is a server-side failure and
+   * must not be mistaken for an absent entity.
+   */
+  async call(method, path, body) {
+    const res = await this.client.request(method, path, body);
+    return (typeof res === 'string' ? tryParseJson(res) : res) ?? res;
+  }
+  list(params = {}) {
+    return this.call('GET', `${this.route}${buildQuery(params)}`);
+  }
+  async count(params = {}) {
+    return Number(await this.call('GET', `${this.route}/count${buildQuery(params)}`));
+  }
+
+  /** Server paging is 1-based (`repository_query.dart` `paging`), so page 0 would repeat page 1. */
+  async listAll(params = {}, pageSize = 500) {
+    const all = [];
+    for (let page = 1;; page++) {
+      const batch = (await this.list({
+        ...params,
+        limit: pageSize,
+        page
+      })) ?? [];
+      all.push(...batch);
+      if (batch.length < pageSize) return all;
+    }
+  }
+  async find(id, include) {
+    try {
+      return await this.call('GET', `${this.route}/${encodeURIComponent(id)}${buildQuery({
+        include
+      })}`);
+    } catch (err) {
+      if (err?.apiError?.errorCode === 404 || err?.message === 'Not Found') return null;
+      throw err;
+    }
+  }
+  save(json, query) {
+    return this.call('POST', `${this.route}${query ? '?' + query : ''}`, ensureBodyId(json));
+  }
+  delete(id) {
+    return this.call('DELETE', `${this.route}/${encodeURIComponent(id)}`);
+  }
+}
+exports.InternalDataSource = InternalDataSource;
 const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
 class NodeGroupsDataSource extends NodeHttpDataSource {
   constructor(client) {
-    super(client, 'groups');
+    super(client, 'groups', '/groups');
   }
   async save(group, saveRelations = false) {
     const q = buildQuery({
@@ -190,6 +388,10 @@ class NodeGroupsDataSource extends NodeHttpDataSource {
     const matches = await this.lookup(idOrName);
     let candidates = matches;
     if (opts.personalOnly) candidates = matches.filter(g => g?.personal === true);
+    // lookup is a substring search ('admin' also finds 'Administrators'): an exact name wins
+    const want = idOrName.toLowerCase();
+    const exact = candidates.filter(g => [g?.name, g?.friendlyName].some(n => (n ?? '').toLowerCase() === want));
+    if (exact.length) candidates = exact;
     if (!candidates.length) {
       const suffix = opts.personalOnly ? ' (personal)' : '';
       throw new Error(`No group matching '${idOrName}'${suffix}`);
@@ -305,15 +507,19 @@ class NodeGroupsDataSource extends NodeHttpDataSource {
     }
     return results;
   }
-  async getMembers(group, admin) {
-    const parent = await this.resolve(group);
+  async getMembers(group, admin, personalOnly = false) {
+    const parent = await this.resolve(group, {
+      personalOnly
+    });
     const q = buildQuery({
       admin: admin === undefined ? undefined : String(admin)
     });
     return this.client.get(`/public/v1/groups/${encodeURIComponent(parent.id)}/members${q}`);
   }
-  async getMemberships(group, admin) {
-    const parent = await this.resolve(group);
+  async getMemberships(group, admin, personalOnly = false) {
+    const parent = await this.resolve(group, {
+      personalOnly
+    });
     const q = buildQuery({
       admin: admin === undefined ? undefined : String(admin)
     });
@@ -333,9 +539,12 @@ class NodeSharesDataSource {
     });
     return this.client.post(`/public/v1/entities/${name}/shares${q}`);
   }
+
+  /** `all` walks the entity's project links the way the browser's sharing dialog does; a bare `entityId` answers nothing for most entities. */
   async list(entityId) {
     const q = buildQuery({
-      entityId
+      entityId,
+      all: 'true'
     });
     return this.client.get(`/privileges/permissions${q}`);
   }
@@ -343,11 +552,21 @@ class NodeSharesDataSource {
 exports.NodeSharesDataSource = NodeSharesDataSource;
 class NodeUsersDataSource extends NodeHttpDataSource {
   constructor(client) {
-    super(client, 'users');
+    super(client, 'users', '/users');
   }
   async save(user) {
     return this.client.post('/public/v1/users', ensureBodyId(user));
   }
+
+  /**
+   * Datagrok has no user deletion: the server offers no route, the UI only blocks, and removing
+   * the entity record (`DELETE /entities/{id}`, what the batch API does) leaves the `users` row,
+   * the personal group and the root project behind, so the login can never be re-created.
+   */
+  async delete(idOrLogin) {
+    const login = typeof idOrLogin === 'string' ? idOrLogin : idOrLogin?.id ?? '';
+    throw new Error(`Users cannot be deleted through the API; block the account instead: grok s users block ${login}`);
+  }
   async block(user) {
     await this.client.post('/public/v1/users/block', user);
   }
@@ -358,7 +577,7 @@ class NodeUsersDataSource extends NodeHttpDataSource {
 exports.NodeUsersDataSource = NodeUsersDataSource;
 class NodeConnectionsDataSource extends NodeHttpDataSource {
   constructor(client) {
-    super(client, 'connections');
+    super(client, 'connections', '/connectors/connections');
   }
   async save(conn, saveCredentials = false) {
     const q = buildQuery({
@@ -366,6 +585,12 @@ class NodeConnectionsDataSource extends NodeHttpDataSource {
     });
     return this.client.post(`/public/v1/connections${q}`, conn);
   }
+
+  /** The route answers 200 for an unknown id, so look the connection up first. */
+  async delete(idOrName) {
+    const conn = typeof idOrName === 'string' ? await this.find(idOrName) : idOrName;
+    await super.delete(conn?.id ?? '');
+  }
   async test(conn) {
     const result = await this.client.post(`/public/v1/connections/test`, conn);
     const text = typeof result === 'string' ? result.replace(/^"|"$/g, '') : String(result ?? '');
@@ -374,25 +599,38 @@ class NodeConnectionsDataSource extends NodeHttpDataSource {
 }
 exports.NodeConnectionsDataSource = NodeConnectionsDataSource;
 class NodeFuncsDataSource extends NodeHttpDataSource {
+  constructor(client) {
+    super(client, 'functions', '/log/funcs');
+  }
   async run(name, params) {
     const normalizedName = name.replace(':', '.');
-    const result = await this.client.post(`/public/v1/functions/${encodeURIComponent(normalizedName)}/call`, params ?? {});
-    // Datagrok returns HTTP 200 with an ApiError body when the function doesn't exist
-    const parsed = typeof result === 'string' ? tryParseJson(result) : result;
-    if (parsed?.['#type'] === 'ApiError') {
-      const err = {
-        error: parsed.message ?? 'Function call failed',
-        errorCode: parsed.errorCode,
-        stackTrace: parsed.stackTrace
-      };
-      throw Object.assign(new Error(err.error), {
-        apiError: err
-      });
-    }
-    return result;
+    return this.client.post(`/public/v1/functions/${encodeURIComponent(normalizedName)}/call`, params ?? {});
+  }
+
+  /** The public API has no DELETE for functions; scripts and queries go to their own routers. */
+  async delete(idOrName) {
+    const func = typeof idOrName === 'string' ? await this.find(idOrName) : idOrName;
+    const route = func?.['#type'] === 'Script' ? '/scripts' : func?.['#type'] === 'DataQuery' ? '/connectors/queries' : null;
+    if (!route) throw new Error(`Only scripts and queries can be deleted; '${func?.name ?? idOrName}' is a ${func?.['#type'] ?? 'function'} (package functions go away with their package)`);
+    await this.client.del(`${route}/${encodeURIComponent(func.id)}`);
   }
 }
+
+/**
+ * The server binds a call's arguments by parameter name, so positional values have to be
+ * mapped onto the function's inputs (`parameterInfos`, declared order; outputs carry
+ * `isInput: false`) before the call.
+ */
 exports.NodeFuncsDataSource = NodeFuncsDataSource;
+function mapPositionalParams(params, parameterInfos, funcName) {
+  const positional = Object.keys(params).filter(k => /^\d+$/.test(k)).sort((a, b) => Number(a) - Number(b));
+  if (!positional.length) return params;
+  const inputs = Object.values(parameterInfos ?? {}).filter(p => p?.isInput !== false).map(p => p.name);
+  if (positional.length > inputs.length) throw new Error(`${funcName} takes ${inputs.length} input${inputs.length === 1 ? '' : 's'} (${inputs.join(', ')}), got ${positional.length}`);
+  const mapped = {};
+  for (const [k, v] of Object.entries(params)) mapped[/^\d+$/.test(k) ? inputs[Number(k)] : k] = v;
+  return mapped;
+}
 function tryParseJson(s) {
   try {
     return JSON.parse(s);
@@ -400,6 +638,106 @@ function tryParseJson(s) {
     return null;
   }
 }
+class NodePackagesDataSource extends NodeHttpDataSource {
+  constructor(client) {
+    super(client, 'packages');
+  }
+
+  /** GET /packages (internal API) — includes publishedVersions and repository. */
+  async listFull(text) {
+    return this.client.get(`/packages${buildQuery({
+      text
+    })}`);
+  }
+
+  /** No `/count` route for packages; the catalog is small enough to count client-side. */
+  async count() {
+    return (await this.listFull(this._filter || undefined)).length;
+  }
+
+  /** Resolve by UUID, name, or friendlyName (case-insensitive). Returns null when
+      not found so callers can pass the raw string through and let the server's own
+      'Package not found' surface. */
+  async resolve(idOrName) {
+    const all = await this.listFull();
+    if (UUID_RE.test(idOrName)) return all.find(p => p?.id === idOrName) ?? null;
+    const want = idOrName.toLowerCase();
+    const matches = all.filter(p => (p?.name ?? '').toLowerCase() === want || (p?.friendlyName ?? '').toLowerCase() === want);
+    if (matches.length > 1) {
+      const list = matches.map(p => `  ${p.id}  ${p.name}`).join('\n');
+      throw new Error(`Multiple packages match '${idOrName}':\n${list}\nUse the ID to disambiguate.`);
+    }
+    return matches[0] ?? null;
+  }
+
+  /** Install/activate via the DeployPackageVersion server func: pulls the version
+      from the package repository (npm) when needed, then makes it current.
+      'latest' marks the package for server-side auto-update. Synchronous — returns
+      the new published-package id, or null when nothing changed. */
+  async install(name, desiredVersion = 'latest') {
+    const result = await new NodeFuncsDataSource(this.client).run('DeployPackageVersion', {
+      name,
+      desiredVersion
+    });
+    const value = result && typeof result === 'object' ? result.result ?? result.id ?? null : result;
+    const text = typeof value === 'string' ? value.replace(/^"|"$/g, '') : '';
+    return text && text !== 'null' ? text : null;
+  }
+  async uninstall(idOrName) {
+    const pkg = await this.resolve(idOrName);
+    if (!pkg) throw new Error(`Package '${idOrName}' not found`);
+    await this.client.del(`/packages/${encodeURIComponent(pkg.id)}`);
+    return {
+      id: pkg.id,
+      name: pkg.name,
+      repoBacked: !!pkg.repository
+    };
+  }
+  async versions(idOrName) {
+    const pkg = await this.resolve(idOrName);
+    if (!pkg) throw new Error(`Package '${idOrName}' not found`);
+    return Array.isArray(pkg.publishedVersions) ? pkg.publishedVersions : [];
+  }
+  async outdated() {
+    const all = await this.listFull();
+    const rows = [];
+    for (const p of all) {
+      const versions = Array.isArray(p?.publishedVersions) ? p.publishedVersions : [];
+      // isLocal is not reliably serialized, so 'installed' = has a current non-debug version
+      const current = versions.find(v => v?.isCurrent && !v?.debug);
+      const latest = versions.find(v => v?.isLatest);
+      // Numeric compare so locally published builds (1.2.3.X-suffix) don't count
+      // as outdated against an equal or older registry version.
+      if (current && latest && NodePackagesDataSource.compareVersions(latest.version, current.version) > 0) rows.push({
+        name: p.name,
+        installed: current.version,
+        latest: latest.version,
+        desiredVersion: p.desiredVersion ?? ''
+      });
+    }
+    return rows;
+  }
+
+  /** Compare the leading numeric dot-groups of two version strings (suffixes like
+      '.X-4e32bb91' are ignored): positive when a > b. */
+  static compareVersions(a, b) {
+    const nums = s => (String(s).match(/^\d+(\.\d+)*/)?.[0] ?? '').split('.').map(Number);
+    const av = nums(a),
+      bv = nums(b);
+    for (let i = 0; i < Math.max(av.length, bv.length); i++) {
+      const d = (av[i] ?? 0) - (bv[i] ?? 0);
+      if (d !== 0) return d;
+    }
+    return 0;
+  }
+
+  /** The public API has no DELETE for packages — route to the internal endpoint. */
+  async delete(idOrEntity) {
+    const id = typeof idOrEntity === 'string' ? idOrEntity : idOrEntity?.id ?? '';
+    await this.client.del(`/packages/${encodeURIComponent(id)}`);
+  }
+}
+exports.NodePackagesDataSource = NodePackagesDataSource;
 class NodeFilesDataSource {
   constructor(client) {
     this.client = client;
@@ -428,16 +766,24 @@ class NodeFilesDataSource {
       path
     };
   }
+
+  /**
+   * The public files route only downloads, so listing goes through the internal
+   * `/connectors/connections/{id}/files/<path>` the file browser uses (a trailing `/`
+   * is what makes it a listing). Resolves to `FileInfo` records (`path`, `isFile`, `size`).
+   */
   async list(filePath, recursive = false) {
     const {
       connector,
       path
     } = this.splitPath(filePath);
+    const conn = await new NodeConnectionsDataSource(this.client).find(connector);
     const q = buildQuery({
       recursive: recursive ? 'true' : undefined
     });
-    const seg = path ? `${connector}/${path}` : connector;
-    return this.client.get(`/public/v1/files/${seg}${q}`);
+    const seg = path ? `${path.replace(/\/+$/, '')}/` : '';
+    const res = await this.client.get(`/connectors/connections/${encodeURIComponent(conn.id)}/files/${seg}${q}`);
+    return Array.isArray(res) ? res : [];
   }
   async get(filePath) {
     const {
@@ -447,6 +793,26 @@ class NodeFilesDataSource {
     const seg = path ? `${connector}/${path}` : connector;
     return this.client.get(`/public/v1/files/${seg}`);
   }
+
+  /** The file's raw bytes, for copying a share file between instances. */
+  async readBytes(filePath) {
+    const {
+      connector,
+      path
+    } = this.splitPath(filePath);
+    if (!path) throw new Error(`Path must name a file inside the share: got '${filePath}'`);
+    return this.client.getBytes(`/public/v1/files/${connector}/${path}`);
+  }
+
+  /** Upload bytes already in memory — the bundle holds them, there is no local file. */
+  async writeBytes(filePath, bytes) {
+    const {
+      connector,
+      path
+    } = this.splitPath(filePath);
+    if (!path) throw new Error(`Path must name a file inside the share: got '${filePath}'`);
+    return await this.client.putBytes(`/public/v1/files/${connector}/${path}`, bytes);
+  }
   async delete(filePath) {
     const {
       connector,
@@ -463,43 +829,47 @@ class NodeFilesDataSource {
    */
   async put(localPath, remotePath) {
     const fs = require('fs');
-    const {
-      connector,
-      path
-    } = this.splitPath(remotePath);
-    if (!path) throw new Error(`Remote path must include a file name after the connector: got '${remotePath}'`);
     const bytes = fs.readFileSync(localPath);
-    const res = await this.client.putBytes(`/public/v1/files/${connector}/${path}`, bytes);
     return {
       path: remotePath,
       size: bytes.length,
-      response: res
+      response: await this.writeBytes(remotePath, bytes)
     };
   }
 }
+
+/**
+ * Tables live under a project namespace (`Admin:MyTable:MyTable`), so a table is addressed by
+ * UUID, by that full name, or by its bare name when only one table carries it.
+ */
 exports.NodeFilesDataSource = NodeFilesDataSource;
-class NodeTablesDataSource {
+class NodeTablesDataSource extends NodeHttpDataSource {
   constructor(client) {
-    this.client = client;
+    super(client, 'tables', '/tables');
   }
-
-  /** GET /public/v1/tables/{name} — returns CSV text. Name accepts UUID or `namespace:name`. */
-  async download(name) {
-    const seg = encodeURIComponent(name.replace(/:/g, '.'));
-    const result = await this.client.get(`/public/v1/tables/${seg}`);
-    // Datagrok returns HTTP 200 + ApiError JSON when the table isn't found.
-    const parsed = typeof result === 'string' ? tryParseJson(result) : result;
-    if (parsed && typeof parsed === 'object' && parsed['#type'] === 'ApiError') {
-      const err = {
-        error: parsed.message ?? 'Table download failed',
-        errorCode: parsed.errorCode,
-        stackTrace: parsed.stackTrace
-      };
-      throw Object.assign(new Error(err.error), {
-        apiError: err
-      });
+  async find(idOrName) {
+    if (UUID_RE.test(idOrName)) return this.client.get(`/tables/${encodeURIComponent(idOrName)}`);
+    const all = await this.client.get(`/tables${buildQuery({
+      text: idOrName,
+      limit: 100
+    })}`);
+    const matches = all.filter(t => t?.name === idOrName || `${t?.namespace ?? ''}${t?.name ?? ''}` === idOrName);
+    if (!matches.length) throw new Error(`No table named '${idOrName}'`);
+    if (matches.length > 1) {
+      const list = matches.map(t => `  ${t.id}  ${t.namespace ?? ''}${t.name}`).join('\n');
+      throw new Error(`Multiple tables match '${idOrName}':\n${list}\nUse the full name or the ID.`);
     }
-    return result;
+    return matches[0];
+  }
+  async delete(idOrName) {
+    const table = typeof idOrName === 'string' ? await this.find(idOrName) : idOrName;
+    await this.client.del(`/tables/${encodeURIComponent(table?.id ?? '')}`);
+  }
+
+  /** GET /public/v1/tables/{id} — returns CSV text. */
+  async download(idOrName) {
+    const table = await this.find(idOrName);
+    return this.client.get(`/public/v1/tables/${encodeURIComponent(table.id)}`);
   }
 
   /**
@@ -515,6 +885,193 @@ class NodeTablesDataSource {
   }
 }
 exports.NodeTablesDataSource = NodeTablesDataSource;
+/** `'<schema>'` or `'<schema>.<table>'` — the address every `grok s domains` verb takes. */
+function parseDomainAddress(s, opts = {}) {
+  const str = String(s ?? '');
+  const dot = str.indexOf('.');
+  const address = dot === -1 ? {
+    schema: str
+  } : {
+    schema: str.slice(0, dot),
+    table: str.slice(dot + 1)
+  };
+  if (!address.schema || dot !== -1 && !address.table) throw new Error(`Invalid domain address '${str}': expected <schema> or <schema>.<table>`);
+  if (opts.table === true && !address.table) throw new Error(`'${str}' names a schema; this command needs a table: <schema>.<table>`);
+  if (opts.table === false && address.table) throw new Error(`'${str}' names a table; this command needs a schema`);
+  return address;
+}
+/**
+ * Client for entity-mapped domain tables (`/domains/...`, DomainsRouter). Rows are plain
+ * JSON objects; the server validates, permission-checks and audits every write. Errors
+ * arrive as HTTP 4xx with a JSON envelope, so `client.request` throws with `apiError.body`
+ * carrying the structured fields (per-row `rows`, a dry-run `plan`, version numbers).
+ */
+class NodeDomainsDataSource {
+  constructor(client) {
+    this.client = client;
+  }
+  rows(schema, table) {
+    return `/domains/${encodeURIComponent(schema)}/${encodeURIComponent(table)}`;
+  }
+
+  /** Registered schemas with their tables (`GET /domains/schemas`); [text] is a smart filter. */
+  schemas(text) {
+    return this.client.get(`/domains/schemas${buildQuery({
+      text: text || undefined
+    })}`);
+  }
+  async schema(name) {
+    const all = await this.schemas();
+    const s = all.find(x => x?.name === name);
+    if (!s) throw new Error(`Domain schema '${name}' not found`);
+    return s;
+  }
+
+  /** Registry entity id of a schema or a table — the target of the grants endpoints. */
+  async entityId(address) {
+    const s = await this.schema(address.schema);
+    if (!address.table) return s.id;
+    const t = (s.tables ?? []).find(x => x?.name === address.table);
+    if (!t) throw new Error(`Domain table '${address.schema}.${address.table}' not found`);
+    return t.id;
+  }
+  manifest(schema) {
+    return this.client.get(`/domains/schemas/${encodeURIComponent(schema)}/manifest`);
+  }
+  createSchema(name, friendlyName, description) {
+    return this.client.post('/domains/schemas', {
+      name,
+      friendlyName,
+      description
+    });
+  }
+  applySchema(schema, body, dryRun = false) {
+    const q = buildQuery({
+      dryRun: dryRun ? 'true' : undefined
+    });
+    return this.client.post(`/domains/schemas/${encodeURIComponent(schema)}/apply${q}`, body);
+  }
+  deleteSchema(schema) {
+    return this.client.del(`/domains/schemas/${encodeURIComponent(schema)}`);
+  }
+  schemaAudit(schema, limit) {
+    return this.client.get(`/domains/schemas/${encodeURIComponent(schema)}/audit${buildQuery({
+      limit
+    })}`);
+  }
+  tableAudit(schema, table, limit) {
+    return this.client.get(`${this.rows(schema, table)}/audit${buildQuery({
+      limit
+    })}`);
+  }
+  rowAudit(schema, table, id) {
+    return this.client.get(`${this.rows(schema, table)}/${encodeURIComponent(id)}/audit`);
+  }
+  grants(entityId) {
+    return this.client.get(`/domains/grants/${encodeURIComponent(entityId)}`);
+  }
+  grant(entityId, group, permission) {
+    return this.client.post(`/domains/grants/${encodeURIComponent(entityId)}`, {
+      group,
+      permission
+    });
+  }
+  revoke(entityId, group, permission) {
+    return this.client.del(`/domains/grants/${encodeURIComponent(entityId)}${buildQuery({
+      group,
+      permission
+    })}`);
+  }
+  capabilities(schema, table) {
+    return this.client.get(`${this.rows(schema, table)}/capabilities`);
+  }
+
+  /** JSON rows; spec = {filter, sort, columns, expand, limit, offset} (10k row cap). */
+  query(schema, table, spec = {}) {
+    return this.client.post(`${this.rows(schema, table)}/query`, spec);
+  }
+
+  /** The same query as a d42 DataFrame blob (10M row cap). */
+  queryD42(schema, table, spec = {}) {
+    return this.client.postForBytes(`${this.rows(schema, table)}/query`, {
+      ...spec,
+      format: 'd42'
+    });
+  }
+  aggregate(schema, table, spec) {
+    return this.client.post(`${this.rows(schema, table)}/aggregate`, spec);
+  }
+  async count(schema, table, filter) {
+    const spec = {
+      measures: [{
+        fn: 'count'
+      }]
+    };
+    if (filter) spec.filter = filter;
+    const rows = await this.aggregate(schema, table, spec);
+    return Number(rows?.[0]?.count ?? 0);
+  }
+
+  /** One row, or null when it does not exist or is not visible (the server answers 404). */
+  async getRow(schema, table, id) {
+    try {
+      return await this.client.get(`${this.rows(schema, table)}/${encodeURIComponent(id)}`);
+    } catch (err) {
+      if (err?.apiError?.errorCode === 404) return null;
+      throw err;
+    }
+  }
+
+  /** [rows] is one row object or an array; resolves to per-row `{id, created}` reports. */
+  async insert(schema, table, rows, errorOnDuplicate = false) {
+    const q = buildQuery({
+      errorOnDuplicate: errorOnDuplicate ? 'true' : undefined
+    });
+    const res = await this.client.post(`${this.rows(schema, table)}${q}`, rows);
+    return Array.isArray(res) ? res : [res];
+  }
+  update(schema, table, id, values, version) {
+    const body = {
+      values
+    };
+    if (version !== undefined) body.version = version;
+    return this.client.request('PATCH', `${this.rows(schema, table)}/${encodeURIComponent(id)}`, body);
+  }
+  deleteRow(schema, table, id) {
+    return this.client.del(`${this.rows(schema, table)}/${encodeURIComponent(id)}`);
+  }
+
+  /** Soft-deletes up to [limit] (≤1000) matching rows in one transaction; `{deleted, hasMore}`. */
+  deleteWhere(schema, table, filter, limit) {
+    const body = {
+      filter
+    };
+    if (limit !== undefined) body.limit = limit;
+    return this.client.post(`${this.rows(schema, table)}/delete`, body);
+  }
+
+  /**
+   * Bulk upload: [bytes] is the file content and [contentType] selects how the server
+   * reads it — `text/csv`, `application/octet-stream` (d42), or `application/json` (a row
+   * array, bare or under `rows`). Resolves to the batch report `{inserted, updated,
+   * skipped, errorCount, rows}`; a report-carrying failure rejects with the report in
+   * `apiError.body`.
+   */
+  batch(schema, table, bytes, contentType, options = {}) {
+    const q = buildQuery({
+      mode: options.mode ?? 'insert',
+      allOrNothing: options.allOrNothing === false ? 'false' : 'true',
+      errorOnDuplicate: options.errorOnDuplicate ? 'true' : undefined
+    });
+    return this.client.putBytes(`${this.rows(schema, table)}/batch${q}`, bytes, contentType);
+  }
+
+  /** Ordered ops (`{op, table, ref?, values?, id?, expectedVersion?}`) applied atomically. */
+  transaction(schema, ops) {
+    return this.client.post(`/domains/${encodeURIComponent(schema)}/transaction`, ops);
+  }
+}
+exports.NodeDomainsDataSource = NodeDomainsDataSource;
 class NodeDapi {
   constructor(client) {
     this.client = client;
@@ -526,22 +1083,22 @@ class NodeDapi {
     return new NodeGroupsDataSource(this.client);
   }
   get functions() {
-    return new NodeFuncsDataSource(this.client, 'functions');
+    return new NodeFuncsDataSource(this.client);
   }
   get connections() {
     return new NodeConnectionsDataSource(this.client);
   }
   get queries() {
-    return new NodeHttpDataSource(this.client, 'queries');
+    return this.internal('/connectors/queries');
   }
   get scripts() {
-    return new NodeHttpDataSource(this.client, 'scripts');
+    return this.internal('/scripts');
   }
   get packages() {
-    return new NodeHttpDataSource(this.client, 'packages');
+    return new NodePackagesDataSource(this.client);
   }
   get reports() {
-    return new NodeHttpDataSource(this.client, 'reports');
+    return this.internal('/reports');
   }
   get files() {
     return new NodeFilesDataSource(this.client);
@@ -552,35 +1109,124 @@ class NodeDapi {
   get tables() {
     return new NodeTablesDataSource(this.client);
   }
-  async raw(method, path, body) {
-    // Raw paths are relative to server root (e.g. /api/users/current).
-    // Strip the trailing /api from baseUrl to avoid double prefix.
-    const serverRoot = this.client.baseUrl.replace(/\/api\/?$/, '');
-    const url = `${serverRoot}${path}`;
-    const opts = {
-      method: method.toUpperCase(),
-      headers: {
-        'Authorization': this.client.token,
-        'Content-Type': 'application/json'
-      }
+  get domains() {
+    return new NodeDomainsDataSource(this.client);
+  }
+  internal(route) {
+    return new InternalDataSource(this.client, route);
+  }
+  async serverInfo() {
+    const raw = await this.client.get('/info/server');
+    const info = typeof raw === 'string' ? JSON.parse(raw) : raw;
+    return {
+      version: info?.Version ?? '',
+      commit: info?.Commit
     };
-    if (body !== undefined) opts.body = JSON.stringify(body);
-    const res = await fetch(url, opts);
-    const ct = res.headers.get('content-type') ?? '';
-    if (ct.includes('application/json')) return res.json();
-    return res.text();
+  }
+
+  /** Any API endpoint; [path] is API-relative (`/users/current`), a leading `/api` is accepted. */
+  async raw(method, path, body) {
+    return this.client.request(method.toUpperCase(), apiPath(path), body);
   }
   async batch(request) {
     return this.client.post('/public/v1/batch', request);
   }
-  async describe(entityType) {
-    try {
-      return await this.client.get(`/public/v1/entity-types/${encodeURIComponent(entityType)}`);
-    } catch {
-      return await this.client.get(`/entities/types${buildQuery({
-        name: entityType
-      })}`);
-    }
+
+  /**
+   * The shape of an entity type: its registry record (`/entities/types`) plus the top-level
+   * fields of one existing entity of that type, since the server publishes no JSON schema.
+   * [nameOrAlias] is a `grok s` entity (`connections`) or a type name (`DataConnection`).
+   */
+  async describe(nameOrAlias) {
+    const alias = DESCRIBE_ALIASES[nameOrAlias.toLowerCase()];
+    const typeName = alias?.type ?? nameOrAlias;
+    const types = await this.client.get(`/entities/types${buildQuery({
+      showSystem: 'true',
+      text: `name="${typeName}"`,
+      limit: 5
+    })}`);
+    const type = types.find(t => (t?.name ?? '').toLowerCase() === typeName.toLowerCase()) ?? null;
+    const sampleFrom = alias?.sample ?? DESCRIBE_SAMPLES[typeName];
+    if (!type && !sampleFrom) throw new Error(`Unknown entity type '${nameOrAlias}'. Try one of: ${Object.keys(DESCRIBE_ALIASES).join(', ')}, or a type name such as Project`);
+    const sample = sampleFrom ? (await sampleFrom(this))[0] ?? null : null;
+    const fields = sample ? Object.entries(sample).map(([field, v]) => ({
+      field,
+      type: describeType(v),
+      example: describeExample(v)
+    })) : [];
+    return {
+      type,
+      fields,
+      sample
+    };
+  }
+}
+exports.NodeDapi = NodeDapi;
+const DESCRIBE_SAMPLES = {
+  User: d => d.users.by(1).list(),
+  UserGroup: d => d.groups.by(1).list(),
+  DataConnection: d => d.connections.by(1).list(),
+  DataQuery: d => d.queries.list({
+    limit: 1
+  }),
+  Script: d => d.scripts.list({
+    limit: 1
+  }),
+  Func: d => d.functions.by(1).list(),
+  Package: d => d.packages.by(1).list(),
+  UserReport: d => d.reports.list({
+    limit: 1
+  }),
+  TableInfo: d => d.tables.by(1).list(),
+  Project: d => d.internal('/projects').list({
+    limit: 1
+  }),
+  FileInfo: d => d.internal('/files').list({
+    limit: 1
+  })
+};
+const DESCRIBE_ALIASES = {
+  users: {
+    type: 'User'
+  },
+  groups: {
+    type: 'UserGroup'
+  },
+  connections: {
+    type: 'DataConnection'
+  },
+  queries: {
+    type: 'DataQuery'
+  },
+  scripts: {
+    type: 'Script'
+  },
+  functions: {
+    type: 'Func'
+  },
+  packages: {
+    type: 'Package'
+  },
+  reports: {
+    type: 'UserReport'
+  },
+  tables: {
+    type: 'TableInfo'
+  },
+  projects: {
+    type: 'Project'
+  },
+  files: {
+    type: 'FileInfo'
   }
+};
+function describeType(v) {
+  if (v === null || v === undefined) return 'null';
+  if (Array.isArray(v)) return `array(${v.length})`;
+  if (typeof v === 'object') return v['#type'] ?? (v.id ? 'ref' : 'object');
+  return typeof v;
 }
-exports.NodeDapi = NodeDapi;
+function describeExample(v) {
+  const s = v === null || v === undefined ? '' : typeof v === 'object' ? JSON.stringify(v) : String(v);
+  return s.length > 60 ? `${s.slice(0, 57)}...` : s;
+}