datagrok-tools 6.1.8 → 6.1.10

package/bin/utils/node-dapi.js

@@ -0,0 +1,459 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.NodeUsersDataSource = exports.NodeSharesDataSource = exports.NodeHttpDataSource = exports.NodeGroupsDataSource = exports.NodeFuncsDataSource = exports.NodeFilesDataSource = exports.NodeDapi = exports.NodeConnectionsDataSource = exports.NodeApiClient = void 0;
+exports.ensureBodyId = ensureBodyId;
+var _crypto = require("crypto");
+/// Docs: [Grok Dapi](/docs/plans/grok-dapi/)
+
+function ensureBodyId(body) {
+  if (body && typeof body === 'object' && !body.id) body.id = (0, _crypto.randomUUID)();
+  return body;
+}
+class NodeApiClient {
+  constructor(baseUrl, token) {
+    this.baseUrl = baseUrl;
+    this.token = token;
+  }
+  static async login(baseUrl, devKey) {
+    const res = await fetch(`${baseUrl}/users/login/dev/${devKey}`, {
+      method: 'POST'
+    });
+    const json = await res.json();
+    if (!json.token) throw new Error('Login failed. Check your developer key.');
+    return new NodeApiClient(baseUrl, json.token);
+  }
+  async request(method, path, body, headers) {
+    const url = `${this.baseUrl}${path}`;
+    const opts = {
+      method,
+      headers: {
+        'Authorization': this.token,
+        'Content-Type': 'application/json',
+        ...headers
+      }
+    };
+    if (body !== undefined) opts.body = JSON.stringify(body);
+    const res = await fetch(url, opts);
+    if (!res.ok) {
+      // Read as text first to avoid "Body has already been read" when JSON.parse fails
+      const rawText = await res.text();
+      let errBody;
+      try {
+        errBody = JSON.parse(rawText);
+      } catch {
+        errBody = {
+          error: rawText || `HTTP ${res.status}`
+        };
+      }
+      const err = {
+        error: errBody?.message ?? errBody?.error ?? `HTTP ${res.status}`,
+        source: errBody?.source ?? 'Server',
+        errorCode: errBody?.errorCode ?? res.status,
+        stackTrace: errBody?.stackTrace
+      };
+      throw Object.assign(new Error(err.error), {
+        apiError: err
+      });
+    }
+    if (res.status === 204 || res.headers.get('content-length') === '0') return null;
+    const ct = res.headers.get('content-type') ?? '';
+    if (ct.includes('application/json')) return res.json();
+    return res.text();
+  }
+  get(path) {
+    return this.request('GET', path);
+  }
+  post(path, body) {
+    return this.request('POST', path, body);
+  }
+  del(path) {
+    return this.request('DELETE', path);
+  }
+}
+exports.NodeApiClient = NodeApiClient;
+function buildQuery(params) {
+  const entries = Object.entries(params).filter(([, v]) => v !== undefined && v !== null && v !== '');
+  if (!entries.length) return '';
+  return '?' + entries.map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(String(v))}`).join('&');
+}
+class NodeHttpDataSource {
+  _filter = '';
+  _limit = 50;
+  _page = 0;
+  _order = '';
+  constructor(client, path) {
+    this.client = client;
+    this.path = path;
+  }
+  filter(w) {
+    this._filter = w;
+    return this;
+  }
+  by(n) {
+    this._limit = n;
+    return this;
+  }
+  page(n) {
+    this._page = n;
+    return this;
+  }
+  order(field, desc = false) {
+    this._order = desc ? `-${field}` : field;
+    return this;
+  }
+  async list() {
+    const q = buildQuery({
+      text: this._filter || undefined,
+      limit: this._limit,
+      page: this._page || undefined,
+      order: this._order || undefined
+    });
+    return this.client.get(`/public/v1/${this.path}${q}`);
+  }
+  async find(id) {
+    return this.client.get(`/public/v1/${this.path}/${encodeURIComponent(id.replace(':', '.'))}`);
+  }
+  async count() {
+    const q = buildQuery({
+      text: this._filter || undefined
+    });
+    const res = await this.client.get(`/public/v1/${this.path}/count${q}`);
+    return typeof res === 'number' ? res : res?.count ?? 0;
+  }
+  async delete(idOrEntity) {
+    const id = typeof idOrEntity === 'string' ? idOrEntity : idOrEntity?.id ?? '';
+    await this.client.del(`/public/v1/${this.path}/${encodeURIComponent(id)}`);
+  }
+}
+exports.NodeHttpDataSource = NodeHttpDataSource;
+const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+class NodeGroupsDataSource extends NodeHttpDataSource {
+  constructor(client) {
+    super(client, 'groups');
+  }
+  async save(group, saveRelations = false) {
+    const q = buildQuery({
+      saveRelations: saveRelations ? 'true' : undefined
+    });
+    return this.client.post(`/public/v1/groups${q}`, ensureBodyId(group));
+  }
+  async lookup(name) {
+    const q = buildQuery({
+      query: name
+    });
+    return this.client.get(`/public/v1/groups/lookup${q}`);
+  }
+  async resolve(idOrName, opts = {}) {
+    if (UUID_RE.test(idOrName)) return this.find(idOrName);
+    const matches = await this.lookup(idOrName);
+    let candidates = matches;
+    if (opts.personalOnly) candidates = matches.filter(g => g?.personal === true);
+    if (!candidates.length) {
+      const suffix = opts.personalOnly ? ' (personal)' : '';
+      throw new Error(`No group matching '${idOrName}'${suffix}`);
+    }
+    if (candidates.length > 1) {
+      const list = candidates.map(g => `  ${g.id}  ${g.friendlyName ?? g.name ?? ''}`).join('\n');
+      throw new Error(`Multiple groups match '${idOrName}':\n${list}\nUse the ID to disambiguate.`);
+    }
+    return candidates[0];
+  }
+  async addMembers(group, members, isAdmin = false, personalOnly = false) {
+    const parent = await this.resolve(group);
+    const children = Array.isArray(parent.children) ? parent.children : [];
+    const results = [];
+    let mutated = false;
+    for (const m of members) {
+      let child;
+      try {
+        child = await this.resolve(m, {
+          personalOnly
+        });
+      } catch (err) {
+        results.push({
+          member: m,
+          status: 'error',
+          error: err?.message ?? String(err)
+        });
+        continue;
+      }
+      const existing = children.find(r => r?.child?.id === child.id);
+      if (existing) {
+        if (existing.isAdmin === isAdmin) {
+          results.push({
+            member: m,
+            status: 'noop'
+          });
+        } else {
+          existing.isAdmin = isAdmin;
+          mutated = true;
+          results.push({
+            member: m,
+            status: 'updated'
+          });
+        }
+      } else {
+        children.push({
+          parent: {
+            id: parent.id
+          },
+          child: {
+            id: child.id
+          },
+          isAdmin
+        });
+        mutated = true;
+        results.push({
+          member: m,
+          status: 'added'
+        });
+      }
+    }
+    if (mutated) {
+      parent.children = children;
+      await this.save(parent, true);
+    }
+    return results;
+  }
+  async removeMembers(group, members, personalOnly = false) {
+    const parent = await this.resolve(group);
+    const results = [];
+    const children = Array.isArray(parent.children) ? parent.children : [];
+    let mutated = false;
+    for (const m of members) {
+      let child;
+      try {
+        child = await this.resolve(m, {
+          personalOnly
+        });
+      } catch (err) {
+        results.push({
+          member: m,
+          status: 'error',
+          error: err?.message ?? String(err)
+        });
+        continue;
+      }
+      const idx = children.findIndex(r => r?.child?.id === child.id);
+      if (idx === -1) {
+        results.push({
+          member: m,
+          status: 'not-member'
+        });
+      } else {
+        children.splice(idx, 1);
+        mutated = true;
+        results.push({
+          member: m,
+          status: 'removed'
+        });
+      }
+    }
+    if (mutated) {
+      parent.children = children;
+      await this.save(parent, true);
+    }
+    return results;
+  }
+  async getMembers(group, admin) {
+    const parent = await this.resolve(group);
+    const q = buildQuery({
+      admin: admin === undefined ? undefined : String(admin)
+    });
+    return this.client.get(`/public/v1/groups/${encodeURIComponent(parent.id)}/members${q}`);
+  }
+  async getMemberships(group, admin) {
+    const parent = await this.resolve(group);
+    const q = buildQuery({
+      admin: admin === undefined ? undefined : String(admin)
+    });
+    return this.client.get(`/public/v1/groups/${encodeURIComponent(parent.id)}/memberships${q}`);
+  }
+}
+exports.NodeGroupsDataSource = NodeGroupsDataSource;
+class NodeSharesDataSource {
+  constructor(client) {
+    this.client = client;
+  }
+  async share(entity, groups, access = 'View') {
+    const name = encodeURIComponent(entity.replace(':', '.'));
+    const q = buildQuery({
+      groups,
+      access
+    });
+    return this.client.post(`/public/v1/entities/${name}/shares${q}`);
+  }
+  async list(entityId) {
+    const q = buildQuery({
+      entityId
+    });
+    return this.client.get(`/privileges/permissions${q}`);
+  }
+}
+exports.NodeSharesDataSource = NodeSharesDataSource;
+class NodeUsersDataSource extends NodeHttpDataSource {
+  constructor(client) {
+    super(client, 'users');
+  }
+  async save(user) {
+    return this.client.post('/public/v1/users', ensureBodyId(user));
+  }
+}
+exports.NodeUsersDataSource = NodeUsersDataSource;
+class NodeConnectionsDataSource extends NodeHttpDataSource {
+  constructor(client) {
+    super(client, 'connections');
+  }
+  async save(conn, saveCredentials = false) {
+    const q = buildQuery({
+      saveCredentials: saveCredentials ? 'true' : undefined
+    });
+    return this.client.post(`/public/v1/connections${q}`, conn);
+  }
+  async test(conn) {
+    const result = await this.client.post(`/public/v1/connections/test`, conn);
+    const text = typeof result === 'string' ? result.replace(/^"|"$/g, '') : String(result ?? '');
+    if (text !== 'ok') throw new Error(text || 'Connection test failed');
+  }
+}
+exports.NodeConnectionsDataSource = NodeConnectionsDataSource;
+class NodeFuncsDataSource extends NodeHttpDataSource {
+  async run(name, params) {
+    const normalizedName = name.replace(':', '.');
+    const result = await this.client.post(`/public/v1/functions/${encodeURIComponent(normalizedName)}/call`, params ?? {});
+    // Datagrok returns HTTP 200 with an ApiError body when the function doesn't exist
+    const parsed = typeof result === 'string' ? tryParseJson(result) : result;
+    if (parsed?.['#type'] === 'ApiError') {
+      const err = {
+        error: parsed.message ?? 'Function call failed',
+        errorCode: parsed.errorCode,
+        stackTrace: parsed.stackTrace
+      };
+      throw Object.assign(new Error(err.error), {
+        apiError: err
+      });
+    }
+    return result;
+  }
+}
+exports.NodeFuncsDataSource = NodeFuncsDataSource;
+function tryParseJson(s) {
+  try {
+    return JSON.parse(s);
+  } catch {
+    return null;
+  }
+}
+class NodeFilesDataSource {
+  constructor(client) {
+    this.client = client;
+  }
+  splitPath(filePath) {
+    const colonIdx = filePath.indexOf(':');
+    if (colonIdx === -1) return {
+      connector: filePath.replace(':', '.'),
+      path: ''
+    };
+    const connector = filePath.slice(0, colonIdx).replace(':', '.');
+    const path = filePath.slice(colonIdx + 1).replace(/^\//, '');
+    return {
+      connector,
+      path
+    };
+  }
+  async list(filePath, recursive = false) {
+    const {
+      connector,
+      path
+    } = this.splitPath(filePath);
+    const q = buildQuery({
+      recursive: recursive ? 'true' : undefined
+    });
+    const seg = path ? `${connector}/${path}` : connector;
+    return this.client.get(`/public/v1/files/${seg}${q}`);
+  }
+  async get(filePath) {
+    const {
+      connector,
+      path
+    } = this.splitPath(filePath);
+    return this.client.get(`/public/v1/files/${connector}/${path}`);
+  }
+  async delete(filePath) {
+    const {
+      connector,
+      path
+    } = this.splitPath(filePath);
+    await this.client.del(`/public/v1/files/${connector}/${path}`);
+  }
+}
+exports.NodeFilesDataSource = NodeFilesDataSource;
+class NodeDapi {
+  constructor(client) {
+    this.client = client;
+  }
+  get users() {
+    return new NodeUsersDataSource(this.client);
+  }
+  get groups() {
+    return new NodeGroupsDataSource(this.client);
+  }
+  get functions() {
+    return new NodeFuncsDataSource(this.client, 'functions');
+  }
+  get connections() {
+    return new NodeConnectionsDataSource(this.client);
+  }
+  get queries() {
+    return new NodeHttpDataSource(this.client, 'queries');
+  }
+  get scripts() {
+    return new NodeHttpDataSource(this.client, 'scripts');
+  }
+  get packages() {
+    return new NodeHttpDataSource(this.client, 'packages');
+  }
+  get reports() {
+    return new NodeHttpDataSource(this.client, 'reports');
+  }
+  get files() {
+    return new NodeFilesDataSource(this.client);
+  }
+  get shares() {
+    return new NodeSharesDataSource(this.client);
+  }
+  async raw(method, path, body) {
+    // Raw paths are relative to server root (e.g. /api/users/current).
+    // Strip the trailing /api from baseUrl to avoid double prefix.
+    const serverRoot = this.client.baseUrl.replace(/\/api\/?$/, '');
+    const url = `${serverRoot}${path}`;
+    const opts = {
+      method: method.toUpperCase(),
+      headers: {
+        'Authorization': this.client.token,
+        'Content-Type': 'application/json'
+      }
+    };
+    if (body !== undefined) opts.body = JSON.stringify(body);
+    const res = await fetch(url, opts);
+    const ct = res.headers.get('content-type') ?? '';
+    if (ct.includes('application/json')) return res.json();
+    return res.text();
+  }
+  async batch(request) {
+    return this.client.post('/public/v1/batch', request);
+  }
+  async describe(entityType) {
+    try {
+      return await this.client.get(`/public/v1/entity-types/${encodeURIComponent(entityType)}`);
+    } catch {
+      return await this.client.get(`/entities/types${buildQuery({
+        name: entityType
+      })}`);
+    }
+  }
+}
+exports.NodeDapi = NodeDapi;

package/bin/utils/server-client.js

@@ -0,0 +1,15 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.createClient = createClient;
+var _nodeDapi = require("./node-dapi");
+var _testUtils = require("./test-utils");
+async function createClient(hostArg) {
+  const {
+    url,
+    key
+  } = (0, _testUtils.getDevKey)(hostArg ?? '');
+  return _nodeDapi.NodeApiClient.login(url, key);
+}

package/bin/utils/server-output.js

@@ -0,0 +1,127 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.cellStr = cellStr;
+exports.csvCell = csvCell;
+exports.getKeys = getKeys;
+exports.printBatchOutput = printBatchOutput;
+exports.printError = printError;
+exports.printOutput = printOutput;
+/// Docs: [Grok Dapi](/docs/plans/grok-dapi/)
+
+function printOutput(data, format) {
+  if (data === null || data === undefined) {
+    if (format !== 'quiet') console.log('(empty)');
+    return;
+  }
+  const rows = Array.isArray(data) ? data : [data];
+  switch (format) {
+    case 'json':
+      console.log(JSON.stringify(data, null, 2));
+      break;
+    case 'csv':
+      printCsv(rows);
+      break;
+    case 'quiet':
+      for (const row of rows) {
+        if (typeof row === 'object' && row !== null) console.log(row.id ?? row.name ?? JSON.stringify(row));else console.log(row);
+      }
+      break;
+    default:
+      printTable(rows);
+  }
+}
+function cellStr(v) {
+  if (v === null || v === undefined) return '';
+  if (typeof v === 'object') {
+    if (v.name) return v.name;
+    if (v.id) return v.id;
+    return JSON.stringify(v).slice(0, 40);
+  }
+  return String(v);
+}
+function printTable(rows) {
+  if (!rows.length) {
+    console.log('(no results)');
+    return;
+  }
+  if (typeof rows[0] !== 'object' || rows[0] === null) {
+    for (const r of rows) console.log(r);
+    return;
+  }
+  const keys = getKeys(rows);
+  const widths = {};
+  for (const k of keys) widths[k] = k.length;
+  for (const row of rows) for (const k of keys) widths[k] = Math.max(widths[k], cellStr(row[k]).length);
+  const header = keys.map(k => k.padEnd(widths[k])).join('  ');
+  const sep = keys.map(k => '-'.repeat(widths[k])).join('  ');
+  console.log(header);
+  console.log(sep);
+  for (const row of rows) console.log(keys.map(k => cellStr(row[k]).padEnd(widths[k])).join('  '));
+}
+function printCsv(rows) {
+  if (!rows.length) return;
+  if (typeof rows[0] !== 'object' || rows[0] === null) {
+    for (const r of rows) console.log(csvCell(String(r)));
+    return;
+  }
+  const keys = getKeys(rows);
+  console.log(keys.map(csvCell).join(','));
+  for (const row of rows) console.log(keys.map(k => csvCell(cellStr(row[k]))).join(','));
+}
+function csvCell(s) {
+  return s.includes(',') || s.includes('"') || s.includes('\n') ? `"${s.replace(/"/g, '""')}"` : s;
+}
+function getKeys(rows) {
+  const seen = new Set();
+  const keys = [];
+  for (const row of rows) for (const k of Object.keys(row)) if (!seen.has(k)) {
+    seen.add(k);
+    keys.push(k);
+  }
+  return keys.slice(0, 12);
+}
+function printBatchOutput(response, format) {
+  if (format === 'json') {
+    console.log(JSON.stringify(response, null, 2));
+    return;
+  }
+  if (format === 'quiet') {
+    const s = response.summary;
+    console.log(`${s.total} total: ${s.succeeded} succeeded, ${s.failed} failed, ${s.partial} partial, ${s.skipped} skipped`);
+    return;
+  }
+
+  // table and csv: print summary then per-op rows
+  const s = response.summary;
+  console.log(`Summary: ${s.total} total  ${s.succeeded} succeeded  ${s.failed} failed  ${s.partial} partial  ${s.skipped} skipped`);
+  if (!response.results.length) return;
+  console.log('');
+  const rows = response.results.map(r => {
+    const brief = r.status === 'error' ? r.error?.error ?? 'error' : r.status === 'skipped' ? r.reason ?? 'skipped' : r.status === 'partial' ? `${r.summary?.succeeded}/${r.summary?.total} succeeded` : '';
+    return {
+      id: r.id ?? '',
+      action: r.action,
+      status: r.status,
+      detail: brief
+    };
+  });
+  if (format === 'csv') printCsv(rows);else printTable(rows);
+
+  // Write per-op errors to stderr
+  const errors = response.results.filter(r => r.status === 'error');
+  if (errors.length) process.stderr.write(JSON.stringify(errors.map(r => ({
+    id: r.id,
+    action: r.action,
+    error: r.error
+  })), null, 2) + '\n');
+}
+function printError(err) {
+  const apiErr = err?.apiError;
+  const out = apiErr ?? {
+    error: String(err?.message ?? err)
+  };
+  process.stderr.write(JSON.stringify(out, null, 2) + '\n');
+}

package/package-template/package.json

@@ -1,6 +1,6 @@
 {
   "name": "#{PACKAGE_NAME_LOWERCASE}",
-  "friendlyName": "#{PACKAGE_NAME}",
+  "friendlyName": "#{PACKAGE_FRIENDLY_NAME}",
   "version": "0.0.1",
   "description": "#{PACKAGE_NAME} package",
   "dependencies": {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "datagrok-tools",
-  "version": "6.1.8",
+  "version": "6.1.10",
   "description": "Utility to upload and publish packages to Datagrok",
   "homepage": "https://github.com/datagrok-ai/public/tree/master/tools#readme",
   "dependencies": {
@@ -32,7 +32,11 @@
     "prepublishOnly": "babel bin --extensions .ts -d bin",
     "babel": "babel bin --extensions .ts -d bin",
     "build": "babel bin --extensions .ts -d bin",
-    "debug-source-map": "babel bin --extensions .ts -d bin --source-maps true"
+    "debug-source-map": "babel bin --extensions .ts -d bin --source-maps true",
+    "test": "vitest run --project unit",
+    "test:watch": "vitest --project unit",
+    "test:integration": "vitest run --project integration",
+    "test:all": "vitest run"
   },
   "bin": {
     "datagrok-upload": "./bin/_deprecated/upload.js",
@@ -69,13 +73,14 @@
     "@types/ignore-walk": "^4.0.3",
     "@types/inquirer": "^8.2.10",
     "@types/js-yaml": "^4.0.9",
-    "@types/node": "^16.18.70",
+    "@types/node": "^18.0.0",
     "@types/papaparse": "^5.3.15",
     "@typescript-eslint/eslint-plugin": "^5.62.0",
     "@typescript-eslint/parser": "^5.62.0",
     "eslint": "^8.56.0",
     "eslint-config-google": "^0.14.0",
     "typescript": "^5.3.3",
+    "vitest": "^3.2.4",
     "webpack": "^5.89.0",
     "webpack-cli": "^5.1.4"
   }

package/vitest.config.ts

@@ -0,0 +1,25 @@
+import {defineConfig} from 'vitest/config';
+
+export default defineConfig({
+  test: {
+    projects: [
+      {
+        test: {
+          name: 'unit',
+          environment: 'node',
+          include: ['bin/**/*.test.ts'],
+          exclude: ['bin/**/*.integration.test.ts'],
+        },
+      },
+      {
+        test: {
+          name: 'integration',
+          environment: 'node',
+          include: ['bin/**/*.integration.test.ts'],
+          testTimeout: 30_000,
+          hookTimeout: 30_000,
+        },
+      },
+    ],
+  },
+});