datagrok-tools 6.7.8 → 6.7.10

package/bin/utils/node-dapi.js

@@ -3,13 +3,14 @@
 Object.defineProperty(exports, "__esModule", {
   value: true
 });
-exports.NodeUsersDataSource = exports.NodeTablesDataSource = exports.NodeSharesDataSource = exports.NodePackagesDataSource = exports.NodeHttpDataSource = exports.NodeGroupsDataSource = exports.NodeFuncsDataSource = exports.NodeFilesDataSource = exports.NodeDomainsDataSource = exports.NodeDapi = exports.NodeConnectionsDataSource = exports.NodeApiClient = exports.InternalDataSource = void 0;
+exports.UUID_RE = exports.NodeUsersDataSource = exports.NodeTablesDataSource = exports.NodeSharesDataSource = exports.NodePackagesDataSource = exports.NodeHttpDataSource = exports.NodeGroupsDataSource = exports.NodeFuncsDataSource = exports.NodeFilesDataSource = exports.NodeDomainsDataSource = exports.NodeDapi = exports.NodeConnectionsDataSource = exports.NodeApiClient = exports.InternalDataSource = void 0;
 exports.apiPath = apiPath;
 exports.buildQuery = buildQuery;
 exports.ensureBodyId = ensureBodyId;
 exports.mapPositionalParams = mapPositionalParams;
 exports.parseDomainAddress = parseDomainAddress;
 exports.throwIfApiError = throwIfApiError;
+exports.tryParseJson = tryParseJson;
 var _crypto = require("crypto");
 var _keypair = require("./keypair");
 /// Docs: [Grok Dapi](/docs/plans/grok-dapi/)
@@ -63,6 +64,17 @@ async function fetchOrRetry(url, opts, retriable, timeoutMs = setting('TIMEOUT',
 class NodeApiClient {
   /** Set by `createClient` when the run asked for an admin session, so a re-login restores it. */
   adminMode = false;
+  /**
+   * `--log`: one id for every request of the command, wherever the server runs it, and the logger settings
+   * its records are kept at until `GET /log/requests/<logId>` collects them.
+   */
+
+  get logHeaders() {
+    const headers = {};
+    if (this.logId) headers['datlas-log-id'] = this.logId;
+    if (this.logSpec) headers['datlas-log'] = this.logSpec;
+    return headers;
+  }
   constructor(baseUrl, token, devKey, privateKey) {
     this.baseUrl = baseUrl;
     this.token = token;
@@ -119,6 +131,7 @@ class NodeApiClient {
       headers: {
         'Authorization': this.token,
         'Content-Type': 'application/json',
+        ...this.logHeaders,
         ...headers
       }
     };
@@ -142,7 +155,7 @@ class NodeApiClient {
       return null;
     }
     const ct = res.headers.get('content-type') ?? '';
-    return throwIfApiError(ct.includes('application/json') ? await res.json() : await res.text());
+    return throwIfApiError(ct.includes('application/json') ? await res.json() : await res.text(), res.headers);
   }
   get(path) {
     return this.request('GET', path);
@@ -164,13 +177,14 @@ class NodeApiClient {
       method: 'POST',
       headers: {
         'Authorization': this.token,
-        'Content-Type': contentType
+        'Content-Type': contentType,
+        ...this.logHeaders
       },
       body: bytes
     }, false, BYTES_TIMEOUT_MS);
     if (!res.ok) await throwHttpError(res);
     const ct = res.headers.get('content-type') ?? '';
-    return throwIfApiError(ct.includes('application/json') ? await res.json() : await res.text());
+    return throwIfApiError(ct.includes('application/json') ? await res.json() : await res.text(), res.headers);
   }
 
   /** POST a JSON body and read the response as raw bytes (d42 query results). */
@@ -179,7 +193,8 @@ class NodeApiClient {
       method: 'POST',
       headers: {
         'Authorization': this.token,
-        'Content-Type': 'application/json'
+        'Content-Type': 'application/json',
+        ...this.logHeaders
       },
       body: JSON.stringify(body),
       signal: AbortSignal.timeout(BYTES_TIMEOUT_MS)
@@ -192,7 +207,8 @@ class NodeApiClient {
   async getBytes(path) {
     const res = await fetchOrRetry(`${this.baseUrl}${path}`, {
       headers: {
-        'Authorization': this.token
+        'Authorization': this.token,
+        ...this.logHeaders
       }
     }, true, BYTES_TIMEOUT_MS);
     if (!res.ok) await throwHttpError(res);
@@ -236,13 +252,15 @@ async function throwHttpError(res) {
  * a JSON string when the handler returned text. Every response passes through here so a
  * missing entity, a rejected save or an unknown function throws instead of printing as data.
  */
-function throwIfApiError(payload) {
+/** A user error answers HTTP 200 with the ApiError in the body and its status in the `api-error-code` header. */
+function throwIfApiError(payload, headers) {
   const parsed = typeof payload === 'string' ? tryParseJson(payload) : payload;
   if (parsed?.['#type'] === 'ApiError') {
+    const code = headers?.get('api-error-code');
     const err = {
       error: parsed.message ?? 'Request failed',
       source: 'Server',
-      errorCode: parsed.errorCode,
+      errorCode: parsed.errorCode ?? (code ? Number(code) : undefined),
       stackTrace: parsed.stackTrace,
       body: parsed
     };
@@ -388,7 +406,7 @@ class InternalDataSource {
   }
 }
 exports.InternalDataSource = InternalDataSource;
-const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+const UUID_RE = exports.UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
 class NodeGroupsDataSource extends NodeHttpDataSource {
   constructor(client) {
     super(client, 'groups', '/groups');
@@ -638,9 +656,11 @@ class NodeFuncsDataSource extends NodeHttpDataSource {
   constructor(client) {
     super(client, 'functions', '/log/funcs');
   }
-  async run(name, params) {
+
+  /** [debug] runs it as a debug call, as the query debugger does; its log is kept with the command's `--log`. */
+  async run(name, params, debug = false) {
     const normalizedName = name.replace(':', '.');
-    return this.client.post(`/public/v1/functions/${encodeURIComponent(normalizedName)}/call`, params ?? {});
+    return this.client.post(`/public/v1/functions/${encodeURIComponent(normalizedName)}/call${debug ? '?debug=true' : ''}`, params ?? {});
   }
 
   /** The public API has no DELETE for functions; scripts and queries go to their own routers. */
@@ -936,6 +956,10 @@ function parseDomainAddress(s, opts = {}) {
   if (opts.table === false && address.table) throw new Error(`'${str}' names a table; this command needs a schema`);
   return address;
 }
+/** Every literal a `/domains/{schema}/{table}/{id}` route registers at the id position, the dot
+ * segments a proxy folds away, and the u2 draft sentinel — the Dart `DomainRowKey.reserved` list. */
+const RESERVED_ROW_KEYS = ['access', 'aggregate', 'audit', 'batch', 'delete', 'facets', 'filters', 'query', 'update', 'version', 'watch', 'new', '.', '..'];
+
 /**
  * Client for entity-mapped domain tables (`/domains/...`, DomainsRouter). Rows are plain
  * JSON objects; the server validates, permission-checks and audits every write. Errors
@@ -950,6 +974,13 @@ class NodeDomainsDataSource {
     return `/domains/${encodeURIComponent(schema)}/${encodeURIComponent(table)}`;
   }
 
+  /** The row-id segment as the Dart `DomainRowKey` spells it (grok_shared domain_manifest_rules.dart):
+   * escaped, then percent-encoded once more so an external key's own `%2C` survives the proxy. */
+  row(schema, table, id) {
+    const escaped = RESERVED_ROW_KEYS.includes(id) || id.startsWith('~') ? `~${id}` : id;
+    return `${this.rows(schema, table)}/${encodeURIComponent(escaped)}`;
+  }
+
   /** Registered schemas with their tables (`GET /domains/schemas`); [text] is a smart filter. */
   schemas(text) {
     return this.client.get(`/domains/schemas${buildQuery({
@@ -1001,7 +1032,7 @@ class NodeDomainsDataSource {
     })}`);
   }
   rowAudit(schema, table, id) {
-    return this.client.get(`${this.rows(schema, table)}/${encodeURIComponent(id)}/audit`);
+    return this.client.get(`${this.row(schema, table, id)}/audit`);
   }
   grants(entityId) {
     return this.client.get(`/domains/grants/${encodeURIComponent(entityId)}`);
@@ -1053,7 +1084,7 @@ class NodeDomainsDataSource {
   /** One row, or null when it does not exist or is not visible (the server answers 404). */
   async getRow(schema, table, id) {
     try {
-      return await this.client.get(`${this.rows(schema, table)}/${encodeURIComponent(id)}`);
+      return await this.client.get(this.row(schema, table, id));
     } catch (err) {
       if (err?.apiError?.errorCode === 404) return null;
       throw err;
@@ -1073,10 +1104,10 @@ class NodeDomainsDataSource {
       values
     };
     if (version !== undefined) body.version = version;
-    return this.client.request('PATCH', `${this.rows(schema, table)}/${encodeURIComponent(id)}`, body);
+    return this.client.request('PATCH', this.row(schema, table, id), body);
   }
   deleteRow(schema, table, id) {
-    return this.client.del(`${this.rows(schema, table)}/${encodeURIComponent(id)}`);
+    return this.client.del(this.row(schema, table, id));
   }
 
   /** Soft-deletes up to [limit] (≤1000) matching rows in one transaction; `{deleted, hasMore}`. */
@@ -1138,6 +1169,9 @@ class NodeDapi {
   get reports() {
     return this.internal('/reports');
   }
+  get problems() {
+    return this.internal('/problems');
+  }
   get files() {
     return new NodeFilesDataSource(this.client);
   }
@@ -1153,9 +1187,14 @@ class NodeDapi {
   internal(route) {
     return new InternalDataSource(this.client, route);
   }
+
+  /** `/info/server`; [extended] adds the in-flight requests per endpoint (admins only). */
+  async serverStatus(extended = false) {
+    const raw = await this.client.get(extended ? '/info/server?extended=true' : '/info/server');
+    return typeof raw === 'string' ? JSON.parse(raw) : raw;
+  }
   async serverInfo() {
-    const raw = await this.client.get('/info/server');
-    const info = typeof raw === 'string' ? JSON.parse(raw) : raw;
+    const info = await this.serverStatus();
     return {
       version: info?.Version ?? '',
       commit: info?.Commit

package/bin/utils/server-client.js

@@ -4,6 +4,7 @@ Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.createClient = createClient;
+exports.logSpecOf = logSpecOf;
 var _nodeDapi = require("./node-dapi");
 var _keypair = require("./keypair");
 /**
@@ -13,11 +14,17 @@ var _keypair = require("./keypair");
  * flag into the token, and a dev key mints a fresh session per invocation, so it cannot outlive
  * the command or reach another session.
  */
-async function createClient(hostArg, admin = false) {
+/** `--log` alone is `verbose`. */
+function logSpecOf(argv) {
+  return argv.log === undefined || argv.log === false ? undefined : argv.log === true ? 'verbose' : String(argv.log);
+}
+async function createClient(hostArg, admin = false, log) {
   // Resolved from the alias the caller named, not from its URL: two aliases can point at the
   // same server, and only this side knows which one was asked for.
   const cred = (0, _keypair.getServerCredentials)(hostArg ?? '');
   const client = await _nodeDapi.NodeApiClient.login(cred.url, cred.key ?? '', cred.privateKey);
+  client.logId = log?.logId;
+  client.logSpec = log?.spec;
   if (!admin) return client;
   const token = (await client.post('/users/sessions/current/admin'))?.token;
   if (!token) throw new Error(`${cred.url} refused an admin session — the account behind this key cannot start one`);

package/bin/utils/test-utils.js

@@ -16,6 +16,7 @@ exports.getToken = getToken;
 exports.getWebUrl = getWebUrl;
 exports.getWebUrlFromPage = getWebUrlFromPage;
 exports.isPackageOnServer = isPackageOnServer;
+exports.isPackageU2Usage = isPackageU2Usage;
 exports.loadPackage = loadPackage;
 exports.loadPackages = loadPackages;
 exports.loadTestsList = loadTestsList;
@@ -159,6 +160,36 @@ function getDevKey(hostKey) {
     key
   };
 }
+const U2_USAGE = 'u2 usage:';
+/** The u2 usage mode a package built on u2 or ems is tested in: a misuse throws, unless `U2_STRICT` says otherwise
+ * (`report` logs each one as a `u2 usage:` line and the run does not fail on them). */
+const U2_MODE = process.env.U2_STRICT ?? 'throw';
+
+/** Whether a `u2 usage:` line the page logged fails the run: the package is tested with u2 strict. */
+function failsOnU2Usage() {
+  return U2_MODE !== 'report' && usesU2(process.cwd());
+}
+
+/** Whether a `u2 usage:` line is the tested package's: a stack frame it carries is in the package's bundle
+ * (`…/files/<package>/…`), or it carries none to tell. A line whose frames are all in other bundles is another
+ * package's misuse, reported but not failed on. */
+function isPackageU2Usage(line, packageName) {
+  const frames = line.match(/https?:\/\/[^\s()]+/g) ?? [];
+  const bundle = `/files/${packageName?.toLowerCase()}/`;
+  return packageName === undefined || frames.length === 0 || frames.some(f => f.toLowerCase().includes(bundle));
+}
+function usesU2(packageDir) {
+  try {
+    const json = JSON.parse(_fs.default.readFileSync(_path.default.join(packageDir, 'package.json'), 'utf-8'));
+    const deps = {
+      ...json.dependencies,
+      ...json.devDependencies
+    };
+    return '@datagrok-libraries/u2' in deps || '@datagrok-libraries/ems' in deps;
+  } catch {
+    return false;
+  }
+}
 function appendUrlParams(url, extraParams) {
   if (!extraParams) return url;
   return url + (url.includes('?') ? '&' : '?') + extraParams;
@@ -194,6 +225,9 @@ async function getBrowserPage(puppeteer, params = defaultLaunchParameters, urlPa
     height: 1080
   });
   page.setDefaultNavigationTimeout(0);
+  if (usesU2(process.cwd())) await page.evaluateOnNewDocument(mode => {
+    globalThis.__U2_STRICT__ = mode;
+  }, U2_MODE);
   await page.goto(`${url}/oauth/`);
   await page.browser().setCookie({
     name: 'auth',
@@ -537,6 +571,16 @@ function printBrowsersResult(browserResult, verbose = false) {
     console.log('Skipped tests: ' + (browserResult?.skippedAmount ?? 0));
     console.log('Failed tests:  ' + (browserResult?.failedAmount ?? 0));
   }
+  const own = browserResult.u2Usage?.filter(l => isPackageU2Usage(l, process.env.TARGET_PACKAGE)) ?? [];
+  const others = browserResult.u2Usage?.filter(l => !own.includes(l)) ?? [];
+  if (own.length) {
+    color.warn(failsOnU2Usage() ? `${own.length} u2 usage error(s) caught on the page (the run fails):` : `${own.length} u2 usage report(s), in report mode (the run does not fail on them):`);
+    for (const line of own) color.warn(`  ${line}`);
+  }
+  if (others.length) {
+    color.warn(`${others.length} u2 usage line(s) from other packages' code (the run does not fail on them):`);
+    for (const line of others) color.warn(`  ${line}`);
+  }
   if (browserResult.failed) {
     if (browserResult.verboseFailed === 'Package not found') color.fail('Tests not found');else color.fail('Tests failed.');
   } else color.success('Tests passed.');
@@ -860,6 +904,11 @@ async function runBrowser(testExecutionData, browserOptions, browsersId, testInv
     // Subscribe to page console events for modern output formatting
     // On retry, old listeners were removed so we need to re-attach
     page.on('console', msg => reporter.onLine(msg.text()));
+    const u2Usage = [];
+    page.on('console', msg => {
+      const text = msg.text();
+      if (text.includes(U2_USAGE) && !u2Usage.includes(text)) u2Usage.push(text);
+    });
     const testingResults = await page.evaluate((testData, options) => {
       if (options.benchmark) window.DG.Test.isInBenchmark = true;
       if (options.reproduce) window.DG.Test.isReproducing = true;
@@ -918,12 +967,14 @@ async function runBrowser(testExecutionData, browserOptions, browsersId, testInv
     // Return browser session for potential reuse
     return {
       ...testingResults,
+      failed: testingResults.failed || failsOnU2Usage() && u2Usage.some(l => isPackageU2Usage(l, process.env.TARGET_PACKAGE)),
       browserSession: browserOptions.keepBrowserOpen ? {
         browser,
         page,
         webUrl
       } : undefined,
-      modernOutput: reporter.modernOutput
+      modernOutput: reporter.modernOutput,
+      u2Usage
     };
   }, testInvocationTimeout);
 }
@@ -938,7 +989,8 @@ async function mergeBrowsersResults(browsersResults) {
     failedAmount: browsersResults[0].failedAmount,
     csv: browsersResults[0].csv,
     error: '',
-    modernOutput: browsersResults.some(r => r.modernOutput)
+    modernOutput: browsersResults.some(r => r.modernOutput),
+    u2Usage: [...new Set(browsersResults.flatMap(r => r.u2Usage ?? []))]
   };
   for (const browsersResult of browsersResults) {
     if (mergedResult.csv === browsersResult.csv) continue;

package/domain-schema.schema.json

@@ -16,6 +16,36 @@
       "type": "string",
       "description": "Manifest version (semver recommended). Recorded in the registry on deploy."
     },
+    "storage": {
+      "type": "object",
+      "description": "Where the rows live. Absent = domain: the platform creates and owns the tables. external: the tables already exist in the database of the named platform connection and are read through it (no row security, soft delete, audit, relations, constraints, property schemas or extensions; read-only unless 'writable'); every table then declares its remote primary key as businessKey, and 'table'/'column' carry the remote names when they differ from the declared ones.",
+      "required": ["kind"],
+      "additionalProperties": false,
+      "properties": {
+        "kind": {"enum": ["domain", "external"]},
+        "connection": {
+          "type": "string",
+          "minLength": 1,
+          "description": "external: the qualified name of the platform connection to read through ('Package:Connection'); required."
+        },
+        "schema": {
+          "$ref": "#/definitions/remoteIdentifier",
+          "description": "external: the schema inside the remote database (e.g. 'public', 'dbo'); required."
+        },
+        "catalog": {
+          "$ref": "#/definitions/remoteIdentifier",
+          "description": "external: the catalog/database, for targets that have one."
+        },
+        "writable": {
+          "type": "boolean",
+          "default": false,
+          "description": "external: rows may be inserted, patched and deleted through the connection (one remote transaction per request, keyed by businessKey; on ClickHouse, which commits every statement, a request is one multi-row INSERT and patch/delete/upsert are refused). The publisher must hold Query, AddRows, ChangeValues and RemoveRows on the connection; a table opts out with its own 'writable': false."
+        }
+      },
+      "if": {"properties": {"kind": {"const": "external"}}},
+      "then": {"required": ["connection", "schema"]},
+      "else": {"properties": {"connection": false, "schema": false, "catalog": false, "writable": false}}
+    },
     "extensible": {
       "type": "object",
       "description": "Opt-in for user extensions of this schema. Users holding the Extend permission on the schema entity may then add their own tables; per-table column extension is opted in with the table's own 'extensible' flag. User objects are retained across plugin upgrades and never appear in generated typed clients.",
@@ -64,11 +94,26 @@
         {"not": {"type": "string", "pattern": "^x_"}}
       ]
     },
+    "remoteIdentifier": {
+      "description": "A name inside the remote database, verbatim (spaces allowed, the connector quotes it); it may not contain brackets, quotes, backslashes or dots.",
+      "type": "string",
+      "minLength": 1,
+      "maxLength": 256,
+      "pattern": "^[^\\[\\]\"'`\\\\.]+$"
+    },
     "table": {
       "type": "object",
       "required": ["columns"],
       "additionalProperties": false,
       "properties": {
+        "table": {
+          "$ref": "#/definitions/remoteIdentifier",
+          "description": "External storage only: the remote table's name when it differs from the declared one (e.g. 'Order Details')."
+        },
+        "writable": {
+          "type": "boolean",
+          "description": "Writable external storage only: false keeps this table read-only while the rest of the binding accepts writes."
+        },
         "securityMode": {
           "enum": ["table", "master", "row"],
           "default": "table",
@@ -263,7 +308,11 @@
       "properties": {
         "type": {
           "enum": ["string", "int", "float", "bool", "datetime", "string_list", "ref", "file", "json", "user", "group"],
-          "description": "Property type; ref = FK to another table in this manifest; user/group = aliases of ref Core.users / ref Core.groups (semantic type User / Group); file = a file://<connection>/<path> string into platform file storage (semantic type File); json = a jsonb-backed opaque object (filters, sorts, facets and aggregates refuse it)."
+          "description": "Property type; ref = FK to another table in this manifest; user/group = aliases of ref Core.users / ref Core.groups (semantic type User / Group); file = a file://<connection>/<path> string into platform file storage (semantic type File); json = a jsonb-backed opaque object (filters, sorts, facets and aggregates refuse it). An external storage admits string, int, float, bool, datetime and same-manifest ref only."
+        },
+        "column": {
+          "$ref": "#/definitions/remoteIdentifier",
+          "description": "External storage only: the remote column's name when it differs from the declared one (e.g. 'OrderID')."
         },
         "required": {"type": "boolean", "default": false},
         "unique": {

package/entity-template/domain-app-imports.js

@@ -1,4 +1,4 @@
-import {domains} from '@datagrok-libraries/u2/src/dg/index.js';
+import {domains} from '@datagrok-libraries/ems/src/dg';
 
 // the skins of everything the u2 domain stack renders
-import '@datagrok-libraries/u2/src/dg/domain/styles.js';
+import '@datagrok-libraries/ems/src/dg/styles.js';

package/package.json

@@ -4,9 +4,12 @@
     "type": "git",
     "url": "https://github.com/datagrok-ai/public.git"
   },
-  "version": "6.7.8",
+  "version": "6.7.10",
   "description": "Utility to upload and publish packages to Datagrok",
   "homepage": "https://github.com/datagrok-ai/public/tree/master/tools#readme",
+  "engines": {
+    "node": ">=22.12.0"
+  },
   "dependencies": {
     "@babel/parser": "^7.29.7",
     "@babel/runtime": "^7.29.7",
@@ -24,7 +27,7 @@
     "minimist": "^1.2.8",
     "node-fetch": "^2.7.0",
     "papaparse": "^5.5.3",
-    "puppeteer": "^24.15.0",
+    "puppeteer": "^25.14.0",
     "ts-morph": "^28.0.0",
     "@babel/generator": "^7.29.0",
     "datagrok-api": "^1.27.12"

package/plugins/func-gen-plugin.js

@@ -202,8 +202,8 @@ class FuncGeneratorPlugin {
       this._checkPackageFileForDecoratorsExport(packageFilePath);
       // Uncommment to add obvious import/export
       // this._writeToPackageFile(packageFilePath, genImports, genExports);
+      api({_: ['api']});
     });
-    api({_: ['api']});
   }
 
   _addNodeData(

package/vitest.config.mts

@@ -1,9 +1,12 @@
 import {defineConfig} from 'vitest/config';
 
 export default defineConfig({
+  // Babel writes each .js beside its .ts in bin/: tests must load the sources, and one copy of each module.
+  resolve: {extensions: ['.ts', '.mts', '.mjs', '.js', '.jsx', '.tsx', '.json']},
   test: {
     projects: [
       {
+        extends: true,
         test: {
           name: 'unit',
           environment: 'node',
@@ -12,6 +15,7 @@ export default defineConfig({
         },
       },
       {
+        extends: true,
         test: {
           name: 'integration',
           environment: 'node',