datagrok-tools 6.5.6 → 6.5.8

package/bin/utils/playwright-runner.js

@@ -82,6 +82,49 @@ function rowsToCsv(rows) {
     data: rows.map(r => header.map(h => r[h]))
   });
 }
+function writePlaywrightCsv(pkgDir, csv) {
+  // Persist a Playwright-only CSV so the pipeline can ship it to the Datlas
+  // 'playwright' bucket, separate from the merged Puppeteer+Playwright
+  // test-report.csv that feeds the legacy 'package' bucket and JUnit.
+  try {
+    _fs.default.writeFileSync(_path.default.join(pkgDir, 'test-report-playwright.csv'), csv, 'utf-8');
+  } catch (e) {
+    color.warn(`Playwright: failed to write test-report-playwright.csv: ${e.message || e}`);
+  }
+}
+
+// A pass that dies before running specs still has to report a failure. Returning an empty CSV
+// let the caller drop it whenever the Puppeteer pass had rows, so a suite that never launched
+// (a config that fails to load, an unreachable host) was reported as a clean run.
+function runnerFailure(pkgDir, args, message) {
+  const rows = [{
+    date: new Date().toISOString(),
+    category: 'playwright',
+    name: 'Playwright suite',
+    success: false,
+    result: message,
+    ms: 0,
+    skipped: false,
+    logs: '',
+    owner: '',
+    package: process.env.TARGET_PACKAGE || args.package || '',
+    widgetsDifference: '',
+    flaking: false
+  }];
+  const csv = rowsToCsv(rows);
+  writePlaywrightCsv(pkgDir, csv);
+  color.error(`Playwright: ${message}`);
+  return {
+    failed: true,
+    passedAmount: 0,
+    failedAmount: 1,
+    skippedAmount: 0,
+    verbosePassed: '',
+    verboseSkipped: '',
+    verboseFailed: `Playwright: ${message}\n`,
+    csv: csv
+  };
+}
 async function runPlaywrightTests(pkgDir, testDir, args, hostKey) {
   const empty = {
     failed: false,
@@ -101,29 +144,17 @@ async function runPlaywrightTests(pkgDir, testDir, args, hostKey) {
       key
     } = testUtils.getDevKey(hostKey));
   } catch (e) {
-    color.error(`Playwright: cannot resolve host '${hostKey}': ${e.message || e}`);
-    return {
-      ...empty,
-      failed: true,
-      failedAmount: 1,
-      verboseFailed: `Playwright: ${e.message || e}\n`
-    };
+    return runnerFailure(pkgDir, args, `cannot resolve host '${hostKey}': ${e.message || e}`);
   }
   let token;
   try {
     token = await testUtils.getToken(url, key);
   } catch (e) {
-    color.error(`Playwright: cannot exchange dev key for token: ${e.message || e}`);
-    return {
-      ...empty,
-      failed: true,
-      failedAmount: 1,
-      verboseFailed: `Playwright: ${e.message || e}\n`
-    };
+    return runnerFailure(pkgDir, args, `cannot exchange dev key for token: ${e.message || e}`);
   }
   let webUrl;
   try {
-    webUrl = await testUtils.getWebUrl(url, token);
+    webUrl = process.env.DATAGROK_WEB_URL || (await testUtils.getWebUrl(url, token));
     if (webUrl.endsWith('/')) webUrl = webUrl.slice(0, -1);
   } catch {
     webUrl = url.replace(/\/api\/?$/, '');
@@ -137,15 +168,7 @@ async function runPlaywrightTests(pkgDir, testDir, args, hostKey) {
     }
   }
   const configPath = _path.default.join(testDir, 'playwright.config.ts');
-  if (!_fs.default.existsSync(configPath)) {
-    color.error(`Playwright: ${configPath} not found.`);
-    return {
-      ...empty,
-      failed: true,
-      failedAmount: 1,
-      verboseFailed: 'Playwright: missing playwright.config.ts\n'
-    };
-  }
+  if (!_fs.default.existsSync(configPath)) return runnerFailure(pkgDir, args, `${configPath} not found`);
   const reportFile = _path.default.join(pkgDir, 'test-playwright-report.json');
   if (_fs.default.existsSync(reportFile)) _fs.default.unlinkSync(reportFile);
   const cliArgs = ['--no-install', 'playwright', 'test', `--config=${configPath}`];
@@ -168,6 +191,8 @@ async function runPlaywrightTests(pkgDir, testDir, args, hostKey) {
     ...process.env,
     DATAGROK_URL: webUrl,
     DATAGROK_AUTH_TOKEN: token,
+    DATAGROK_API_URL: url,
+    DATAGROK_DEV_KEY: key,
     PLAYWRIGHT_JSON_OUTPUT_NAME: reportFile
   };
   if (token2) env.DATAGROK_AUTH_TOKEN_2 = token2;
@@ -219,20 +244,18 @@ async function runPlaywrightTests(pkgDir, testDir, args, hostKey) {
     } catch {/* ignore */}
   }
   if (!report) {
-    color.error('Playwright: no JSON report produced.');
     const tail = Buffer.concat(stderrChunks).toString('utf-8').slice(-2000);
-    return {
-      ...empty,
-      failed: true,
-      failedAmount: 1,
-      verboseFailed: `Playwright: no JSON report. stderr tail:\n${tail}\n`
-    };
+    return runnerFailure(pkgDir, args, `no JSON report produced — the suite never ran. stderr tail:\n${tail}`);
   }
   const pkgJson = JSON.parse(_fs.default.readFileSync(_path.default.join(pkgDir, 'package.json'), 'utf-8'));
   const owner = pkgJson.author && (pkgJson.author.email || pkgJson.author) || '';
   const pkgName = process.env.TARGET_PACKAGE || args.package || pkgJson.name || '';
   const rows = [];
   flattenSuites(report.suites, testDir, pkgName, typeof owner === 'string' ? owner : '', args.verbose === true, rows);
+
+  // A report with no specs is the same non-event as no report at all: the config loaded but
+  // collection found nothing, or died on a spec that cannot be imported.
+  if (rows.length === 0) return runnerFailure(pkgDir, args, 'the run produced a report with no specs — check the spec imports and testMatch');
   let passedAmount = 0;
   let failedAmount = 0;
   let skippedAmount = 0;
@@ -253,14 +276,7 @@ async function runPlaywrightTests(pkgDir, testDir, args, hostKey) {
     }
   }
   const csv = rowsToCsv(rows);
-  // Persist a Playwright-only CSV so the pipeline can ship it to the Datlas
-  // 'playwright' bucket, separate from the merged Puppeteer+Playwright
-  // test-report.csv that feeds the legacy 'package' bucket and JUnit.
-  try {
-    _fs.default.writeFileSync(_path.default.join(pkgDir, 'test-report-playwright.csv'), csv, 'utf-8');
-  } catch (e) {
-    color.warn(`Playwright: failed to write test-report-playwright.csv: ${e.message || e}`);
-  }
+  writePlaywrightCsv(pkgDir, csv);
   return {
     failed: failedAmount > 0,
     passedAmount: passedAmount,

package/bin/utils/server-client.js

@@ -6,10 +6,23 @@ Object.defineProperty(exports, "__esModule", {
 exports.createClient = createClient;
 var _nodeDapi = require("./node-dapi");
 var _testUtils = require("./test-utils");
-async function createClient(hostArg) {
+/**
+ * `--admin` asks the server for an admin session, which lifts the permission filter for this run:
+ * without it a stand-wide pull sees only what the key's own account can, and content in other
+ * people's spaces is invisible. The server authorises it (`START_ADMIN_SESSION`) and signs the
+ * flag into the token, and a dev key mints a fresh session per invocation, so it cannot outlive
+ * the command or reach another session.
+ */
+async function createClient(hostArg, admin = false) {
   const {
     url,
     key
   } = (0, _testUtils.getDevKey)(hostArg ?? '');
-  return _nodeDapi.NodeApiClient.login(url, key);
+  const client = await _nodeDapi.NodeApiClient.login(url, key);
+  if (!admin) return client;
+  const token = (await client.post('/users/sessions/current/admin'))?.token;
+  if (!token) throw new Error(`${url} refused an admin session — the account behind this key cannot start one`);
+  client.token = token;
+  client.adminMode = true;
+  return client;
 }

package/bin/utils/server-output.js

@@ -9,8 +9,16 @@ exports.getKeys = getKeys;
 exports.printBatchOutput = printBatchOutput;
 exports.printError = printError;
 exports.printOutput = printOutput;
+exports.progressReporter = progressReporter;
+exports.setOutputFormat = setOutputFormat;
 /// Docs: [Grok Dapi](/docs/plans/grok-dapi/)
 
+let errorFormat = 'table';
+
+/** Set once per run so every `printError` call site reports in the requested format. */
+function setOutputFormat(format) {
+  errorFormat = format;
+}
 function printOutput(data, format) {
   if (data === null || data === undefined) {
     if (format !== 'quiet') console.log('(empty)');
@@ -118,10 +126,63 @@ function printBatchOutput(response, format) {
     error: r.error
   })), null, 2) + '\n');
 }
-function printError(err) {
+function printError(err, opts = {}) {
   const apiErr = err?.apiError;
-  const out = apiErr ?? {
-    error: String(err?.message ?? err)
+  const status = apiErr?.errorCode;
+  const base = apiErr?.error ?? String(err?.message ?? err);
+  const message = typeof status === 'number' && status >= 400 && !base.includes(String(status)) ? `${base} (HTTP ${status})` : base;
+  if (errorFormat === 'json') {
+    process.stderr.write(JSON.stringify({
+      ...apiErr,
+      error: message
+    }) + '\n');
+    return;
+  }
+  process.stderr.write(`${message}\n`);
+  printErrorDetails(apiErr?.body);
+  if (opts.verbose && apiErr?.stackTrace) process.stderr.write(apiErr.stackTrace + '\n');
+}
+
+/** Structured fields of a domain error envelope: per-row validation errors, manifest errors, a plan awaiting confirmation. */
+function printErrorDetails(body) {
+  if (!body || typeof body !== 'object') return;
+  for (const r of Array.isArray(body.rows) ? body.rows : []) for (const e of Array.isArray(r?.errors) ? r.errors : []) process.stderr.write(`  row ${r.index ?? ''}${e?.column ? ` ${e.column}` : ''}: ${e?.message ?? e?.code ?? ''}\n`);
+  for (const e of Array.isArray(body.errors) ? body.errors : []) process.stderr.write(`  ${typeof e === 'string' ? e : e?.message ?? JSON.stringify(e)}\n`);
+  if (body.plan) process.stderr.write(JSON.stringify(body.plan, null, 2) + '\n');
+}
+
+/**
+ * Progress for the long migration walks. A whole-stand pull is minutes of network waiting with
+ * nothing to show for it, and silence is indistinguishable from a hang. Goes to stderr, so it
+ * shows even under `--output json` without disturbing the document, and rewrites a single line
+ * on a terminal.
+ */
+function progressReporter(quiet = false) {
+  if (quiet) return () => {};
+  const tty = process.stderr.isTTY;
+  let last = '';
+  let drawn = 0;
+  return (stage, done, total) => {
+    const count = done === undefined ? '' : total === undefined ? ` ${done}` : ` ${done}/${total}`;
+    const line = `${stage}${count}`;
+    if (!tty) {
+      // A log cannot be rewritten in place, so it gets a line every so often rather than one per
+      // item — enough to show the run is alive, few enough to stay readable.
+      if (line !== last && (done === undefined || done === total || done % 250 === 0)) process.stderr.write(`${line}\n`);
+      last = line;
+      return;
+    }
+    // Redrawing on every item writes megabytes a minute into anything that is not really a
+    // terminal — a detached run produced a 7 GB file of carriage returns. The eye cannot
+    // follow faster than this anyway.
+    const finished = done !== undefined && done === total;
+    if (!finished && Date.now() - drawn < 100) return;
+    drawn = Date.now();
+    process.stderr.write(`\r${' '.repeat(last.length)}\r${line}`);
+    last = line;
+    if (finished) {
+      process.stderr.write('\n');
+      last = '';
+    }
   };
-  process.stderr.write(JSON.stringify(out, null, 2) + '\n');
 }

package/bin/utils/test-utils.js

@@ -127,7 +127,7 @@ function getDevKey(hostKey) {
   let key = '';
   let url = '';
   try {
-    let url = new URL(host).href;
+    url = new URL(host).href;
     if (url.endsWith('/')) url = url.slice(0, -1);
     if (url in urls) key = config['servers'][urls[url]]['key'];
   } catch (error) {

package/domain-schema.schema.json

@@ -77,12 +77,12 @@
         "promotion": {
           "enum": ["lazy", "eager"],
           "default": "lazy",
-          "description": "Row mode only. lazy: entities row created on first share; eager: at insert."
+          "description": "Row mode only. lazy: entities row created on first share; eager: at insert, with the author granted View/Edit/Delete/Share."
         },
         "defaultRowVisibility": {
           "enum": ["table", "none"],
           "default": "table",
-          "description": "Row/master modes: whether table-level View shows unshared rows."
+          "description": "Row/master modes: whether table-level View shows unshared rows. Their author always sees, edits and shares them; none hides them from everyone else."
         },
         "delegate": {
           "$ref": "#/definitions/identifier",
@@ -138,6 +138,29 @@
           "items": {"$ref": "#/definitions/identifier"},
           "description": "Property schemas contributing jsonb keys to this table."
         },
+        "constraints": {
+          "type": "object",
+          "description": "Table-level CHECK constraints keyed by constraint name. The expression may reference this table's relational columns, literals, comparison/arithmetic operators and a whitelist of keywords and immutable functions; it is normalized before it reaches the DDL.",
+          "propertyNames": {"$ref": "#/definitions/identifier"},
+          "additionalProperties": {
+            "type": "object",
+            "required": ["check"],
+            "additionalProperties": false,
+            "properties": {
+              "check": {"type": "string"}
+            }
+          }
+        },
+        "grants": {
+          "type": "object",
+          "description": "Permissions granted on the table entity at deploy time, keyed by group name (a platform group such as 'All users' or 'Developers'); each lists the permissions to grant.",
+          "minProperties": 1,
+          "additionalProperties": {
+            "type": "array",
+            "minItems": 1,
+            "items": {"enum": ["view", "edit", "delete"]}
+          }
+        },
         "relations": {
           "type": "object",
           "description": "Many-to-many relations of this table keyed by relation name: each links the table to a target table through a junction table. The name shares one namespace with the columns (it is what expand and filter paths address), so it may not collide with a column of this table. Declaration order is preserved and is the order every UI surface lays the relations out in.",
@@ -211,8 +234,8 @@
       "additionalProperties": false,
       "properties": {
         "type": {
-          "enum": ["string", "int", "float", "bool", "datetime", "string_list", "ref", "user", "group", "file"],
-          "description": "Property type; ref = FK to another table in this manifest; user/group = FK to the core users/groups tables with the matching semantic type; file = a file://<connection>/<path> string into platform file storage (semantic type File)."
+          "enum": ["string", "int", "float", "bool", "datetime", "string_list", "ref", "file", "json", "user", "group"],
+          "description": "Property type; ref = FK to another table in this manifest; user/group = aliases of ref Core.users / ref Core.groups (semantic type User / Group); file = a file://<connection>/<path> string into platform file storage (semantic type File); json = a jsonb-backed opaque object (filters, sorts, facets and aggregates refuse it)."
         },
         "required": {"type": "boolean", "default": false},
         "unique": {
@@ -220,14 +243,42 @@
           "default": false,
           "description": "Unique among live (not soft-deleted) rows."
         },
+        "immutable": {
+          "type": "boolean",
+          "default": false,
+          "description": "Write-once: the first non-null value wins; a later change is refused."
+        },
+        "autoNumber": {
+          "description": "Auto-numbering (int columns only; implies immutable): a blank value on insert is assigned from a monotonic counter, a supplied value is kept (the counter catches up). true = one counter per table; {scope} = one counter per master row of the named ref column of this table. Changing it on an existing column (on, off, or a scope change) is a physical change that needs a migration.",
+          "oneOf": [
+            {"enum": [true]},
+            {
+              "type": "object",
+              "additionalProperties": false,
+              "properties": {
+                "scope": {
+                  "$ref": "#/definitions/identifier",
+                  "description": "A required ref column of this table; numbering restarts per referenced row."
+                },
+                "start": {
+                  "type": "integer",
+                  "minimum": 1,
+                  "default": 1,
+                  "description": "First number assigned by a fresh counter."
+                }
+              }
+            }
+          ]
+        },
         "isName": {
           "type": "boolean",
           "default": false,
           "description": "Marks the primary display-name column (string columns only; at most one per table). Its value titles cards, tooltips, and entity views, and becomes the friendly name of promoted rows. Convention fallback: a string column literally named 'name'."
         },
         "ref": {
-          "$ref": "#/definitions/identifier",
-          "description": "Target table name (type = ref only). Cross-plugin refs are not allowed."
+          "type": "string",
+          "pattern": "^([a-z][a-z0-9_]*|[A-Za-z][A-Za-z0-9_]*\\.[a-z][a-z0-9_]*)$",
+          "description": "Target table (type = ref only): a table of this schema, or the qualified '<Schema>.<table>' form for any registered table — a Core table such as 'Core.users' or 'Core.queries', or a table of another plugin's schema. The reference is a hard foreign key only for same-schema targets and Core.users/Core.groups; every other target is a soft reference (no FK constraint) that may dangle after the target is hard-deleted."
         },
         "onDelete": {
           "enum": ["cascade", "restrict", "setnull"],

package/package.json

@@ -1,6 +1,10 @@
 {
   "name": "datagrok-tools",
-  "version": "6.5.6",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/datagrok-ai/public.git"
+  },
+  "version": "6.5.8",
   "description": "Utility to upload and publish packages to Datagrok",
   "homepage": "https://github.com/datagrok-ai/public/tree/master/tools#readme",
   "dependencies": {
@@ -32,6 +36,7 @@
     "update:ivp-parser": "esbuild plugins/ivp-parser.entry.mjs --bundle --format=cjs --platform=node --alias:diff-grok=../libraries/compute-utils/node_modules/diff-grok --outfile=plugins/ivp-parser.bundle.cjs",
     "debug-source-map": "node build.js --source-maps",
     "test": "vitest run --project unit",
+    "test:server": "vitest run --project unit bin/__tests__/node-dapi bin/__tests__/server bin/__tests__/migrate",
     "test:watch": "vitest --project unit",
     "test:integration": "vitest run --project integration",
     "test:all": "vitest run"