datagrok-tools 6.5.6 → 6.5.8

package/bin/commands/api.js

@@ -42,6 +42,11 @@ function tableProp(tableName) {
   return utils.snakeToCamelCase(pluralizeTableName(tableName), false);
 }
 const domainSchemaPath = _path.default.join(_path.default.dirname(_path.default.dirname(__dirname)), 'domain-schema.schema.json');
+/** The platform's own schema, whose tables a qualified ref may target (`Core.users`, ...). */
+const CORE_SCHEMA = 'Core';
+/** The sealed Core declaration (a copy of core/server/db/snapshots/Core.json, pinned equal by the
+ * server's seal test) — what a `ref: <CORE_SCHEMA>.<table>` resolves against at build time. */
+const coreDeclarationPath = _path.default.join(_path.default.dirname(_path.default.dirname(__dirname)), `${CORE_SCHEMA}.json`);
 const domainSystemColumns = [['id', 'string'], ['version', 'number'], ['created_on', 'Dayjs'], ['updated_on', 'Dayjs'], ['author_id', 'string']];
 const domainTypeMap = {
   string: 'string',
@@ -51,10 +56,22 @@ const domainTypeMap = {
   datetime: 'Dayjs',
   string_list: 'string[]',
   ref: 'string',
-  user: 'string',
-  group: 'string',
-  file: 'string'
+  file: 'string',
+  json: '{[key: string]: any}'
 };
+
+/** `type: user` / `type: group` are aliases of a Core ref (the server's
+ * `DomainTableColumn.canonicalize`); codegen spells them the same way so both generate alike. */
+const coreRefAliases = {
+  user: `${CORE_SCHEMA}.users`,
+  group: `${CORE_SCHEMA}.groups`
+};
+
+/** The sealed declaration lists a table's columns and a property schema's columns as
+ * `[{name, ...}]`; manifests key them by name. */
+function columnsByName(columns) {
+  return Array.isArray(columns) ? Object.fromEntries(columns.map(c => [c.name, c])) : columns;
+}
 function normEol(s) {
   return s.replace(/\r\n/g, '\n');
 }
@@ -202,7 +219,7 @@ function generateDomainClients(packageDir = curDir, options) {
       for (const err of validateManifest.errors ?? []) color.error(`${relPath}: ${err.instancePath || '/'} ${err.message}`);
       return false;
     }
-    const code = generateDomainSchemaCode(manifest, relPath, emittedTypes);
+    const code = generateDomainSchemaCode(manifest, relPath, emittedTypes, packageDir);
     if (code == null) return false;
     parts.push(code);
     if (ui) uiParts.push(generateDomainUiCode(manifest, dbImports));
@@ -220,6 +237,26 @@ function generateDomainClients(packageDir = curDir, options) {
   }
   return true;
 }
+/** Subdirectory names of [dir], or none when it does not exist. */
+function subdirs(dir) {
+  return _fs.default.existsSync(dir) ? _fs.default.readdirSync(dir, {
+    withFileTypes: true
+  }).filter(e => e.isDirectory()).map(e => e.name) : [];
+}
+
+/** The `databases/<schema>/schema.json` manifest named [schemaName] among the package's
+ * installed dependencies (scoped packages included), or null. */
+function findDependencyManifest(packageDir, schemaName) {
+  const nm = _path.default.join(packageDir, 'node_modules');
+  const pkgDirs = subdirs(nm).flatMap(d => d.startsWith('@') ? subdirs(_path.default.join(nm, d)).map(s => _path.default.join(nm, d, s)) : [_path.default.join(nm, d)]);
+  for (const pkgDir of pkgDirs) for (const schemaDir of subdirs(_path.default.join(pkgDir, 'databases'))) {
+    const manifestPath = _path.default.join(pkgDir, 'databases', schemaDir, 'schema.json');
+    if (!_fs.default.existsSync(manifestPath)) continue;
+    const manifest = JSON.parse(_fs.default.readFileSync(manifestPath, 'utf8'));
+    if (manifest.name === schemaName) return manifest;
+  }
+  return null;
+}
 
 /** A resolved many-to-many relation of one table: the expand key and the target
  * table whose row ids its write-side link set carries. */
@@ -227,10 +264,12 @@ function generateDomainClients(packageDir = curDir, options) {
 /** Emits choices aliases, row/insert interfaces, column-name unions, expand maps, the
  * `<Schema>TransactionOp` union, and the lazy `<schema>Db` clients for one manifest.
  * Returns null on a semantic error (reported to the console). */
-function generateDomainSchemaCode(manifest, manifestPath, emittedTypes) {
+function generateDomainSchemaCode(manifest, manifestPath, emittedTypes, packageDir) {
   const decls = [];
   const systemColumnNames = new Set(domainSystemColumns.map(([name]) => name));
   const tableNames = Object.keys(manifest.tables);
+  // Keyed by table name; a qualified ref target ('Core.queries', 'grit.issue') lands here
+  // under its qualified key so the expand map can list its columns — it gets no accessor.
   const tableColumns = {};
   // Choices aliases are deduplicated by name: identical value sets share the first alias,
   // different sets fall back to a `<alias><PascalTable>` name (deterministic).
@@ -250,57 +289,112 @@ function generateDomainSchemaCode(manifest, manifestPath, emittedTypes) {
     return alias;
   };
 
-  // Pass 1: resolve every table's full column list (relational + property-schema columns).
-  for (const tableName of tableNames) {
-    const table = manifest.tables[tableName];
-    const typeName = utils.snakeToCamelCase(tableName);
-    if (emittedTypes.has(typeName)) {
-      color.error(`${manifestPath}: table '${tableName}' emits interface '${typeName}Row' ` + `already generated for another table (table names must be unique across the package's manifests)`);
-      return null;
+  // A qualified ref target, resolved from files: `Core.<table>` from the CLI's sealed Core
+  // declaration, any other schema from a dependency's databases/*/schema.json.
+  const externalManifests = new Map();
+  function loadExternalTable(ref, where) {
+    if (tableColumns[ref] != null) return true;
+    const [schemaName, tableName] = ref.split('.');
+    const core = schemaName === CORE_SCHEMA;
+    if (!externalManifests.has(schemaName)) externalManifests.set(schemaName, core ? _fs.default.existsSync(coreDeclarationPath) ? JSON.parse(_fs.default.readFileSync(coreDeclarationPath, 'utf8')) : null : findDependencyManifest(packageDir, schemaName));
+    const owner = externalManifests.get(schemaName);
+    if (owner == null) {
+      color.error(`${where} references '${ref}', but ` + (core ? `the sealed ${CORE_SCHEMA} declaration is missing: ${coreDeclarationPath}` : `no installed dependency declares schema '${schemaName}' — install the package that declares ` + `schema '${schemaName}' as a dependency`));
+      return false;
     }
-    emittedTypes.add(typeName);
+    if (owner.tables[tableName] == null) {
+      color.error(`${where} references '${ref}', but schema '${schemaName}' declares no table '${tableName}'`);
+      return false;
+    }
+    const columns = collectColumns(tableName, {
+      ...owner,
+      name: schemaName
+    }, true);
+    if (columns == null) return false;
+    tableColumns[ref] = columns;
+    return true;
+  }
+
+  // Pass 1: resolve every table's full column list (relational + property-schema columns).
+  // [owner] is the manifest the table comes from; an external (qualified-ref target) table
+  // skips the system-column check — its declaration may legitimately name one (Core.packages
+  // has `version`) — and its own refs are never followed (no nested expand).
+  function collectColumns(tableName, owner, external) {
+    const table = owner.tables[tableName];
+    const key = external ? `${owner.name}.${tableName}` : tableName;
     const columns = [];
-    const columnNames = new Set(systemColumnNames);
+    const columnNames = new Set(external ? [] : systemColumnNames);
     const addColumn = (name, column) => {
+      if (coreRefAliases[column.type] != null) {
+        column = {
+          ...column,
+          type: 'ref',
+          ref: coreRefAliases[column.type]
+        };
+        delete column.onDelete;
+      }
       if (columnNames.has(name)) {
-        color.error(systemColumnNames.has(name) ? `${manifestPath}: table '${tableName}' column '${name}' collides with a generated system column` : `${manifestPath}: table '${tableName}' declares duplicate column '${name}'`);
+        color.error(systemColumnNames.has(name) && !external ? `${manifestPath}: table '${key}' column '${name}' collides with a generated system column` : `${manifestPath}: table '${key}' declares duplicate column '${name}'`);
         return false;
       }
       columnNames.add(name);
       let tsType = domainTypeMap[column.type];
       if (Array.isArray(column.choices) && column.choices.length > 0) {
-        const alias = choicesAlias(tableName, name, column.choices);
+        const alias = choicesAlias(key.replace('.', '_'), name, column.choices);
         if (alias == null) return false;
         tsType = alias;
       }
+      let ref;
+      if (column.type === 'ref' && !external) {
+        if (column.ref == null) {
+          color.error(`${manifestPath}: table '${tableName}' column '${name}' is a ref column without a 'ref' target`);
+          return false;
+        }
+        if (column.ref.includes('.')) {
+          if (!loadExternalTable(column.ref, `${manifestPath}: table '${tableName}' column '${name}'`)) return false;
+          ref = column.ref;
+        } else if (manifest.tables[column.ref] != null) ref = column.ref;
+      }
       columns.push({
         name: name,
         rawType: column.type,
         tsType: tsType,
         insertType: column.type === 'datetime' ? 'Dayjs | string' : tsType,
         required: column.required === true,
-        ref: column.type === 'ref' && manifest.tables[column.ref] != null ? column.ref : undefined
+        autoNumber: column.autoNumber != null,
+        ref: ref
       });
       return true;
     };
-    for (const columnName of Object.keys(table.columns)) if (!addColumn(columnName, table.columns[columnName])) return null;
+    const declared = columnsByName(table.columns);
+    for (const columnName of Object.keys(declared)) if (!addColumn(columnName, declared[columnName])) return null;
     for (const schemaName of table.schemas ?? []) {
-      const props = manifest.propertySchemas?.[schemaName];
+      const props = columnsByName(owner.propertySchemas?.[schemaName]);
       if (props == null) {
-        color.error(`${manifestPath}: table '${tableName}' references unknown property schema '${schemaName}'`);
+        color.error(`${manifestPath}: table '${key}' references unknown property schema '${schemaName}'`);
         return null;
       }
       for (const propName of Object.keys(props)) if (!addColumn(propName, props[propName])) return null;
     }
+    return columns;
+  }
+  for (const tableName of tableNames) {
+    const typeName = utils.snakeToCamelCase(tableName);
+    if (emittedTypes.has(typeName)) {
+      color.error(`${manifestPath}: table '${tableName}' emits interface '${typeName}Row' ` + `already generated for another table (table names must be unique across the package's manifests)`);
+      return null;
+    }
+    emittedTypes.add(typeName);
+    const columns = collectColumns(tableName, manifest, false);
+    if (columns == null) return null;
     tableColumns[tableName] = columns;
   }
 
-  // Pass 1.5: resolve declared many-to-many relations. Same rules the server's
-  // manifest parser applies (manifest.dart 'relations'), so a manifest that
-  // generates here is one that deploys: via/target declared in this manifest,
-  // via distinct from owner and target, self-referential relations explicit,
-  // FK sides unique-or-explicit, and a junction business key covering both —
-  // that key is what keeps re-linking the same pair idempotent.
+  // Pass 1.5: declared many-to-many relations. Only what the generated code needs
+  // is checked here: the target table must be declared (its name lands in the
+  // generated doc comments) and the name must not collide with a column (relations
+  // and columns share one expand/filter namespace). Everything else about a
+  // relation (via, viaSelf/viaTarget, the junction business key) is the deploy's gate.
   const tableRelations = {};
   for (const tableName of tableNames) {
     const relations = [];
@@ -315,53 +409,10 @@ function generateDomainSchemaCode(manifest, manifestPath, emittedTypes) {
         color.error(`${where} collides with a column of '${tableName}' — relations and columns ` + `share one expand/filter namespace`);
         return null;
       }
-      if (manifest.tables[r.via] == null) {
-        color.error(`${where}: junction table '${r.via}' is not declared in this manifest`);
-        return null;
-      }
       if (manifest.tables[r.target] == null) {
         color.error(`${where}: target table '${r.target}' is not declared in this manifest`);
         return null;
       }
-      if (r.via === tableName || r.via === r.target) {
-        color.error(`${where}: junction table '${r.via}' must differ from the owner and the target table`);
-        return null;
-      }
-      if (r.target === tableName && (r.viaSelf == null || r.viaTarget == null)) {
-        color.error(`${where}: a self-referential relation must name both 'viaSelf' and 'viaTarget'`);
-        return null;
-      }
-      // One side of the junction: the declared column when explicit, otherwise
-      // the single ref column of `via` pointing at `to`.
-      const resolveSide = (key, explicit, to) => {
-        const candidates = tableColumns[r.via].filter(c => c.ref === to).map(c => c.name);
-        if (explicit != null) {
-          if (candidates.includes(explicit)) return explicit;
-          color.error(`${where}: '${explicit}' is not a ref column of junction table '${r.via}' ` + `targeting '${to}'`);
-          return null;
-        }
-        if (candidates.length === 0) {
-          color.error(`${where}: junction table '${r.via}' has no ref column targeting '${to}'`);
-          return null;
-        }
-        if (candidates.length > 1) {
-          color.error(`${where}: junction table '${r.via}' has more than one ref column targeting ` + `'${to}' (${candidates.join(', ')}) — declare '${key}' explicitly`);
-          return null;
-        }
-        return candidates[0];
-      };
-      const viaSelf = resolveSide('viaSelf', r.viaSelf, tableName);
-      const viaTarget = resolveSide('viaTarget', r.viaTarget, r.target);
-      if (viaSelf == null || viaTarget == null) return null;
-      if (viaSelf === viaTarget) {
-        color.error(`${where}: 'viaSelf' and 'viaTarget' must be different columns of '${r.via}'`);
-        return null;
-      }
-      const businessKey = manifest.tables[r.via].businessKey ?? [];
-      if (!businessKey.includes(viaSelf) || !businessKey.includes(viaTarget)) {
-        color.error(`${where}: junction table '${r.via}' must declare a 'businessKey' containing ` + `both '${viaSelf}' and '${viaTarget}', so linking the same pair twice stays idempotent`);
-        return null;
-      }
       relations.push({
         name: name,
         target: r.target
@@ -379,7 +430,7 @@ function generateDomainSchemaCode(manifest, manifestPath, emittedTypes) {
     const relations = tableRelations[tableName];
     const rowLines = [`/** Row of \`${manifest.name}.${tableName}\`. */`, `export interface ${typeName}Row {`];
     for (const [name, tsType] of domainSystemColumns) rowLines.push(`  ${name}: ${tsType};`);
-    for (const c of columns) rowLines.push(`  ${c.name}${c.required ? '' : '?'}: ${c.tsType};`);
+    for (const c of columns) rowLines.push(`  ${c.name}${c.required || c.autoNumber ? '' : '?'}: ${c.tsType};`);
     rowLines.push('}');
     decls.push(rowLines.join(sep));
     const insertLines = [`/** Insert payload for \`${manifest.name}.${tableName}\`. */`, `export interface ${typeName}Insert {`];

package/bin/commands/help.js

@@ -4,6 +4,7 @@ Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.help = void 0;
+var _server = require("./server");
 const HELP = `
 Usage: grok <command>
 
@@ -391,69 +392,6 @@ Examples:
 
 The instance name must match a server alias in ~/.grok/config.yaml.
 `;
-const HELP_SERVER = `
-Usage: grok server <entity> <verb> [args] [options]
-       grok s <entity> <verb> [args] [options]
-
-Manage a Datagrok server from the command line.
-
-Entities:
-  users, groups, functions, connections, queries, scripts, packages, reports, files, tables
-
-Verbs:
-  list      List entities
-  get       Get a single entity by ID or name
-  delete    Delete an entity by ID
-
-Special commands:
-  grok s functions run <Name:func(args)>             Call a function
-  grok s functions list [--type <t>] [--language <l>] [--package <p>] [--filter <expr>]
-                                                     Type: script|query|function|package; language applies to scripts
-  grok s files list [path] [-r]                      List files (recursive with -r)
-  grok s shares add <entity> <group>[,<group>...] [--access View|Edit]
-                                                     Share an entity with groups
-  grok s shares list <entity-id>                     List who an entity (UUID) is shared with
-  grok s users save --json user.json                 Create or update a user from JSON
-  grok s users block <id-or-login>                   Block a user from the platform
-  grok s users unblock <id-or-login>                 Unblock a previously blocked user
-  grok s groups save --json group.json [--save-relations]
-                                                     Create or update a group from JSON
-  grok s connections save --json conn.json [--save-credentials]
-                                                     Create or update a connection from JSON
-  grok s connections test <id-or-name>               Test connectivity of an existing connection
-  grok s connections test --json conn.json           Test connectivity of a connection defined in JSON
-  grok s tables upload <name> <file.csv>             Upload a CSV as a Datagrok table
-  grok s tables download <name-or-id> [-O <file>]    Download a table as CSV (stdout by default)
-  grok s raw <METHOD> <path>                         Hit any API endpoint
-  grok s describe <entity-type>                      Show entity JSON schema
-
-Options:
-  --host <alias|url>    Server alias from config or full URL
-  --output <format>     Output format: table (default), json, csv, quiet
-  --filter <text>       Smart filter expression
-  --limit <n>           Page size (default: 50)
-  --offset <n>          Start offset (default: 0)
-  -r, --recursive       Recursive (for files list)
-  --json <file>         Read function parameters from JSON file
-
-Examples:
-  grok s users list
-  grok s connections list --filter "PostgreSQL" --output json
-  grok s connections get <id>
-  grok s connections delete <id>
-  grok s connections save --json conn.json --save-credentials
-  grok s connections test "JohnDoe:MyConnection"
-  grok s connections test --json conn.json
-  grok s users save --json user.json
-  grok s groups save --json group.json --save-relations
-  grok s shares add "JohnDoe:MyConnection" Chemists,Admins --access Edit
-  grok s shares list <entity-uuid>
-  grok s functions run 'Chem:smilesToMw("ccc")'
-  grok s files list "System:AppData" -r
-  grok s raw GET /api/users/current
-  grok s describe connections
-  grok s users list --host dev
-`;
 const help = exports.help = {
   add: HELP_ADD,
   api: HELP_API,
@@ -471,7 +409,7 @@ const help = exports.help = {
   test: HELP_TEST,
   testall: HELP_TESTALL,
   migrate: HELP_MIGRATE,
-  server: HELP_SERVER,
-  s: HELP_SERVER,
+  server: _server.HELP_SERVER,
+  s: _server.HELP_SERVER,
   help: HELP
 };