@tabbr/module-cli 0.2.0 → 0.3.0

package/README.md

@@ -85,9 +85,16 @@ export function pollFlights() {
   only by a `publish --local` (e.g. `localhost` for a mock of the external service you develop against) - never by a normal publish.
 - `tabbr.settings.get(key)` / `.set(key, value)` - this module's own per-club settings, including
   its own secrets (never sent back to any settings GET response once saved - `secretFieldNames`).
+- `tabbr.access` - the club's roles (`roles()`), and - with `"manageRoles": true` in the `backend` block - the roles
+  this module gives persons: `setManagedRoles(personId, roleCodes)` makes them exactly these (e.g. groups giving its
+  members a role), `managedRoles(personId)`, `managedAssignments()`. A module only ever touches the roles it gives
+  itself (roles assigned by hand stay), can't give a system role such as ADMIN, and the roles it gave go when it is
+  uninstalled or blocked. The club sees "gives roles" when it installs the module.
 - `triggers[].type: "CRON"` runs on the given cron expression (server-local time), with no
   triggering entity. `"ENTITY_HOOK"` runs on the named `entity`'s `event` (`onCreate`/`onUpdate`/
-  `onDelete`), with that entity bound under its own name (e.g. `flight`) in the handler's scope.
+  `onDelete`), with that entity bound under its own name (e.g. `flight`) in the handler's scope - before the save.
+  `onCreated` / `onUpdated` run after it, once committed, in the background: for a handler that reads the database
+  (e.g. "is this person in that list now"). A module's own writes don't trigger its handlers.
 - Needs `esbuild` installed in the workspace (`npm install -D esbuild`) - only used at publish
   time, bundled into one dependency-free `bundle.js` the server runs in a GraalVM sandbox, so
   regular npm packages can be imported in your TS but nothing Node-specific (no `fs`, no native
@@ -96,6 +103,32 @@ export function pollFlights() {
 A "resync now" button (or similar) can fire a `CRON` trigger manually:
 `POST /api/custommodule/{moduleId}/trigger/{triggerName}` (ADMIN).
 
+## Entities
+
+A module can declare its own **entities** - records Tabbr stores and serves like its own (lists, filters, access
+rules, the club's extra fields, hooks, audit). Add an `"entities"` array to `tabbr.config.json`:
+
+```json
+{
+  "entities": [
+    {
+      "name": "group",
+      "label": "Groep",
+      "display": ["name"],
+      "fields": [
+        { "key": "name", "label": "Naam", "type": "tabbr.text", "includeInSearch": true },
+        { "key": "parent", "label": "Bovenliggende groep", "type": "tabbr.entity", "target": "group" }
+      ]
+    }
+  ]
+}
+```
+
+`publish` checks the declarations (names, keys, types) before uploading and sends them in the backend manifest - also
+for a module that has no backend code (then only the entities are published). The frontend addresses records by field
+key with the SDK's `injectModuleEntity()`, the backend with `tabbr.entities.<name>`. See `docs/module-entities.md` in the
+Tabbr repository for the field types, access rules and what the club gets; `tabbr-groups-module/` is a complete example.
+
 ## Usage: publish
 
 ```
@@ -135,6 +168,46 @@ doesn't do on its own (`NODE_EXTRA_CA_CERTS` is set automatically; if mkcert's r
 set it yourself - `mkcert -CAROOT` shows where it lives). `--club` picks the club in that environment;
 login still goes through the real Keycloak, which local backends validate too.
 
+### Publishing to the module store
+
+`npx tabbr-module publish --store [--changelog "..."]` submits a new **version of a store
+package** instead of publishing it into one club (see `docs/module-store.md` in the Tabbr repo):
+the workspace's **source** - the files git tracks (plus new ones git doesn't ignore), never
+`node_modules` or build output - with its `package-lock.json`. The store builds it with the
+platform's toolchain (`tabbr-module-builder`: its own `@tabbr/module-sdk` and `-cli`, your locked
+dependencies, your `test` and `typecheck` scripts, no network during the build); then it goes into
+review; once approved, every club can install it under *Instellingen › Modules › Store*. Unlink
+yalc first (`npm run unlink-local`): the store installs from the lockfile. `--prebuilt` uploads a
+local build instead (trusted publishers only). For now only platform admins (`SUPER_ADMIN`) can
+submit.
+
+`npx tabbr-module build --out module.zip` builds the module into a zip without publishing it -
+what the store's builder runs.
+
+It needs, in `tabbr.config.json`:
+
+```jsonc
+{
+  "moduleId": "groups",
+  "package": "tabbr/groups",            // <publisher>/<name>
+  "version": "1.2.0",                   // higher than every earlier version of the package
+  "engines": { "tabbr": "^1.0" },       // the module API levels it runs on (MODULE_API_LEVEL in the SDK)
+  "dependencies": { },                  // optional: { "acme/payments": "^1", "sys.messaging": "*" }
+  "optionalDependencies": { "sys.messaging": "*" },
+  "permissions": {                      // shown to the club before it installs or updates
+    "entities": { "read": ["person"], "write": [] },
+    "personalData": ["contact"],
+    "messaging": true
+  },
+  "summary": "One line for the store",
+  "visibility": "PUBLIC"                // or UNLISTED
+}
+```
+
+The server refuses a version that can never work: a version not higher than the last, `engines`
+that don't include the server's module API level, a changed `moduleId`, or - within the same major
+version - entity declarations that remove or change something (a new major may).
+
 ## Usage: fetch-types
 
 ```

package/bin/tabbr-module.js

@@ -5,7 +5,8 @@ const fs = require('fs');
 const path = require('path');
 const { getAccessToken } = require('../src/auth');
 const { buildAndZip } = require('../src/build');
-const { publish } = require('../src/publish');
+const { publish, publishToStore, submitSourceToStore, storeMeta } = require('../src/publish');
+const { packSource } = require('../src/source');
 const { fetchScriptTypes, fetchAllEntityTypes, VALID_KINDS } = require('../src/fetch-types');
 const { CONFIG_FILE, DEFAULT_API_HOST, readConfig, requireClub, runInit } = require('../src/config');
 const { DEFAULT_LOCAL_API_HOST, trustLocalCertificateOrRespawn } = require('../src/local');
@@ -29,6 +30,9 @@ function parseArgs(argv) {
       case '--entity': args.entity = next(); break;
       case '--kind': args.kind = next(); break;
       case '--out': args.out = next(); break;
+      case '--store': args.store = true; break;
+      case '--prebuilt': args.prebuilt = true; break;
+      case '--changelog': args.changelog = next(); break;
       default:
         throw new Error(`Unknown option: ${arg}`);
     }
@@ -80,6 +84,24 @@ Optional:
 Requires the tabbr Keycloak client's Valid Redirect URIs to include http://localhost:*
 (a Keycloak admin-console setting - see README.md).
 
+Usage: tabbr-module publish --store [--changelog <text>] [--prebuilt] [options]
+
+Submits a new version of a package to the module store (docs/module-store.md) instead of
+publishing it into one club: the workspace's SOURCE (git-tracked files, with package-lock.json).
+The store builds it with the platform's toolchain, then it waits for review; once approved,
+clubs install it from Instellingen > Modules. --prebuilt uploads a local build instead (only
+for trusted publishers). No --club needed. Needs, in ${CONFIG_FILE}:
+  package                    "<publisher>/<name>", e.g. "tabbr/groups"
+  version                    "1.4.0" - higher than every earlier version of the package
+  engines                    { "tabbr": "^1.0" } - the module API levels it runs on
+and optionally dependencies, optionalDependencies, permissions, summary, visibility (PUBLIC or
+UNLISTED) - see the groups module's ${CONFIG_FILE} for an example.
+
+Usage: tabbr-module build --out <module.zip> [--cwd <path>]
+
+Builds the module (frontend and/or backend part, as publish does) into a zip without
+publishing it - what the store's builder runs.
+
 Usage: tabbr-module fetch-types [--club <CODE>] [--entity <name>] [options]
 
 With --entity: fetches the ambient .d.ts for that ONE entity's script sandbox - the same typings
@@ -113,6 +135,10 @@ function requireFlags(args, requiredFlags) {
 }
 
 async function runPublish(args) {
+  if (args.store) {
+    await runPublishToStore(args);
+    return;
+  }
   args.club = requireClub(args.club);
   requireFlags(args, {
     moduleId: '--module-id',
@@ -139,6 +165,33 @@ async function runPublish(args) {
   console.log(JSON.stringify(result, null, 2));
 }
 
+async function runPublishToStore(args) {
+  requireFlags(args, { moduleId: '--module-id' });
+  const meta = storeMeta(args);
+  let version;
+  if (args.prebuilt) {
+    const zipBuffer = await buildAndZip(args.cwd, args);
+    const accessToken = await getAccessToken();
+    version = await publishToStore({ apiHost: args.apiHost, packageName: args.package, meta, zipBuffer, accessToken });
+  } else {
+    const { zip, files } = await packSource(args.cwd);
+    console.log(`Source: ${files.length} files, ${Math.round(zip.length / 1024)} KB`);
+    const accessToken = await getAccessToken();
+    version = await submitSourceToStore({ apiHost: args.apiHost, packageName: args.package, meta, sourceZip: zip, accessToken });
+  }
+  console.log(`Submitted ${version.packageName} ${version.version} - ${version.state}.` +
+    (version.state === 'BUILDING' ? ' The store builds it; then it is reviewed, and installable once approved.' : ' It can be installed once it is approved.'));
+}
+
+async function runBuild(args) {
+  requireFlags(args, { out: '--out' });
+  const zipBuffer = await buildAndZip(args.cwd, args);
+  const out = path.resolve(args.cwd, args.out);
+  fs.mkdirSync(path.dirname(out), { recursive: true });
+  fs.writeFileSync(out, zipBuffer);
+  console.log(`Wrote ${out} (${Math.round(zipBuffer.length / 1024)} KB)`);
+}
+
 async function fetchAndWriteOne({ apiHost, club, entity, kind, cwd, out, accessToken }) {
   const dts = await fetchScriptTypes({ apiHost, club, entity, kind, accessToken });
 
@@ -219,7 +272,7 @@ async function runFetchTypes(args) {
 async function main() {
   const [command, ...rest] = process.argv.slice(2);
 
-  if (command !== 'init' && command !== 'publish' && command !== 'fetch-types') {
+  if (!['init', 'publish', 'build', 'fetch-types'].includes(command)) {
     printUsage();
     process.exit(command ? 1 : 0);
   }
@@ -238,6 +291,8 @@ async function main() {
   trustLocalCertificateOrRespawn(args.apiHost);
   if (command === 'publish') {
     await runPublish(args);
+  } else if (command === 'build') {
+    await runBuild(args);
   } else {
     await runFetchTypes(args);
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tabbr/module-cli",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "Set up, build and publish a Native Federation club module to a tabbr club.",
   "license": "SEE LICENSE IN LICENSE",
   "author": "SystemWorks BV",

package/src/backend-build.js

@@ -9,6 +9,43 @@ const DEFAULT_ENTRY = 'backend/src/index.ts';
  * in the module's TS ends up reachable as the global `__module.foo` once bundled this way. */
 const GLOBAL_NAME = '__module';
 
+/**
+ * The entities the module declares (`"entities"` in tabbr.config.json), checked just enough to fail here rather than at
+ * upload with a vaguer message: Tabbr validates them again against the club (names taken, entity fields pointing at
+ * entities the club has, ...). See `ModuleEntityDefinition` in tabbr-api-server for the full shape.
+ */
+function entitiesOf(config) {
+  const entities = config.entities || [];
+  if (!Array.isArray(entities)) {
+    throw new Error('"entities" in tabbr.config.json must be an array.');
+  }
+  for (const entity of entities) {
+    if (!entity || !/^[a-z][a-zA-Z0-9]{1,39}$/.test(entity.name || '')) {
+      throw new Error(`Entity name "${entity && entity.name}" is not valid: camelCase letters and digits, starting lowercase.`);
+    }
+    if (!Array.isArray(entity.fields) || entity.fields.length === 0) {
+      throw new Error(`Entity "${entity.name}" declares no fields.`);
+    }
+    const keys = new Set();
+    for (const field of entity.fields) {
+      if (!/^[a-z][a-zA-Z0-9]{0,39}$/.test(field.key || '')) {
+        throw new Error(`Entity "${entity.name}": field key "${field.key}" is not valid.`);
+      }
+      if (keys.has(field.key)) {
+        throw new Error(`Entity "${entity.name}": field "${field.key}" is declared twice.`);
+      }
+      keys.add(field.key);
+      if (!field.type && !field.persistentType) {
+        throw new Error(`Entity "${entity.name}": field "${field.key}" has no type (e.g. "tabbr.text").`);
+      }
+      if (field.type === 'tabbr.entity' && !field.target) {
+        throw new Error(`Entity "${entity.name}": entity field "${field.key}" has no "target".`);
+      }
+    }
+  }
+  return entities;
+}
+
 /**
  * Bundles a module workspace's backend part (if configured) into a single, dependency-free
  * `bundle.js`, using esbuild so module authors can write normal TypeScript (types, imports,
@@ -16,14 +53,35 @@ const GLOBAL_NAME = '__module';
  * server evaluates it in a sandboxed GraalVM context with no module resolution of its own (see
  * `BackendModuleRuntimeService`).
  *
- * Returns `null` when the workspace has no backend part configured, so `build.js` can skip it
- * entirely for a frontend-only module.
+ * Also writes the manifest the server reads (`manifest.json`): the backend config plus the entities the module declares
+ * - which a module can do without any backend code, so then only the manifest is produced (`bundleJs` null).
+ *
+ * Returns `null` when the workspace has neither a backend part nor entities, so `build.js` can skip
+ * it entirely for a frontend-only module.
  */
 async function buildBackend(cwd, config) {
   const backendConfig = config.backend;
-  if (!backendConfig) {
+  const entities = entitiesOf(config);
+  const bundleJs = backendConfig ? await bundle(cwd, backendConfig) : null;
+  if (bundleJs == null && entities.length === 0) {
     return null;
   }
+  const settings = backendConfig || {};
+  const manifest = {
+    // localAllowedHosts (e.g. a mock on localhost) only ever go into a --local publish, never into one for a real environment.
+    allowedHosts: [...(settings.allowedHosts || []), ...(config.local ? settings.localAllowedHosts || [] : [])],
+    secretFieldNames: settings.secretFieldNames || [],
+    triggers: bundleJs == null ? [] : settings.triggers || [],
+    entities,
+    // the module gives persons roles (tabbr.access.setManagedRoles) - shown to the club when it installs the module
+    manageRoles: bundleJs != null && settings.manageRoles === true,
+  };
+
+  return { bundleJs, manifestJson: JSON.stringify(manifest, null, 2) };
+}
+
+/** The backend entry bundled into one script, or null when there is no entry file (and none was configured explicitly). */
+async function bundle(cwd, backendConfig) {
   const entry = backendConfig.entry || DEFAULT_ENTRY;
   const entryPath = path.join(cwd, entry);
   if (!fs.existsSync(entryPath)) {
@@ -59,16 +117,7 @@ async function buildBackend(cwd, config) {
   if (!result.outputFiles || result.outputFiles.length === 0) {
     throw new Error(`esbuild produced no output for ${entryPath}`);
   }
-  const bundleJs = result.outputFiles[0].text;
-
-  const manifest = {
-    // localAllowedHosts (e.g. a mock on localhost) only ever go into a --local publish, never into one for a real environment.
-    allowedHosts: [...(backendConfig.allowedHosts || []), ...(config.local ? backendConfig.localAllowedHosts || [] : [])],
-    secretFieldNames: backendConfig.secretFieldNames || [],
-    triggers: backendConfig.triggers || [],
-  };
-
-  return { bundleJs, manifestJson: JSON.stringify(manifest, null, 2) };
+  return result.outputFiles[0].text;
 }
 
-module.exports = { buildBackend, DEFAULT_ENTRY };
+module.exports = { buildBackend, entitiesOf, DEFAULT_ENTRY };

package/src/build.js

@@ -78,7 +78,9 @@ function zipModule(frontendDir, backend) {
       archive.directory(frontendDir, false);
     }
     if (backend) {
-      archive.append(backend.bundleJs, { name: 'backend/bundle.js' });
+      if (backend.bundleJs != null) {
+        archive.append(backend.bundleJs, { name: 'backend/bundle.js' });
+      }
       archive.append(backend.manifestJson, { name: 'backend/manifest.json' });
     }
     archive.finalize();

package/src/publish.js

@@ -28,4 +28,76 @@ async function publish({ apiHost, club, moduleId, exposedModule, exportName, ico
   return response.json();
 }
 
-module.exports = { publish };
+/**
+ * Submits the zipped build as a new version of a store package (docs/module-store.md): POST
+ * /api/store/packages/<publisher>/<name>/versions with the build and the version's metadata. The
+ * version then waits for review; clubs install it from the store once it is approved.
+ */
+async function publishToStore({ apiHost, packageName, meta, zipBuffer, accessToken }) {
+  const form = new FormData();
+  form.append('file', new Blob([zipBuffer], { type: 'application/zip' }), 'module.zip');
+  form.append('meta', new Blob([JSON.stringify(meta)], { type: 'application/json' }));
+
+  const url = `${apiHost}/api/store/packages/${packageName}/versions`;
+  console.log(`Submitting ${packageName} ${meta.version} to the module store at ${url} ...`);
+  const response = await fetch(url, {
+    method: 'POST',
+    headers: { Authorization: `Bearer ${accessToken}` },
+    body: form,
+  });
+  if (!response.ok) {
+    throw new Error(`Submit failed: ${response.status} ${await response.text()}`);
+  }
+  return response.json();
+}
+
+/** The version's metadata for the store, from tabbr.config.json (and the command line). */
+function storeMeta(args) {
+  const missing = [];
+  if (!args.package) missing.push('"package" (e.g. "acme/groups")');
+  if (!args.version) missing.push('"version" (e.g. "1.0.0")');
+  if (!args.engines || !args.engines.tabbr) missing.push('"engines": { "tabbr": "^1.0" } (the module API levels it runs on)');
+  if (missing.length) {
+    throw new Error(`Missing in tabbr.config.json for --store: ${missing.join(', ')}`);
+  }
+  return {
+    version: args.version,
+    moduleId: args.moduleId,
+    exposedModule: args.exposedModule,
+    exportName: args.exportName,
+    title: args.title,
+    icon: args.icon,
+    summary: args.summary,
+    changelog: args.changelog,
+    visibility: args.visibility,
+    engines: args.engines,
+    dependencies: args.dependencies || {},
+    optionalDependencies: args.optionalDependencies || {},
+    permissions: args.permissions || {},
+  };
+}
+
+/**
+ * Submits the workspace's source as a new version of a store package: POST
+ * /api/store/packages/<publisher>/<name>/sources. The store builds it (tabbr-module-builder) and
+ * only then puts it in review.
+ */
+async function submitSourceToStore({ apiHost, packageName, meta, sourceZip, accessToken }) {
+  const form = new FormData();
+  form.append('source', new Blob([sourceZip], { type: 'application/zip' }), 'source.zip');
+  form.append('meta', new Blob([JSON.stringify(meta)], { type: 'application/json' }));
+
+  const url = `${apiHost}/api/store/packages/${packageName}/sources`;
+  console.log(`Submitting the source of ${packageName} ${meta.version} to the module store at ${url} ...`);
+  const response = await fetch(url, {
+    method: 'POST',
+    headers: { Authorization: `Bearer ${accessToken}` },
+    body: form,
+  });
+  if (!response.ok) {
+    throw new Error(`Submit failed: ${response.status} ${await response.text()}`);
+  }
+  return response.json();
+}
+
+module.exports = { publish, publishToStore, submitSourceToStore, storeMeta };

package/src/source.js

@@ -0,0 +1,70 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const { execFileSync } = require('child_process');
+const archiver = require('archiver');
+
+/** Never part of a module's source: installed packages, build output, caches, editor state. */
+const SKIP = new Set(['node_modules', 'dist', '.angular', 'out-tsc', '.git', '.yalc', '.idea', '.vscode', 'coverage']);
+
+/**
+ * The workspace's source files, as the module store builds them: the files git tracks (plus new ones git doesn't ignore)
+ * when the workspace is in a git repository, otherwise every file outside the SKIP folders. Paths relative to `cwd`.
+ */
+function sourceFiles(cwd) {
+  let listed = null;
+  try {
+    const out = execFileSync('git', ['ls-files', '-z', '--cached', '--others', '--exclude-standard', '--', '.'],
+      { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] });
+    listed = out.split('\0').filter(Boolean);
+  } catch {
+    // not a git repository (or no git): walk the folder instead
+  }
+  const files = listed ?? walk(cwd, '');
+  return files
+    .filter(file => !file.split('/').some(part => SKIP.has(part)))
+    .filter(file => fs.existsSync(path.join(cwd, file)) && fs.statSync(path.join(cwd, file)).isFile())
+    .sort();
+}
+
+function walk(root, rel) {
+  const result = [];
+  for (const entry of fs.readdirSync(path.join(root, rel), { withFileTypes: true })) {
+    if (SKIP.has(entry.name)) continue;
+    const child = rel ? `${rel}/${entry.name}` : entry.name;
+    if (entry.isDirectory()) result.push(...walk(root, child));
+    else if (entry.isFile()) result.push(child);
+  }
+  return result;
+}
+
+/** A zip of the workspace's source (see sourceFiles), checked to have what the store needs to build it. */
+function packSource(cwd) {
+  const files = sourceFiles(cwd);
+  for (const required of ['package.json', 'package-lock.json']) {
+    if (!files.includes(required)) {
+      throw new Error(`${required} is missing in ${cwd}` +
+        (required === 'package-lock.json' ? ' - the store installs exactly the locked dependencies; run "npm install" and commit it' : ''));
+    }
+  }
+  // yalc-linked packages exist only on this machine: the store uses the platform's own SDK and CLI instead
+  const lock = fs.readFileSync(path.join(cwd, 'package-lock.json'), 'utf8');
+  if (lock.includes('file:.yalc/')) {
+    throw new Error('package-lock.json links @tabbr packages with yalc ("npm run link-local") - run "npm run unlink-local" ' +
+      'first: the store builds against the published packages and replaces @tabbr/module-sdk and -cli with the platform\'s own.');
+  }
+  return new Promise((resolve, reject) => {
+    const archive = archiver('zip', { zlib: { level: 9 } });
+    const chunks = [];
+    archive.on('data', chunk => chunks.push(chunk));
+    archive.on('end', () => resolve({ zip: Buffer.concat(chunks), files }));
+    archive.on('error', reject);
+    for (const file of files) {
+      archive.file(path.join(cwd, file), { name: file });
+    }
+    archive.finalize();
+  });
+}
+
+module.exports = { packSource, sourceFiles };