@jskit-ai/jskit-catalog 0.2.35 → 0.2.37

package/scripts/jskit.mjs

@@ -11,6 +11,7 @@ import {
   formatCheckReport,
   formatExportSummary
 } from "./i18n.mjs";
+import { checkDataAccess, formatDataAccessReport } from "./data-access.mjs";
 import { checkMaterial3, formatMaterial3Report, formatNavigationReport } from "./material3.mjs";
 import { checkProject, updateProject } from "./project-packages.mjs";
 import { runTests } from "./test-command.mjs";
@@ -45,7 +46,11 @@ function usage() {
     "               workers do not come from testConcurrency() of @jskit-ai/jskit-catalog/test-budget. A Playwright",
     "               config with a webServer uses browserTestConfig() of @jskit-ai/jskit-catalog/browser-tests (one",
     "               production-build server and database per worker), never a development server, and every one",
-    "               has a globalTimeout.",
+    "               has a globalTimeout. It fails when two table resources (defineTableResource) of the app and its",
+    "               installed JSKIT packages share a table or a name (data/one-resource-per-table), or when a system",
+    "               scope (a feature's systemScope, a systemContext() call) gives no literal reason (data/system-scope),",
+    "               and it lists every system scope and every file the infrastructure exception lifts past the",
+    "               data-access lint rules for review.",
     "  check m3     List application UI source that departs from the JSKIT Material 3 guide, each finding with its",
     "               guide section: the whole app, or only the given files and directories (a section, or a task's",
     "               changed files). Its visual rules (m3/*) are a report; its nav/* rules are plain jskit check's.",
@@ -231,6 +236,9 @@ async function main(argv = process.argv.slice(2)) {
       additionalManifests: options.additionalManifests,
       install: options.install
     });
+    for (const { packageId, replacedBy } of result.retiredPackages) {
+      process.stdout.write(`Removed ${packageId}, which this release retired: ${replacedBy} replaced it.\n`);
+    }
     if (result.changedFiles.length < 1) {
       process.stdout.write(`JSKIT manifests already match catalog ${result.catalogVersion}.\n`);
     } else {
@@ -239,8 +247,18 @@ async function main(argv = process.argv.slice(2)) {
         `${result.changedFiles.map((file) => `- ${file}`).join("\n")}\n`
       );
     }
+    if (result.addedMigrations.length > 0) {
+      process.stdout.write(
+        `The update installed ${result.addedMigrations.length} new package migration(s); run npm run db:migrate before ` +
+        `the app serves requests, since the new packages read what they create:\n` +
+        `${result.addedMigrations.map((entry) => `- ${entry}`).join("\n")}\n`
+      );
+    }
     if (!options.install) {
-      process.stdout.write("Skipped npm install; run it before jskit check.\n");
+      process.stdout.write(
+        "Skipped npm install; run it before jskit check. If it installs new package migrations, run npm run db:migrate " +
+        "before the app serves requests.\n"
+      );
     }
     return;
   }
@@ -253,7 +271,10 @@ async function main(argv = process.argv.slice(2)) {
   });
   const ui = await checkMaterial3({ projectRoot: options.projectRoot, navigationOnly: !MATERIAL3_GATE });
   const tests = await checkTestRuns({ projectRoot: options.projectRoot, manifests: result.manifests });
+  const data = await checkDataAccess({ projectRoot: options.projectRoot });
+  // The data rules first: a graph that fails on many pin versions would otherwise bury them.
   const reports = [
+    formatDataAccessReport(data),
     result.ok
       ? `JSKIT project graph is coordinated across ${result.manifests.length} manifest(s) and ` +
         `${result.installations.length} resolved package installation(s).`
@@ -261,7 +282,7 @@ async function main(argv = process.argv.slice(2)) {
     MATERIAL3_GATE ? formatMaterial3Report(ui) : formatNavigationReport(ui),
     formatTestRunsReport(tests)
   ];
-  if (!result.ok || !ui.ok || !tests.ok) {
+  if (!result.ok || !ui.ok || !tests.ok || !data.ok) {
     throw new Error(reports.join("\n"));
   }
   process.stdout.write(`${reports.join("\n")}\n`);

package/scripts/project-packages.mjs

@@ -1,4 +1,4 @@
-import { realpath, rename, rm, writeFile } from "node:fs/promises";
+import { readdir, readFile, realpath, rename, rm, writeFile } from "node:fs/promises";
 import path from "node:path";
 import { fileExists, toPosixPath } from "./files.mjs";
 import { readOptionalFile, restoreProjectFiles, runNpmInstall, writeManifestUpdates } from "./project-install.mjs";
@@ -13,6 +13,16 @@ const DEPENDENCY_FIELDS = Object.freeze([
   "optionalDependencies"
 ]);
 const UPDATE_SCRIPT = "npx --yes @jskit-ai/jskit-catalog@latest update";
+// JSKIT packages a release retired, each with what replaced it. The updater removes their declarations itself, and
+// refuses while the application's own code still imports one, naming the import to use instead.
+const RETIRED_PACKAGES = Object.freeze({
+  "@jskit-ai/json-rest-api-core": Object.freeze({
+    packageId: "@jskit-ai/resource-runtime",
+    importFrom: "@jskit-ai/resource-runtime/server (its types from @jskit-ai/resource-runtime/server/types)"
+  })
+});
+const SOURCE_FILE_PATTERN = /\.(?:[cm]?[jt]s|vue)$/u;
+const SKIPPED_DIRECTORIES = new Set(["node_modules", "dist", "build", "coverage"]);
 const CHECK_SCRIPT = "jskit check";
 
 function normalizeCatalog(catalog = {}) {
@@ -395,6 +405,65 @@ async function preparePackageLockForInstall(projectRoot, manifests, packageLockP
   }
 }
 
+/**
+ * The application's files, outside node_modules and build output, that import one of the packages.
+ * @param {string} root
+ * @param {string[]} packageIds
+ * @returns {Promise<Map<string, string[]>>} The importing files, relative to root, by package.
+ */
+async function findImports(root, packageIds) {
+  const imports = new Map();
+  const patterns = packageIds.map((packageId) => {
+    const name = packageId.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
+    return [packageId, new RegExp(`["']${name}(?:/[^"']*)?["']`, "u")];
+  });
+  const visit = async (directory) => {
+    for (const entry of await readdir(directory, { withFileTypes: true })) {
+      const absolutePath = path.join(directory, entry.name);
+      if (entry.isDirectory()) {
+        if (!entry.name.startsWith(".") && !SKIPPED_DIRECTORIES.has(entry.name)) {
+          await visit(absolutePath);
+        }
+      } else if (entry.isFile() && SOURCE_FILE_PATTERN.test(entry.name)) {
+        const source = await readFile(absolutePath, "utf8");
+        for (const [packageId, pattern] of patterns) {
+          if (pattern.test(source)) {
+            imports.set(packageId, [...(imports.get(packageId) || []), toPosixPath(path.relative(root, absolutePath))]);
+          }
+        }
+      }
+    }
+  };
+  await visit(root);
+  return imports;
+}
+
+// The migration files the installed @jskit-ai packages ship (jskit.migrations.directories and their constraints/),
+// each "<package>: <path in the package>", as the app's migrations read them from node_modules.
+async function installedPackageMigrations(root) {
+  const scope = path.join(root, "node_modules", JSKIT_SCOPE_PREFIX);
+  const names = await readdir(scope).catch((error) => {
+    if (error?.code === "ENOENT") return [];
+    throw error;
+  });
+  const migrations = new Set();
+  for (const name of names) {
+    const packageRoot = path.join(scope, name);
+    const manifestPath = path.join(packageRoot, "package.json");
+    if (!(await fileExists(manifestPath))) continue;
+    const directories = (await readJsonRecord(manifestPath)).value.jskit?.migrations?.directories;
+    for (const directory of Array.isArray(directories) ? directories : []) {
+      for (const folder of [directory, path.join(directory, "constraints")]) {
+        const files = await readdir(path.join(packageRoot, folder)).catch(() => []);
+        for (const file of files.filter((entry) => entry.endsWith(".cjs"))) {
+          migrations.add(`${JSKIT_SCOPE_PREFIX}${name}: ${toPosixPath(path.join(folder, file))}`);
+        }
+      }
+    }
+  }
+  return migrations;
+}
+
 /**
  * Pins a project's JSKIT dependencies to the catalog and installs them; a
  * failed install or check restores the manifests, lockfile and node_modules.
@@ -414,10 +483,20 @@ async function updateProject({
   const localPackageNames = collectLocalPackageNames(manifests);
   const declarations = collectManifestDeclarations(manifests, root)
     .filter((entry) => !localPackageNames.has(entry.packageId));
+  const retiredPackageIds = [...new Set(
+    declarations.map((entry) => entry.packageId).filter((packageId) => Object.hasOwn(RETIRED_PACKAGES, packageId))
+  )].sort();
+  const retiredImports = retiredPackageIds.length > 0 ? await findImports(root, retiredPackageIds) : new Map();
+  if (retiredImports.size > 0) {
+    throw new Error(`Cannot update while the application imports retired JSKIT packages:\n${[...retiredImports]
+      .map(([packageId, files]) => `- ${packageId} (in ${files.sort().join(", ")}): ${RETIRED_PACKAGES[packageId].packageId} ` +
+        `replaced it; import from ${RETIRED_PACKAGES[packageId].importFrom} instead.`)
+      .join("\n")}`);
+  }
   const unknownPackageIds = [...new Set(
     declarations
       .map((entry) => entry.packageId)
-      .filter((packageId) => !normalizedCatalog.versions.has(packageId))
+      .filter((packageId) => !normalizedCatalog.versions.has(packageId) && !retiredPackageIds.includes(packageId))
   )].sort();
   if (unknownPackageIds.length > 0) {
     throw new Error(`Cannot update JSKIT packages missing from the coordinated catalog: ${unknownPackageIds.join(", ")}.`);
@@ -449,6 +528,11 @@ async function updateProject({
         if (!packageId.startsWith(JSKIT_SCOPE_PREFIX) || localPackageNames.has(packageId)) {
           continue;
         }
+        if (retiredPackageIds.includes(packageId)) {
+          delete next[field][packageId];
+          changed = true;
+          continue;
+        }
         const expectedVersion = normalizedCatalog.versions.get(packageId);
         if (next[field][packageId] !== expectedVersion) {
           next[field][packageId] = expectedVersion;
@@ -486,8 +570,11 @@ async function updateProject({
 
   const packageLockPath = path.join(root, "package-lock.json");
   const packageLockSnapshot = install ? await readOptionalFile(packageLockPath) : null;
+  // npm install replaces node_modules, so the migrations the packages shipped are read before it.
+  const migrationsBefore = install ? await installedPackageMigrations(root) : new Set();
   await writeManifestUpdates(updates);
   let installTransaction = null;
+  let addedMigrations = [];
   try {
     if (install) {
       const updatedManifests = await discoverProjectManifests(root, additionalManifests);
@@ -497,6 +584,7 @@ async function updateProject({
       if (!result.ok) {
         throw new Error(`JSKIT graph remains inconsistent after npm install:\n- ${result.issues.join("\n- ")}`);
       }
+      addedMigrations = [...await installedPackageMigrations(root)].filter((entry) => !migrationsBefore.has(entry)).sort();
       await installTransaction.commit();
     }
   } catch (error) {
@@ -527,7 +615,12 @@ async function updateProject({
     changedFiles: Object.freeze(
       updates.map((update) => toPosixPath(path.relative(root, update.manifest.absolutePath)))
     ),
-    catalogVersion
+    catalogVersion,
+    addedMigrations: Object.freeze(addedMigrations),
+    retiredPackages: Object.freeze(retiredPackageIds.map((packageId) => Object.freeze({
+      packageId,
+      replacedBy: RETIRED_PACKAGES[packageId].packageId
+    })))
   });
 }
 

package/scripts/test-databases.mjs

@@ -3,6 +3,7 @@ import { createRequire } from "node:module";
 import path from "node:path";
 import process from "node:process";
 import { pathToFileURL } from "node:url";
+import { readTableResourceSources } from "./data-access.mjs";
 
 // `jskit test --databases`: each test file running at once gets a database of its own, a copy of TEST_DB_NAME restored
 // from a schema snapshot, from the application's installed @jskit-ai/database-runtime and its knexfile.js, which loads
@@ -13,6 +14,9 @@ import { pathToFileURL } from "node:url";
 const OPTION = "--databases";
 const KNEXFILES = Object.freeze(["knexfile.js", "knexfile.mjs", "knexfile.cjs"]);
 const TEST_DATABASES = "@jskit-ai/database-runtime/server/testDatabases";
+const RESOURCE_RUNTIME = "@jskit-ai/resource-runtime/server";
+const TABLE_RESOURCES = "@jskit-ai/resource-core/shared/tableResource";
+const GUIDE = "https://mobily-enterprises.github.io/jskit-ai/guide/framework/data-access#layer-1-the-table-resource";
 
 /** Runs `load` and puts process.env back as it was, returning what `load` returned and the environment it saw. */
 async function withOwnEnvironment(load) {
@@ -26,6 +30,76 @@ async function withOwnEnvironment(load) {
   }
 }
 
+// A module's exported table resources: each export, or each item of an exported list or object.
+function exportedTableResources(namespace, isTableResource) {
+  return Object.values(namespace).flatMap((value) => {
+    if (isTableResource(value)) return [value];
+    const items = Array.isArray(value) ? value : value && typeof value === "object" ? Object.values(value) : [];
+    return items.filter((item) => isTableResource(item));
+  });
+}
+
+const keyText = ({ tableName, columns, referencedTableName, referencedColumns }) =>
+  `${tableName} (${columns.join(", ")}) -> ${referencedTableName} (${referencedColumns.join(", ")})`;
+
+/**
+ * Compares the foreignKeys the app's table resources and its @jskit-ai packages' declare with the keys TEST_DB_NAME
+ * holds, once a run's databases are ready: json-rest-api relies on a declaration to skip work, so a declared key the
+ * database lacks fails the run. A package's table that the database does not have is left out: node_modules can hold
+ * a package whose migrations the app never runs (one only a devDependency brings, or one left behind), and the app
+ * never reads that table. The app's own tables are always checked. A key of one of the app's own tables that nothing
+ * declares is reported, as work json-rest-api could skip. It imports only the modules that declare foreignKeys; an app
+ * without the resource runtime has nothing to check.
+ * @param {string} cwd
+ * @returns {(knex: object) => Promise<string>}
+ */
+function foreignKeyCheckFor(cwd) {
+  return async (knex) => {
+    const appRequire = createRequire(path.join(cwd, "package.json"));
+    let runtime;
+    let tableResources;
+    try {
+      [runtime, tableResources] = [appRequire.resolve(RESOURCE_RUNTIME), appRequire.resolve(TABLE_RESOURCES)];
+    } catch {
+      return "";
+    }
+    const sources = await readTableResourceSources({ projectRoot: cwd });
+    const tables = sources.filter((source) => !source.framework && source.tableName).map((source) => source.tableName);
+    const declaringFiles = [...new Set(sources.filter((source) => source.foreignKeys).map((source) => source.absolute))];
+    if (tables.length < 1 && declaringFiles.length < 1) return "";
+    const { isTableResource } = await import(pathToFileURL(tableResources).href);
+    const resources = [];
+    const absentPackageTables = new Set();
+    for (const file of declaringFiles) {
+      const declaring = sources.filter((entry) => entry.absolute === file && entry.foreignKeys);
+      const exported = exportedTableResources(await import(pathToFileURL(file).href), isTableResource);
+      for (const source of declaring) {
+        if (!exported.some((resource) => resource.foreignKeys && resource.name === source.name)) {
+          throw new Error(`${source.file}:${source.line} declares foreignKeys on a table resource the module does not export, so ` +
+            `jskit test ${OPTION} cannot check them against the database: export the resource.`);
+        }
+      }
+      for (const resource of exported.filter((entry) => entry.foreignKeys)) {
+        if (declaring[0].framework && !(await knex.schema.hasTable(resource.tableName))) absentPackageTables.add(resource.tableName);
+        else resources.push(resource);
+      }
+    }
+    const { checkForeignKeys } = await import(pathToFileURL(runtime).href);
+    const { missing, undeclared } = await checkForeignKeys({ knex, resources: [...new Set(resources)], tables });
+    if (missing.length > 0) {
+      throw new Error(`jskit test ${OPTION}: ${missing.length} declared foreign key(s) the test database does not hold. ` +
+        `json-rest-api relies on a declared key, so a wrong one changes results: fix the declaration or add the key ` +
+        `with a migration (${GUIDE}).\n` + missing.map((key) => `  ${key.resource}: ${keyText(key)}`).join("\n"));
+    }
+    const declared = [...new Set(resources)].reduce((count, resource) => count + resource.foreignKeys.length, 0);
+    const checked = `foreign keys: ${declared} declared, all in the database` + (absentPackageTables.size < 1 ? "" :
+      ` (${absentPackageTables.size} JSKIT package table(s) the database does not have are not checked)`);
+    if (undeclared.length < 1) return checked;
+    return `${checked}; ${undeclared.length} database key(s) on the app's tables are not declared, so json-rest-api cannot ` +
+      `skip the work they would save (declare each on its table resource's foreignKeys): ${undeclared.map(keyText).join("; ")}`;
+  };
+}
+
 /**
  * The run's databases for `count` files at once, or one file at a time on the connected database.
  * @param {number} count at most the run's test budget, and no more than its test files
@@ -44,7 +118,7 @@ async function testDatabaseRunFor(count, cwd = process.cwd()) {
   const { result: { default: knexConfig, testDatabaseSchema: schema }, environment } = await withOwnEnvironment(async () =>
     import(pathToFileURL(knexfile).href));
   const { testDatabaseRun } = await import(pathToFileURL(testDatabases).href);
-  return testDatabaseRun({ knexConfig, environment, count, schema, appRoot: cwd });
+  return testDatabaseRun({ knexConfig, environment, count, schema, appRoot: cwd, checkDatabase: foreignKeyCheckFor(cwd) });
 }
 
-export { OPTION as DATABASES_OPTION, testDatabaseRunFor };
+export { OPTION as DATABASES_OPTION, foreignKeyCheckFor, testDatabaseRunFor };

package/scripts/verify-packages.mjs

@@ -249,6 +249,17 @@ async function validateMigrationDirectory(packageName, absoluteDirectory, relati
   return entries;
 }
 
+// Each apiSummary surface names an entrypoint the package exports, or a group of them ("./server" for its "./server/..."
+// subpaths), so the catalog never describes an import that fails.
+function validateApiSummarySurfaces(packageJson) {
+  const exported = packageJson.exports && typeof packageJson.exports === "object" ? Object.keys(packageJson.exports) : [];
+  for (const { subpath } of packageJson.jskit?.metadata?.apiSummary?.surfaces || []) {
+    if (!exported.some((entry) => entry === subpath || entry.startsWith(`${subpath}/`))) {
+      throw new Error(`${packageJson.name}#jskit.metadata.apiSummary describes ${subpath}, which the package does not export.`);
+    }
+  }
+}
+
 async function validatePackage(packageRecord, localVersions, migrationOwners) {
   const { packageJson, packageRoot, manifestPath } = packageRecord;
   const packageId = String(packageJson.name || "").trim();
@@ -282,6 +293,7 @@ async function validatePackage(packageRecord, localVersions, migrationOwners) {
     throw new Error(`${packageId} cannot both provide and require the same capability.`);
   }
 
+  validateApiSummarySurfaces(packageJson);
   const serverProviderCount = await validateProviderList(packageRecord, "server");
   await validateProviderList(packageRecord, "client");
   await validateMigrations(packageRecord, migrationOwners);
@@ -337,12 +349,22 @@ function validateCapabilityClosure(packages, { builtinCapabilities = BUILTIN_CAP
   }
 }
 
+// The application's one table runtime: a package that takes it as a dependency can install a second copy of it beside
+// the application's, with its own tables, transactions and error classes. Every package takes it as a peer.
+const TABLE_RUNTIME_PACKAGE_IDS = Object.freeze(["@jskit-ai/database-runtime", "@jskit-ai/resource-runtime"]);
+
 function validateSingletonPeerDependencies(packages) {
   const singletonPackageIds = new Set(collectSingletonPackageIds(packages.map(({ packageJson }) => packageJson)));
 
   for (const { packageJson } of packages) {
     for (const section of ["dependencies", "optionalDependencies"]) {
       for (const dependencyId of Object.keys(packageJson[section] || {})) {
+        if (TABLE_RUNTIME_PACKAGE_IDS.includes(dependencyId)) {
+          throw new Error(
+            `${packageJson.name}#${section}.${dependencyId} must be declared in peerDependencies because ` +
+            `${dependencyId} is the application's one table runtime.`
+          );
+        }
         if (!singletonPackageIds.has(dependencyId)) {
           continue;
         }
@@ -400,6 +422,7 @@ if (process.argv[1] && path.resolve(process.argv[1]) === SCRIPT_PATH) {
 
 export {
   main,
+  validateApiSummarySurfaces,
   validateCapabilityClosure,
   validateMigrations,
   validateProviderExport,